Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

19.9 KB
Raw Blame
use std::collections::HashMap;
use std::ops::Add;
use std::sync::atomic::{AtomicU8, Ordering};
use std::sync::{Arc, RwLock};

use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use std::time::Duration;
use tokio::runtime;
use tokio::sync::mpsc;
use tracing::{debug, warn};

/// Shutdown status state machine for graceful rollout/termination.
///
/// During Kubernetes rolling deployments, the pre-stop hook creates `/tmp/shutdown`
/// to signal the pod is being terminated. The readiness probe detects this and
/// transitions to `Prestop`, returning 503 to stop receiving new traffic while
/// existing requests complete.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
#[repr(u8)]
pub enum ShutdownStatus {
    Unknown = 0,
    Running = 1,
    Prestop = 2,
    Terminating = 3,
    Completed = 4,
}

impl ShutdownStatus {
    pub fn as_str(self) -> &'static str {
        match self {
            Self::Unknown => "unknown",
            Self::Running => "running",
            Self::Prestop => "prestop",
            Self::Terminating => "terminating",
            Self::Completed => "completed",
        }
    }
}

impl From<u8> for ShutdownStatus {
    fn from(v: u8) -> Self {
        match v {
            1 => Self::Running,
            2 => Self::Prestop,
            3 => Self::Terminating,
            4 => Self::Completed,
            _ => Self::Unknown,
        }
    }
}

static SHUTDOWN_STATUS: AtomicU8 = AtomicU8::new(ShutdownStatus::Running as u8);

pub fn set_shutdown_status(status: ShutdownStatus) {
    SHUTDOWN_STATUS.store(status as u8, Ordering::Relaxed);
}

pub fn get_shutdown_status() -> ShutdownStatus {
    SHUTDOWN_STATUS.load(Ordering::Relaxed).into()
}

/// Readiness check that returns 503 when shutting down.
///
/// This endpoint should be used as the Kubernetes readiness probe. When the
/// pre-stop hook creates `/tmp/shutdown`, subsequent readiness checks will
/// return 503, causing Kubernetes to stop routing traffic to this pod.
pub async fn readiness_handler() -> StatusCode {
    let shutdown_status = get_shutdown_status();
    let is_running_or_unknown =
        shutdown_status == ShutdownStatus::Running || shutdown_status == ShutdownStatus::Unknown;

    if is_running_or_unknown && std::path::Path::new("/tmp/shutdown").exists() {
        set_shutdown_status(ShutdownStatus::Prestop);
        tracing::info!("Shutdown file detected, transitioning to PRESTOP status");
    }

    if is_running_or_unknown {
        StatusCode::OK
    } else {
        StatusCode::SERVICE_UNAVAILABLE
    }
}

/// Health reporting for components of the service.
///
/// The capture server contains several asynchronous loops, and
/// the process can only be trusted with user data if all the
/// loops are properly running and reporting.
///
/// HealthRegistry allows an arbitrary number of components to
/// be registered and report their health. The process' health
/// status is the combination of these individual health status:
///   - if any component is unhealthy, the process is unhealthy
///   - if all components recently reported healthy, the process is healthy
///   - if a component failed to report healthy for its defined deadline,
///     it is considered unhealthy, and the check fails.
///
/// Trying to merge the k8s concepts of liveness and readiness in
/// a single state is full of foot-guns, so HealthRegistry does not
/// try to do it. Each probe should have its separate instance of
/// the registry to avoid confusions.

#[derive(Default, Debug)]
pub struct HealthStatus {
    /// The overall status: true of all components are healthy
    pub healthy: bool,
    /// Current status of each registered component, for display
    pub components: HashMap<String, ComponentStatus>,
}
impl IntoResponse for HealthStatus {
    /// Computes the axum status code based on the overall health status,
    /// and prints each component status in the body for debugging.
    fn into_response(self) -> Response {
        let body = format!("{self:?}");
        match self.healthy {
            true => (StatusCode::OK, body),
            false => (StatusCode::INTERNAL_SERVER_ERROR, body),
        }
        .into_response()
    }
}

#[derive(Debug, Clone, Eq, PartialEq)]
pub enum ComponentStatus {
    /// Automatically set when a component is newly registered
    Starting,
    /// Recently reported healthy, will need to report again before the date
    HealthyUntil(time::OffsetDateTime),
    /// Reported unhealthy
    Unhealthy,
    /// Automatically set when the HealthyUntil deadline is reached
    Stalled,
}

impl ComponentStatus {
    /// Returns true if the component is currently healthy (i.e., has a valid HealthyUntil status)
    pub fn is_healthy(&self) -> bool {
        match self {
            ComponentStatus::HealthyUntil(until) => until.gt(&time::OffsetDateTime::now_utc()),
            _ => false,
        }
    }
}

struct HealthMessage {
    component: String,
    status: ComponentStatus,
}

#[derive(Clone)]
pub struct HealthHandle {
    component: String,
    deadline: Duration,
    sender: mpsc::Sender<HealthMessage>,
}

impl HealthHandle {
    /// Asynchronously report healthy, returns when the message is queued.
    /// Must be called more frequently than the configured deadline.
    pub async fn report_healthy(&self) {
        self.report_status(ComponentStatus::HealthyUntil(
            time::OffsetDateTime::now_utc().add(self.deadline),
        ))
        .await
    }

    /// Asynchronously report component status, returns when the message is queued.
    pub async fn report_status(&self, status: ComponentStatus) {
        let message = HealthMessage {
            component: self.component.clone(),
            status,
        };
        if let Err(err) = self.sender.send(message).await {
            warn!("failed to report heath status: {}", err)
        }
    }

    /// Synchronously report as healthy, returns when the message is queued.
    /// Must be called more frequently than the configured deadline.
    pub fn report_healthy_blocking(&self) {
        self.report_status_blocking(ComponentStatus::HealthyUntil(
            time::OffsetDateTime::now_utc().add(self.deadline),
        ))
    }

    /// Synchronously report component status, returns when the message is queued.
    pub fn report_status_blocking(&self, status: ComponentStatus) {
        let message = HealthMessage {
            component: self.component.clone(),
            status,
        };
        // Don't panic if we're called from within an async context,
        // just spawn instead
        if let Ok(h) = runtime::Handle::try_current() {
            let m = self.clone();
            h.spawn(async move { m.report_status(message.status).await });
        } else if let Err(err) = self.sender.blocking_send(message) {
            warn!("failed to report heath status: {}", err)
        }
    }
}

impl common_liveness::SyncLivenessReporter for HealthHandle {
    fn report_healthy(&self) {
        self.report_healthy_blocking();
    }

    fn report_unhealthy(&self) {
        self.report_status_blocking(ComponentStatus::Unhealthy);
    }
}

#[derive(Clone, Debug, PartialEq)]
pub enum HealthStrategy {
    /// All components must be healthy for the registry to be healthy
    All,
    /// At least one component must be healthy for the registry to be healthy
    Any,
}

impl std::str::FromStr for HealthStrategy {
    type Err = String;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.trim().to_lowercase().as_ref() {
            "all" => Ok(HealthStrategy::All),
            "any" => Ok(HealthStrategy::Any),
            _ => Err(format!("Unknown Health Strategy: {s}, must be ALL or ANY")),
        }
    }
}

#[derive(Clone)]
pub struct HealthRegistry {
    name: String,
    strategy: HealthStrategy,
    components: Arc<RwLock<HashMap<String, ComponentStatus>>>,
    sender: mpsc::Sender<HealthMessage>,
}

impl HealthRegistry {
    pub fn new(name: &str) -> Self {
        Self::new_with_strategy(name, HealthStrategy::All)
    }

    pub fn new_with_strategy(name: &str, strategy: HealthStrategy) -> Self {
        let (tx, mut rx) = mpsc::channel::<HealthMessage>(16);
        let registry = Self {
            name: name.to_owned(),
            strategy,
            components: Default::default(),
            sender: tx,
        };

        let components = registry.components.clone();
        tokio::spawn(async move {
            while let Some(message) = rx.recv().await {
                if let Ok(mut map) = components.write() {
                    _ = map.insert(message.component, message.status);
                } else {
                    // Poisoned mutex: Just warn, the probes will fail and the process restart
                    warn!("poisoned HeathRegistry mutex")
                }
            }
        });

        registry
    }

    /// Registers a new component in the registry. The returned handle should be passed
    /// to the component, to allow it to frequently report its health status.
    pub async fn register<D>(&self, component: String, deadline: D) -> HealthHandle
    where
        // HACK: to let callers user time::Duration or std::time::Duration (and therefore chrono::Duration),
        // since apparently we use all three
        D: TryInto<Duration>,
    {
        let Ok(deadline) = deadline.try_into() else {
            // TODO - I should return an error here, but I don't want to refactor everything that uses this right now
            panic!("invalid deadline")
        };
        let handle = HealthHandle {
            component,
            deadline,
            sender: self.sender.clone(),
        };
        handle.report_status(ComponentStatus::Starting).await;
        handle
    }

    /// Returns the overall process status, computed from the status of all the components
    /// currently registered. Can be used as an axum handler.
    pub fn get_status(&self) -> HealthStatus {
        let components = self
            .components
            .read()
            .expect("poisoned HeathRegistry mutex");

        let result = HealthStatus {
            // unhealthy if no component has registered yet or if we're using the "Any" strategy
            // "All" defaults to true and is set to false if any healthcheck fails
            // "Any" defaults to false and is set to true if any healthcheck passes
            healthy: !components.is_empty() && self.strategy == HealthStrategy::All,
            components: Default::default(),
        };
        let now = time::OffsetDateTime::now_utc();

        let result = components
            .iter()
            .fold(result, |mut result, (name, status)| {
                match status {
                    ComponentStatus::HealthyUntil(until) => {
                        if until.gt(&now) {
                            if self.strategy == HealthStrategy::Any {
                                result.healthy = true;
                            }
                            _ = result.components.insert(name.clone(), status.clone())
                        } else {
                            if self.strategy == HealthStrategy::All {
                                result.healthy = false;
                            }
                            _ = result
                                .components
                                .insert(name.clone(), ComponentStatus::Stalled)
                        }
                    }
                    _ => {
                        if self.strategy == HealthStrategy::All {
                            result.healthy = false;
                        }
                        _ = result.components.insert(name.clone(), status.clone())
                    }
                }
                result
            });
        match result.healthy {
            true => debug!("{} health check ok", self.name),
            false => warn!("{} health check failed: {:?}", self.name, result.components),
        }
        result
    }
}

#[cfg(test)]
mod tests {
    use crate::{
        get_shutdown_status, set_shutdown_status, ComponentStatus, HealthRegistry, HealthStatus,
        HealthStrategy, ShutdownStatus,
    };
    use axum::http::StatusCode;
    use axum::response::IntoResponse;
    use std::ops::{Add, Sub};
    use time::{Duration, OffsetDateTime};

    async fn assert_or_retry<F>(check: F)
    where
        F: Fn() -> bool,
    {
        assert_or_retry_for_duration(check, Duration::seconds(5)).await
    }

    async fn assert_or_retry_for_duration<F>(check: F, timeout: Duration)
    where
        F: Fn() -> bool,
    {
        let deadline = OffsetDateTime::now_utc().add(timeout);
        while !check() && OffsetDateTime::now_utc().lt(&deadline) {
            tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
        }
        assert!(check())
    }
    #[tokio::test]
    async fn defaults_to_unhealthy() {
        let registry = HealthRegistry::new("liveness");
        assert!(!registry.get_status().healthy);
    }

    #[tokio::test]
    async fn one_component() {
        let registry = HealthRegistry::new("liveness");

        // New components are registered in Starting
        let handle = registry
            .register("one".to_string(), Duration::seconds(30))
            .await;
        assert_or_retry(|| registry.get_status().components.len() == 1).await;
        let mut status = registry.get_status();
        assert!(!status.healthy);
        assert_eq!(
            status.components.get("one"),
            Some(&ComponentStatus::Starting)
        );

        // Status goes healthy once the component reports
        handle.report_healthy().await;
        assert_or_retry(|| registry.get_status().healthy).await;
        status = registry.get_status();
        assert_eq!(status.components.len(), 1);

        // Status goes unhealthy if the components says so
        handle.report_status(ComponentStatus::Unhealthy).await;
        assert_or_retry(|| !registry.get_status().healthy).await;
        status = registry.get_status();
        assert_eq!(status.components.len(), 1);
        assert_eq!(
            status.components.get("one"),
            Some(&ComponentStatus::Unhealthy)
        );
    }

    #[tokio::test]
    async fn staleness_check() {
        let registry = HealthRegistry::new("liveness");
        let handle = registry
            .register("one".to_string(), Duration::seconds(30))
            .await;

        // Status goes healthy once the component reports
        handle.report_healthy().await;
        assert_or_retry(|| registry.get_status().healthy).await;
        let mut status = registry.get_status();
        assert_eq!(status.components.len(), 1);

        // If the component's ping is too old, it is considered stalled and the healthcheck fails
        // FIXME: we should mock the time instead
        handle
            .report_status(ComponentStatus::HealthyUntil(
                OffsetDateTime::now_utc().sub(Duration::seconds(1)),
            ))
            .await;
        assert_or_retry(|| !registry.get_status().healthy).await;
        status = registry.get_status();
        assert_eq!(status.components.len(), 1);
        assert_eq!(
            status.components.get("one"),
            Some(&ComponentStatus::Stalled)
        );
    }

    #[tokio::test]
    async fn several_components() {
        let registry = HealthRegistry::new("liveness");
        let handle1 = registry
            .register("one".to_string(), Duration::seconds(30))
            .await;
        let handle2 = registry
            .register("two".to_string(), Duration::seconds(30))
            .await;
        assert_or_retry(|| registry.get_status().components.len() == 2).await;

        // First component going healthy is not enough
        handle1.report_healthy().await;
        assert_or_retry(|| {
            registry.get_status().components.get("one").unwrap() != &ComponentStatus::Starting
        })
        .await;
        assert!(!registry.get_status().healthy);

        // Second component going healthy brings the health to green
        handle2.report_healthy().await;
        assert_or_retry(|| {
            registry.get_status().components.get("two").unwrap() != &ComponentStatus::Starting
        })
        .await;
        assert!(registry.get_status().healthy);

        // First component going unhealthy takes down the health to red
        handle1.report_status(ComponentStatus::Unhealthy).await;
        assert_or_retry(|| !registry.get_status().healthy).await;

        // First component recovering returns the health to green
        handle1.report_healthy().await;
        assert_or_retry(|| registry.get_status().healthy).await;

        // Second component going unhealthy takes down the health to red
        handle2.report_status(ComponentStatus::Unhealthy).await;
        assert_or_retry(|| !registry.get_status().healthy).await;
    }

    #[tokio::test]
    async fn into_response() {
        let nok = HealthStatus::default().into_response();
        assert_eq!(nok.status(), StatusCode::INTERNAL_SERVER_ERROR);

        let ok = HealthStatus {
            healthy: true,
            components: Default::default(),
        }
        .into_response();
        assert_eq!(ok.status(), StatusCode::OK);
    }

    #[tokio::test]
    async fn any_strategy() {
        let registry = HealthRegistry::new_with_strategy("liveness", HealthStrategy::Any);
        let handle1 = registry
            .register("one".to_string(), Duration::seconds(30))
            .await;
        let handle2 = registry
            .register("two".to_string(), Duration::seconds(30))
            .await;
        assert_or_retry(|| registry.get_status().components.len() == 2).await;

        // Initially unhealthy with Any strategy
        assert!(!registry.get_status().healthy);

        // First component going healthy is enough in Any strategy
        handle1.report_healthy().await;
        assert_or_retry(|| registry.get_status().healthy).await;

        // Still healthy even if second component is unhealthy
        handle2.report_status(ComponentStatus::Unhealthy).await;
        assert_or_retry(|| registry.get_status().healthy).await;

        // Becomes unhealthy only when all components are unhealthy
        handle1.report_status(ComponentStatus::Unhealthy).await;
        assert_or_retry(|| !registry.get_status().healthy).await;
    }

    #[tokio::test]
    async fn health_strategy_from_str() {
        assert_eq!(
            "ALL".parse::<HealthStrategy>().unwrap(),
            HealthStrategy::All
        );
        assert_eq!(
            "ANY".parse::<HealthStrategy>().unwrap(),
            HealthStrategy::Any
        );
        assert_eq!(
            "all".parse::<HealthStrategy>().unwrap(),
            HealthStrategy::All
        );
        assert_eq!(
            "any".parse::<HealthStrategy>().unwrap(),
            HealthStrategy::Any
        );
        assert!("invalid".parse::<HealthStrategy>().is_err());
    }

    #[test]
    fn shutdown_status_conversion() {
        assert_eq!(ShutdownStatus::from(0), ShutdownStatus::Unknown);
        assert_eq!(ShutdownStatus::from(1), ShutdownStatus::Running);
        assert_eq!(ShutdownStatus::from(2), ShutdownStatus::Prestop);
        assert_eq!(ShutdownStatus::from(3), ShutdownStatus::Terminating);
        assert_eq!(ShutdownStatus::from(4), ShutdownStatus::Completed);
        assert_eq!(ShutdownStatus::from(255), ShutdownStatus::Unknown);
    }

    #[test]
    fn shutdown_status_as_str() {
        assert_eq!(ShutdownStatus::Unknown.as_str(), "unknown");
        assert_eq!(ShutdownStatus::Running.as_str(), "running");
        assert_eq!(ShutdownStatus::Prestop.as_str(), "prestop");
        assert_eq!(ShutdownStatus::Terminating.as_str(), "terminating");
        assert_eq!(ShutdownStatus::Completed.as_str(), "completed");
    }

    #[test]
    fn get_set_shutdown_status() {
        set_shutdown_status(ShutdownStatus::Running);
        assert_eq!(get_shutdown_status(), ShutdownStatus::Running);

        set_shutdown_status(ShutdownStatus::Prestop);
        assert_eq!(get_shutdown_status(), ShutdownStatus::Prestop);

        // Reset for other tests
        set_shutdown_status(ShutdownStatus::Running);
    }
}