Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.9 KB
Raw Blame
//! Periodic Tokio runtime monitor: per-worker busy-ratio, queue depths, and blocking-pool depth.
//! Full per-worker and blocking-pool metrics require `tokio_unstable` (enabled workspace-wide).

use std::sync::Arc;
use std::time::{Duration, Instant};

use lifecycle::Handle;
use metrics::gauge;
use tokio::runtime::RuntimeMetrics;
use tokio::time::interval;

use crate::observability::metrics::{
    TOKIO_RUNTIME_ALIVE_TASKS, TOKIO_RUNTIME_BUSY_RATIO, TOKIO_RUNTIME_GLOBAL_QUEUE_DEPTH,
    TOKIO_RUNTIME_NUM_WORKERS, TOKIO_WORKER_BUSY_DURATION_DELTA, TOKIO_WORKER_PARK_DELTA,
};

#[cfg(tokio_unstable)]
use crate::observability::metrics::{
    TOKIO_BLOCKING_QUEUE_DEPTH, TOKIO_BLOCKING_THREADS, TOKIO_IDLE_BLOCKING_THREADS,
    TOKIO_WORKER_LOCAL_QUEUE_DEPTH, TOKIO_WORKER_MEAN_POLL_TIME_US, TOKIO_WORKER_OVERFLOW_DELTA,
    TOKIO_WORKER_POLL_DELTA, TOKIO_WORKER_STEAL_DELTA,
};

/// Samples [`RuntimeMetrics`] on a fixed cadence and publishes worker-utilization gauges. Per-worker
/// counters are published as deltas over the interval.
pub struct TokioRuntimeMonitor {
    metrics: RuntimeMetrics,
    interval: Duration,
}

impl TokioRuntimeMonitor {
    pub fn new(handle: &tokio::runtime::Handle, interval: Duration) -> Self {
        Self {
            metrics: handle.metrics(),
            interval,
        }
    }

    pub async fn start_monitoring(self, shutdown: Handle) {
        let _scope = shutdown.process_scope();
        let num_workers = self.metrics.num_workers();

        tracing::info!(
            workers = num_workers,
            interval_secs = self.interval.as_secs(),
            "starting Tokio runtime monitor",
        );

        gauge!(TOKIO_RUNTIME_NUM_WORKERS).set(num_workers as f64);

        let mut stable_state = WorkerState::new(num_workers, &self.metrics);

        #[cfg(tokio_unstable)]
        let mut unstable_state = UnstableWorkerState::new(num_workers, &self.metrics);

        self.run_loop(
            &shutdown,
            &mut stable_state,
            #[cfg(tokio_unstable)]
            &mut unstable_state,
        )
        .await;
    }

    async fn run_loop(
        &self,
        shutdown: &Handle,
        stable_state: &mut WorkerState,
        #[cfg(tokio_unstable)] unstable_state: &mut UnstableWorkerState,
    ) {
        #[cfg(not(tokio_unstable))]
        tracing::warn!(
            "tokio_unstable not enabled — blocking pool and some per-worker metrics \
             (poll count, steal count, overflow count, local queue depth, mean poll time) \
             are unavailable. Enable with rustflags = [\"--cfg\", \"tokio_unstable\"] for full metrics."
        );

        let mut ticker = interval(self.interval);
        // Skip the immediate first tick so the first report reflects a full interval of deltas.
        ticker.tick().await;

        loop {
            tokio::select! {
                _ = shutdown.shutdown_recv() => {
                    tracing::info!("Tokio runtime monitor shutting down");
                    break;
                }
                _ = ticker.tick() => {
                    let now = Instant::now();
                    let elapsed = now.duration_since(stable_state.last_sample);
                    stable_state.last_sample = now;

                    self.report_stable_metrics(stable_state, elapsed);

                    #[cfg(tokio_unstable)]
                    self.report_unstable_metrics(unstable_state, &stable_state.worker_labels);
                }
            }
        }
    }

    fn report_stable_metrics(&self, state: &mut WorkerState, elapsed: Duration) {
        gauge!(TOKIO_RUNTIME_ALIVE_TASKS).set(self.metrics.num_alive_tasks() as f64);
        gauge!(TOKIO_RUNTIME_GLOBAL_QUEUE_DEPTH).set(self.metrics.global_queue_depth() as f64);

        let num_workers = self.metrics.num_workers();
        let elapsed_secs = elapsed.as_secs_f64();
        let mut total_busy_delta = Duration::ZERO;

        // busy-duration and park-count are tokio's only stable per-worker metrics; the rest are
        // gathered under `tokio_unstable` in `report_unstable_metrics`.
        for i in 0..num_workers {
            let label = state.worker_labels[i].clone();

            let busy = self.metrics.worker_total_busy_duration(i);
            let busy_delta = busy.saturating_sub(state.prev_busy[i]);
            state.prev_busy[i] = busy;
            total_busy_delta += busy_delta;
            gauge!(TOKIO_WORKER_BUSY_DURATION_DELTA, "worker" => label.clone())
                .set(busy_delta.as_secs_f64());

            let parks = self.metrics.worker_park_count(i);
            let park_delta = parks.saturating_sub(state.prev_parks[i]);
            state.prev_parks[i] = parks;
            gauge!(TOKIO_WORKER_PARK_DELTA, "worker" => label).set(park_delta as f64);
        }

        let busy_ratio = if elapsed_secs > 0.0 && num_workers > 0 {
            total_busy_delta.as_secs_f64() / (elapsed_secs * num_workers as f64)
        } else {
            0.0
        };
        gauge!(TOKIO_RUNTIME_BUSY_RATIO).set(busy_ratio);
    }

    #[cfg(tokio_unstable)]
    fn report_unstable_metrics(&self, state: &mut UnstableWorkerState, worker_labels: &[Arc<str>]) {
        let num_workers = self.metrics.num_workers();

        gauge!(TOKIO_BLOCKING_THREADS).set(self.metrics.num_blocking_threads() as f64);
        gauge!(TOKIO_IDLE_BLOCKING_THREADS).set(self.metrics.num_idle_blocking_threads() as f64);
        gauge!(TOKIO_BLOCKING_QUEUE_DEPTH).set(self.metrics.blocking_queue_depth() as f64);

        for (i, label) in worker_labels.iter().enumerate().take(num_workers) {
            let polls = self.metrics.worker_poll_count(i);
            let poll_delta = polls.saturating_sub(state.prev_polls[i]);
            state.prev_polls[i] = polls;
            gauge!(TOKIO_WORKER_POLL_DELTA, "worker" => label.clone()).set(poll_delta as f64);

            let steals = self.metrics.worker_steal_count(i);
            let steal_delta = steals.saturating_sub(state.prev_steals[i]);
            state.prev_steals[i] = steals;
            gauge!(TOKIO_WORKER_STEAL_DELTA, "worker" => label.clone()).set(steal_delta as f64);

            let overflows = self.metrics.worker_overflow_count(i);
            let overflow_delta = overflows.saturating_sub(state.prev_overflows[i]);
            state.prev_overflows[i] = overflows;
            gauge!(TOKIO_WORKER_OVERFLOW_DELTA, "worker" => label.clone())
                .set(overflow_delta as f64);

            // Instantaneous gauges, not deltas.
            gauge!(TOKIO_WORKER_LOCAL_QUEUE_DEPTH, "worker" => label.clone())
                .set(self.metrics.worker_local_queue_depth(i) as f64);
            gauge!(TOKIO_WORKER_MEAN_POLL_TIME_US, "worker" => label.clone())
                .set(self.metrics.worker_mean_poll_time(i).as_micros() as f64);
        }
    }
}

struct WorkerState {
    last_sample: Instant,
    /// Per-worker `worker` label, built once and cloned into each macro call.
    worker_labels: Vec<Arc<str>>,
    prev_busy: Vec<Duration>,
    prev_parks: Vec<u64>,
}

impl WorkerState {
    fn new(num_workers: usize, metrics: &RuntimeMetrics) -> Self {
        let worker_labels: Vec<Arc<str>> =
            (0..num_workers).map(|i| Arc::from(i.to_string())).collect();

        let mut prev_busy = vec![Duration::ZERO; num_workers];
        let mut prev_parks = vec![0u64; num_workers];

        // Snapshot current values so the first delta is accurate.
        for i in 0..num_workers {
            prev_busy[i] = metrics.worker_total_busy_duration(i);
            prev_parks[i] = metrics.worker_park_count(i);
        }

        Self {
            last_sample: Instant::now(),
            worker_labels,
            prev_busy,
            prev_parks,
        }
    }
}

#[cfg(tokio_unstable)]
struct UnstableWorkerState {
    prev_polls: Vec<u64>,
    prev_steals: Vec<u64>,
    prev_overflows: Vec<u64>,
}

#[cfg(tokio_unstable)]
impl UnstableWorkerState {
    fn new(num_workers: usize, metrics: &RuntimeMetrics) -> Self {
        let mut prev_polls = vec![0u64; num_workers];
        let mut prev_steals = vec![0u64; num_workers];
        let mut prev_overflows = vec![0u64; num_workers];

        // Snapshot current values so the first delta is accurate.
        for i in 0..num_workers {
            prev_polls[i] = metrics.worker_poll_count(i);
            prev_steals[i] = metrics.worker_steal_count(i);
            prev_overflows[i] = metrics.worker_overflow_count(i);
        }

        Self {
            prev_polls,
            prev_steals,
            prev_overflows,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // Runs the emission path and checks the busy-ratio stays in [0, 1] over a real elapsed window.
    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn busy_ratio_bounded_with_real_elapsed() {
        let handle = tokio::runtime::Handle::current();
        let metrics = handle.metrics();
        let num_workers = metrics.num_workers();
        let monitor = TokioRuntimeMonitor::new(&handle, Duration::from_secs(15));
        let mut state = WorkerState::new(num_workers, &metrics);

        let busy_before: Vec<Duration> = (0..num_workers)
            .map(|i| metrics.worker_total_busy_duration(i))
            .collect();

        let start = Instant::now();
        for _ in 0..1000 {
            tokio::task::yield_now().await;
        }
        tokio::time::sleep(Duration::from_millis(50)).await;
        let elapsed = start.elapsed();

        monitor.report_stable_metrics(&mut state, elapsed);

        let mut total_busy_delta = Duration::ZERO;
        for (i, &before) in busy_before.iter().enumerate().take(num_workers) {
            let busy_now = metrics.worker_total_busy_duration(i);
            total_busy_delta += busy_now.saturating_sub(before);
        }

        let ratio = total_busy_delta.as_secs_f64() / (elapsed.as_secs_f64() * num_workers as f64);
        assert!(
            (0.0..=1.0).contains(&ratio),
            "busy ratio should be in [0, 1], got {ratio}"
        );
    }

    // Covers the unstable delta path (poll/steal/overflow), which runs in production because
    // `tokio_unstable` is enabled workspace-wide: a wrong `saturating_sub` order or a missed
    // `prev_*` update would emit silently wrong deltas that the stable-path test cannot catch.
    // A current-thread runtime publishes worker metrics before it resumes this test.
    // A multi-thread worker publishes them only when it parks or runs maintenance.
    // Either can happen after the test reads them.
    #[cfg(tokio_unstable)]
    #[tokio::test]
    async fn poll_and_busy_deltas_advance_after_work() {
        let handle = tokio::runtime::Handle::current();
        let metrics = handle.metrics();
        let num_workers = metrics.num_workers();
        let monitor = TokioRuntimeMonitor::new(&handle, Duration::from_secs(15));

        let mut stable_state = WorkerState::new(num_workers, &metrics);
        let mut unstable_state = UnstableWorkerState::new(num_workers, &metrics);
        let polls_before: Vec<u64> = unstable_state.prev_polls.clone();
        let busy_before: Vec<Duration> = stable_state.prev_busy.clone();

        let mut handles = Vec::new();
        for _ in 0..50 {
            handles.push(tokio::spawn(async {
                tokio::task::yield_now().await;
            }));
        }
        for h in handles {
            h.await.ok();
        }

        monitor.report_stable_metrics(&mut stable_state, Duration::from_secs(15));
        monitor.report_unstable_metrics(&mut unstable_state, &stable_state.worker_labels);

        let polls_advanced = unstable_state
            .prev_polls
            .iter()
            .zip(polls_before.iter())
            .any(|(after, before)| after > before);
        assert!(polls_advanced, "poll counts should advance after work");

        let busy_advanced = stable_state
            .prev_busy
            .iter()
            .zip(busy_before.iter())
            .any(|(after, before)| after > before);
        assert!(busy_advanced, "busy durations should advance after work");
    }
}