Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

12.3 KB
Raw Blame
use std::time::{Duration, Instant};

use common_metrics::gauge;
use lifecycle::Handle;
use tokio::runtime::RuntimeMetrics;
use tokio::time::interval;

use crate::metrics::consts::{
    TOKIO_RUNTIME_ALIVE_TASKS, TOKIO_RUNTIME_BUSY_RATIO, TOKIO_RUNTIME_GLOBAL_QUEUE_DEPTH,
    TOKIO_RUNTIME_NUM_WORKERS, TOKIO_WORKER_BUSY_DURATION_DELTA, TOKIO_WORKER_PARK_DELTA,
};

#[cfg(tokio_unstable)]
use crate::metrics::consts::{
    TOKIO_BLOCKING_QUEUE_DEPTH, TOKIO_BLOCKING_THREADS, TOKIO_IDLE_BLOCKING_THREADS,
    TOKIO_WORKER_LOCAL_QUEUE_DEPTH, TOKIO_WORKER_MEAN_POLL_TIME_US, TOKIO_WORKER_OVERFLOW_DELTA,
    TOKIO_WORKER_POLL_DELTA, TOKIO_WORKER_STEAL_DELTA,
};

const SAMPLING_INTERVAL: Duration = Duration::from_secs(15);

pub struct TokioRuntimeMonitor {
    metrics: RuntimeMetrics,
}

impl TokioRuntimeMonitor {
    pub fn new(handle: &tokio::runtime::Handle) -> Self {
        Self {
            metrics: handle.metrics(),
        }
    }

    pub async fn start_monitoring(self, shutdown: Handle) {
        let _scope = shutdown.process_scope();
        let num_workers = self.metrics.num_workers();

        tracing::info!(
            "Starting Tokio runtime monitoring (workers={}, interval={}s)",
            num_workers,
            SAMPLING_INTERVAL.as_secs()
        );

        gauge(TOKIO_RUNTIME_NUM_WORKERS, &[], num_workers as f64);

        let mut stable_state = WorkerState::new(num_workers, &self.metrics);

        #[cfg(tokio_unstable)]
        let mut unstable_state = UnstableWorkerState::new(num_workers, &self.metrics);

        self.run_loop(
            &shutdown,
            &mut stable_state,
            #[cfg(tokio_unstable)]
            &mut unstable_state,
        )
        .await;
    }

    async fn run_loop(
        &self,
        shutdown: &Handle,
        stable_state: &mut WorkerState,
        #[cfg(tokio_unstable)] unstable_state: &mut UnstableWorkerState,
    ) {
        #[cfg(not(tokio_unstable))]
        tracing::warn!(
            "tokio_unstable not enabled — blocking pool and some per-worker metrics \
             (poll count, steal count, overflow count, local queue depth, mean poll time) \
             are unavailable. Enable with rustflags = [\"--cfg\", \"tokio_unstable\"] for full metrics."
        );

        let mut ticker = interval(SAMPLING_INTERVAL);
        // Skip the immediate first tick so the first report comes after a full interval
        ticker.tick().await;

        loop {
            tokio::select! {
                _ = shutdown.shutdown_recv() => {
                    tracing::info!("Tokio runtime monitor shutting down");
                    break;
                }
                _ = ticker.tick() => {
                    let now = Instant::now();
                    let elapsed = now.duration_since(stable_state.last_sample);
                    stable_state.last_sample = now;

                    self.report_stable_metrics(stable_state, elapsed);

                    #[cfg(tokio_unstable)]
                    self.report_unstable_metrics(unstable_state, &stable_state.worker_labels);
                }
            }
        }
    }

    fn report_stable_metrics(&self, state: &mut WorkerState, elapsed: Duration) {
        gauge(
            TOKIO_RUNTIME_ALIVE_TASKS,
            &[],
            self.metrics.num_alive_tasks() as f64,
        );
        gauge(
            TOKIO_RUNTIME_GLOBAL_QUEUE_DEPTH,
            &[],
            self.metrics.global_queue_depth() as f64,
        );

        let num_workers = self.metrics.num_workers();
        let elapsed_secs = elapsed.as_secs_f64();
        let mut total_busy_delta = Duration::ZERO;

        for i in 0..num_workers {
            let label = &state.worker_labels[i];

            // Busy duration (stable API)
            let busy = self.metrics.worker_total_busy_duration(i);
            let busy_delta = busy.saturating_sub(state.prev_busy[i]);
            state.prev_busy[i] = busy;
            total_busy_delta += busy_delta;

            gauge(
                TOKIO_WORKER_BUSY_DURATION_DELTA,
                label,
                busy_delta.as_secs_f64(),
            );

            // Park count (stable API)
            let parks = self.metrics.worker_park_count(i);
            let park_delta = parks.saturating_sub(state.prev_parks[i]);
            state.prev_parks[i] = parks;
            gauge(TOKIO_WORKER_PARK_DELTA, label, park_delta as f64);
        }

        let busy_ratio = if elapsed_secs > 0.0 && num_workers > 0 {
            total_busy_delta.as_secs_f64() / (elapsed_secs * num_workers as f64)
        } else {
            0.0
        };
        gauge(TOKIO_RUNTIME_BUSY_RATIO, &[], busy_ratio);
    }

    #[cfg(tokio_unstable)]
    fn report_unstable_metrics(
        &self,
        state: &mut UnstableWorkerState,
        worker_labels: &[[(String, String); 1]],
    ) {
        let num_workers = self.metrics.num_workers();

        // Blocking pool
        gauge(
            TOKIO_BLOCKING_THREADS,
            &[],
            self.metrics.num_blocking_threads() as f64,
        );
        gauge(
            TOKIO_IDLE_BLOCKING_THREADS,
            &[],
            self.metrics.num_idle_blocking_threads() as f64,
        );
        gauge(
            TOKIO_BLOCKING_QUEUE_DEPTH,
            &[],
            self.metrics.blocking_queue_depth() as f64,
        );

        for (i, label) in worker_labels.iter().enumerate().take(num_workers) {
            // Poll count
            let polls = self.metrics.worker_poll_count(i);
            let poll_delta = polls.saturating_sub(state.prev_polls[i]);
            state.prev_polls[i] = polls;
            gauge(TOKIO_WORKER_POLL_DELTA, label, poll_delta as f64);

            // Steal count
            let steals = self.metrics.worker_steal_count(i);
            let steal_delta = steals.saturating_sub(state.prev_steals[i]);
            state.prev_steals[i] = steals;
            gauge(TOKIO_WORKER_STEAL_DELTA, label, steal_delta as f64);

            // Overflow count
            let overflows = self.metrics.worker_overflow_count(i);
            let overflow_delta = overflows.saturating_sub(state.prev_overflows[i]);
            state.prev_overflows[i] = overflows;
            gauge(TOKIO_WORKER_OVERFLOW_DELTA, label, overflow_delta as f64);

            // Instantaneous gauges (not deltas)
            gauge(
                TOKIO_WORKER_LOCAL_QUEUE_DEPTH,
                label,
                self.metrics.worker_local_queue_depth(i) as f64,
            );
            gauge(
                TOKIO_WORKER_MEAN_POLL_TIME_US,
                label,
                self.metrics.worker_mean_poll_time(i).as_micros() as f64,
            );
        }
    }
}

struct WorkerState {
    last_sample: Instant,
    worker_labels: Vec<[(String, String); 1]>,
    prev_busy: Vec<Duration>,
    prev_parks: Vec<u64>,
}

impl WorkerState {
    fn new(num_workers: usize, metrics: &RuntimeMetrics) -> Self {
        let worker_labels: Vec<[(String, String); 1]> = (0..num_workers)
            .map(|i| [("worker".to_string(), i.to_string())])
            .collect();

        let mut prev_busy = vec![Duration::ZERO; num_workers];
        let mut prev_parks = vec![0u64; num_workers];

        // Snapshot current values so the first delta is accurate
        for i in 0..num_workers {
            prev_busy[i] = metrics.worker_total_busy_duration(i);
            prev_parks[i] = metrics.worker_park_count(i);
        }

        Self {
            last_sample: Instant::now(),
            worker_labels,
            prev_busy,
            prev_parks,
        }
    }
}

#[cfg(tokio_unstable)]
struct UnstableWorkerState {
    prev_polls: Vec<u64>,
    prev_steals: Vec<u64>,
    prev_overflows: Vec<u64>,
}

#[cfg(tokio_unstable)]
impl UnstableWorkerState {
    fn new(num_workers: usize, metrics: &RuntimeMetrics) -> Self {
        let mut prev_polls = vec![0u64; num_workers];
        let mut prev_steals = vec![0u64; num_workers];
        let mut prev_overflows = vec![0u64; num_workers];

        // Snapshot current values so the first delta is accurate
        for i in 0..num_workers {
            prev_polls[i] = metrics.worker_poll_count(i);
            prev_steals[i] = metrics.worker_steal_count(i);
            prev_overflows[i] = metrics.worker_overflow_count(i);
        }

        Self {
            prev_polls,
            prev_steals,
            prev_overflows,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn spawned_task_increases_alive_count() {
        let handle = tokio::runtime::Handle::current();
        let before = handle.metrics().num_alive_tasks();

        // Hold a task alive — single-threaded runtime won't poll it until we yield
        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
        let task = tokio::spawn(async move {
            rx.await.ok();
        });

        let after = handle.metrics().num_alive_tasks();
        assert!(
            after > before,
            "alive tasks should increase after spawn: before={before}, after={after}"
        );

        let monitor = TokioRuntimeMonitor::new(&handle);
        let num_workers = handle.metrics().num_workers();
        let mut state = WorkerState::new(num_workers, &handle.metrics());
        monitor.report_stable_metrics(&mut state, Duration::from_secs(15));

        tx.send(()).ok();
        task.await.ok();
    }

    // A current-thread runtime publishes worker metrics before it resumes this test.
    // A multi-thread worker publishes them only when it parks or runs maintenance.
    // Either can happen after the test reads them.
    #[cfg(tokio_unstable)]
    #[tokio::test]
    async fn poll_and_busy_deltas_advance_after_work() {
        let handle = tokio::runtime::Handle::current();
        let metrics = handle.metrics();
        let num_workers = metrics.num_workers();
        let monitor = TokioRuntimeMonitor::new(&handle);

        let mut stable_state = WorkerState::new(num_workers, &metrics);
        let mut unstable_state = UnstableWorkerState::new(num_workers, &metrics);
        let polls_before: Vec<u64> = unstable_state.prev_polls.clone();
        let busy_before: Vec<Duration> = stable_state.prev_busy.clone();

        // Spawn real tasks so the worker registers polls
        let mut handles = Vec::new();
        for _ in 0..50 {
            handles.push(tokio::spawn(async {
                tokio::task::yield_now().await;
            }));
        }
        for h in handles {
            h.await.ok();
        }

        monitor.report_stable_metrics(&mut stable_state, Duration::from_secs(15));
        monitor.report_unstable_metrics(&mut unstable_state, &stable_state.worker_labels);

        let polls_advanced = unstable_state
            .prev_polls
            .iter()
            .zip(polls_before.iter())
            .any(|(after, before)| after > before);
        assert!(
            polls_advanced,
            "poll counts should advance: before={polls_before:?}, after={:?}",
            unstable_state.prev_polls
        );

        let busy_advanced = stable_state
            .prev_busy
            .iter()
            .zip(busy_before.iter())
            .any(|(after, before)| after > before);
        assert!(
            busy_advanced,
            "busy durations should advance: before={busy_before:?}, after={:?}",
            stable_state.prev_busy
        );
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn busy_ratio_bounded_with_real_elapsed() {
        let handle = tokio::runtime::Handle::current();
        let metrics = handle.metrics();
        let num_workers = metrics.num_workers();

        let busy_before: Vec<Duration> = (0..num_workers)
            .map(|i| metrics.worker_total_busy_duration(i))
            .collect();

        let start = std::time::Instant::now();
        for _ in 0..1000 {
            tokio::task::yield_now().await;
        }
        tokio::time::sleep(Duration::from_millis(50)).await;
        let elapsed = start.elapsed();

        let mut total_busy_delta = Duration::ZERO;
        for (i, &before) in busy_before.iter().enumerate().take(num_workers) {
            let busy_now = metrics.worker_total_busy_duration(i);
            total_busy_delta += busy_now.saturating_sub(before);
        }

        let ratio = total_busy_delta.as_secs_f64() / (elapsed.as_secs_f64() * num_workers as f64);
        assert!(
            (0.0..=1.0).contains(&ratio),
            "busy ratio should be in [0, 1], got {ratio}"
        );
    }
}