Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
14.4 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Bounded in-memory run state, latency percentiles, and the final pass/fail summary.

use std::collections::BTreeMap;
use std::time::SystemTime;

use rand::rngs::StdRng;
use rand::{RngExt, SeedableRng};
use serde::Serialize;

/// Cap on retained latency samples; a long run stays within this bound by reservoir sampling.
const RESERVOIR_SIZE: usize = 100_000;

/// Round to three decimals so result files stay readable.
pub fn round3(value: f64) -> f64 {
    (value * 1000.0).round() / 1000.0
}

/// Parse a duration such as `30s`, `15m`, or `48h` into seconds.
pub fn parse_duration(value: &str) -> Result<f64, String> {
    let text = value.trim().to_lowercase();
    let bad = || "duration must use s, m, or h, for example 30s or 48h".to_string();
    let unit = text.chars().last().ok_or_else(bad)?;
    let multiplier = match unit {
        's' => 1.0,
        'm' => 60.0,
        'h' => 3600.0,
        _ => return Err(bad()),
    };
    let number = &text[..text.len() - unit.len_utf8()];
    let seconds = number.parse::<f64>().map_err(|_| bad())? * multiplier;
    if !seconds.is_finite() || seconds <= 0.0 {
        return Err("duration must be greater than zero".to_string());
    }
    Ok(seconds)
}

#[derive(Default)]
pub struct LatencyStats {
    pub p50_ms: Option<f64>,
    pub p95_ms: Option<f64>,
    pub p99_ms: Option<f64>,
    pub max_ms: Option<f64>,
}

fn percentile_sorted(values: &[f64], quantile: f64) -> Option<f64> {
    let index = (((values.len().checked_sub(1)?) as f64) * quantile).round_ties_even() as usize;
    Some(round3(values[index]))
}

pub fn latency_stats(values: &mut [f64]) -> LatencyStats {
    values.sort_by(|a, b| a.total_cmp(b));
    LatencyStats {
        p50_ms: percentile_sorted(values, 0.50),
        p95_ms: percentile_sorted(values, 0.95),
        p99_ms: percentile_sorted(values, 0.99),
        max_ms: values.last().copied().map(round3),
    }
}

/// ISO-8601 UTC timestamp at second precision, e.g. `2026-07-30T18:04:00Z`.
pub fn now_utc_string() -> String {
    humantime::format_rfc3339_seconds(SystemTime::now()).to_string()
}

/// Compact UTC stamp for a results directory, e.g. `20260730T180400Z`.
pub fn utc_dir_stamp() -> String {
    now_utc_string()
        .chars()
        .filter(|character| !matches!(character, '-' | ':'))
        .collect()
}

/// Request results collected since the previous report.
#[derive(Default)]
pub struct IntervalStats {
    pub successes: u64,
    pub failures: u64,
    pub latencies_ms: Vec<f64>,
}

/// Bounded in-memory state for one soak run.
pub struct RunStats {
    rng: StdRng,
    interval: IntervalStats,
    pub total_successes: u64,
    pub total_failures: u64,
    pub endpoint_successes: BTreeMap<String, u64>,
    pub endpoint_failures: BTreeMap<String, u64>,
    pub scenario_successes: BTreeMap<String, u64>,
    pub scenario_failures: BTreeMap<String, u64>,
    pub error_kinds: BTreeMap<String, u64>,
    latency_reservoir: Vec<f64>,
    latency_count: u64,
    pub health_checks: u64,
    pub health_failures: u64,
    pub metrics_checks: u64,
    pub metrics_failures: u64,
    pub process_checks: u64,
    pub process_failures: u64,
    pub canaries: u64,
    pub canary_failures: u64,
    pub server_restarts: u64,
    pub previous_server_requests: Option<f64>,
    pub rss_samples: Vec<f64>,
    pub completed_duration: bool,
}

impl RunStats {
    pub fn new(seed: u64) -> Self {
        Self {
            rng: StdRng::seed_from_u64(seed),
            interval: IntervalStats::default(),
            total_successes: 0,
            total_failures: 0,
            endpoint_successes: BTreeMap::new(),
            endpoint_failures: BTreeMap::new(),
            scenario_successes: BTreeMap::new(),
            scenario_failures: BTreeMap::new(),
            error_kinds: BTreeMap::new(),
            latency_reservoir: Vec::new(),
            latency_count: 0,
            health_checks: 0,
            health_failures: 0,
            metrics_checks: 0,
            metrics_failures: 0,
            process_checks: 0,
            process_failures: 0,
            canaries: 0,
            canary_failures: 0,
            server_restarts: 0,
            previous_server_requests: None,
            rss_samples: Vec::new(),
            completed_duration: false,
        }
    }

    /// Record one completed inference request; `error_kind` is `None` on success.
    pub fn record(
        &mut self,
        endpoint: &str,
        scenario: &str,
        latency_ms: f64,
        error_kind: Option<&str>,
    ) {
        match error_kind {
            None => {
                self.interval.successes += 1;
                self.total_successes += 1;
                *self
                    .endpoint_successes
                    .entry(endpoint.to_string())
                    .or_default() += 1;
                *self
                    .scenario_successes
                    .entry(scenario.to_string())
                    .or_default() += 1;
            }
            Some(kind) => {
                self.interval.failures += 1;
                self.total_failures += 1;
                *self
                    .endpoint_failures
                    .entry(endpoint.to_string())
                    .or_default() += 1;
                *self
                    .scenario_failures
                    .entry(scenario.to_string())
                    .or_default() += 1;
                *self.error_kinds.entry(kind.to_string()).or_default() += 1;
            }
        }
        self.interval.latencies_ms.push(latency_ms);
        self.latency_count += 1;
        if self.latency_reservoir.len() < RESERVOIR_SIZE {
            self.latency_reservoir.push(latency_ms);
        } else {
            let index = self.rng.random_range(0..self.latency_count) as usize;
            if index < RESERVOIR_SIZE {
                self.latency_reservoir[index] = latency_ms;
            }
        }
    }

    /// Return and reset the current interval.
    pub fn take_interval(&mut self) -> IntervalStats {
        std::mem::take(&mut self.interval)
    }
}

/// Final result written to `summary.json`.
#[derive(Serialize)]
pub struct Summary {
    pub passed: bool,
    pub failure_reasons: Vec<String>,
    pub completed_duration: bool,
    pub elapsed_seconds: f64,
    pub requests: u64,
    pub successes: u64,
    pub failures: u64,
    pub error_rate: f64,
    pub requests_per_second: f64,
    pub endpoint_successes: BTreeMap<String, u64>,
    pub endpoint_failures: BTreeMap<String, u64>,
    pub scenario_successes: BTreeMap<String, u64>,
    pub scenario_failures: BTreeMap<String, u64>,
    pub error_kinds: BTreeMap<String, u64>,
    pub latency_p50_ms: Option<f64>,
    pub latency_p95_ms: Option<f64>,
    pub latency_p99_ms: Option<f64>,
    pub health_checks: u64,
    pub health_failures: u64,
    pub metrics_checks: u64,
    pub metrics_failures: u64,
    pub process_checks: u64,
    pub process_failures: u64,
    pub invalid_request_canaries: u64,
    pub invalid_request_canary_failures: u64,
    pub detected_server_restarts: u64,
    pub rss_first_mib: Option<f64>,
    pub rss_last_mib: Option<f64>,
    pub rss_max_mib: Option<f64>,
    pub rss_growth_mib: Option<f64>,
    pub error_records: u64,
    pub dropped_error_records: u64,
}

/// Build the final result and its release-gate reasons.
pub fn build_summary(
    stats: &RunStats,
    elapsed_seconds: f64,
    max_error_rate: f64,
    max_rss_growth_mib: Option<f64>,
    error_records: u64,
    dropped_error_records: u64,
    task_failures: &[String],
) -> Summary {
    let total = stats.total_successes + stats.total_failures;
    let error_rate = if total > 0 {
        stats.total_failures as f64 / total as f64
    } else {
        1.0
    };
    let rss_first = stats.rss_samples.first().copied();
    let rss_last = stats.rss_samples.last().copied();
    let rss_growth = match (rss_first, rss_last) {
        (Some(first), Some(last)) => Some(last - first),
        _ => None,
    };

    let mut reasons: Vec<String> = task_failures.to_vec();
    if !stats.completed_duration {
        reasons.push("the run stopped before the requested duration".to_string());
    }
    if total == 0 {
        reasons.push("no inference requests completed".to_string());
    }
    if error_rate > max_error_rate {
        reasons.push(format!(
            "request error rate {:.4}% exceeded the {:.4}% limit",
            error_rate * 100.0,
            max_error_rate * 100.0
        ));
    }
    if stats.health_failures > 0 {
        reasons.push(format!("{} liveness checks failed", stats.health_failures));
    }
    if stats.metrics_failures > 0 {
        reasons.push(format!(
            "{} server metrics checks failed",
            stats.metrics_failures
        ));
    }
    if stats.process_failures > 0 {
        reasons.push(format!(
            "{} server process checks failed",
            stats.process_failures
        ));
    }
    if stats.canary_failures > 0 {
        reasons.push(format!(
            "{} invalid-request recovery checks failed",
            stats.canary_failures
        ));
    }
    if stats.server_restarts > 0 {
        reasons.push(format!(
            "server counters reset {} time(s)",
            stats.server_restarts
        ));
    }
    if let (Some(limit), Some(growth)) = (max_rss_growth_mib, rss_growth)
        && growth > limit
    {
        reasons.push(format!(
            "server RSS grew {growth:.1} MiB, above the {limit:.1} MiB limit"
        ));
    }

    let rss_max = stats
        .rss_samples
        .iter()
        .copied()
        .fold(None, |acc: Option<f64>, value| {
            Some(acc.map_or(value, |m| m.max(value)))
        });
    let requests_per_second = if elapsed_seconds > 0.0 {
        total as f64 / elapsed_seconds
    } else {
        0.0
    };

    let mut latency_samples = stats.latency_reservoir.clone();
    let latency = latency_stats(&mut latency_samples);

    Summary {
        passed: reasons.is_empty(),
        failure_reasons: reasons,
        completed_duration: stats.completed_duration,
        elapsed_seconds: round3(elapsed_seconds),
        requests: total,
        successes: stats.total_successes,
        failures: stats.total_failures,
        error_rate,
        requests_per_second,
        endpoint_successes: stats.endpoint_successes.clone(),
        endpoint_failures: stats.endpoint_failures.clone(),
        scenario_successes: stats.scenario_successes.clone(),
        scenario_failures: stats.scenario_failures.clone(),
        error_kinds: stats.error_kinds.clone(),
        latency_p50_ms: latency.p50_ms,
        latency_p95_ms: latency.p95_ms,
        latency_p99_ms: latency.p99_ms,
        health_checks: stats.health_checks,
        health_failures: stats.health_failures,
        metrics_checks: stats.metrics_checks,
        metrics_failures: stats.metrics_failures,
        process_checks: stats.process_checks,
        process_failures: stats.process_failures,
        invalid_request_canaries: stats.canaries,
        invalid_request_canary_failures: stats.canary_failures,
        detected_server_restarts: stats.server_restarts,
        rss_first_mib: rss_first,
        rss_last_mib: rss_last,
        rss_max_mib: rss_max,
        rss_growth_mib: rss_growth,
        error_records,
        dropped_error_records,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn parse_duration_accepts_units_and_rejects_invalid_values() {
        assert_eq!(parse_duration("30s"), Ok(30.0));
        assert_eq!(parse_duration(".5s"), Ok(0.5));
        assert_eq!(parse_duration("2.5m"), Ok(150.0));
        assert_eq!(parse_duration("48h"), Ok(172_800.0));
        for value in ["0s", "10", "5x", "-3s", "1d", "NaNs", "infs"] {
            assert!(parse_duration(value).is_err(), "{value} should be rejected");
        }
    }

    #[test]
    fn latency_stats_sorts_once_and_handles_empty_input() {
        let mut values = [60.0, 10.0, 50.0, 20.0, 40.0, 30.0];
        let latency = latency_stats(&mut values);
        assert_eq!(latency.p50_ms, Some(30.0));
        assert_eq!(latency.p95_ms, Some(60.0));
        assert_eq!(latency.p99_ms, Some(60.0));
        assert_eq!(latency.max_ms, Some(60.0));

        let empty = latency_stats(&mut []);
        assert_eq!(empty.p50_ms, None);
        assert_eq!(empty.max_ms, None);
    }

    #[test]
    fn record_tracks_interval_and_cumulative_results() {
        let mut stats = RunStats::new(1);
        stats.record("chat", "short-interactive", 10.0, None);
        stats.record("messages", "long-context", 20.0, Some("timeout"));

        let interval = stats.take_interval();
        assert_eq!(interval.successes, 1);
        assert_eq!(interval.failures, 1);
        assert_eq!(stats.total_successes, 1);
        assert_eq!(stats.total_failures, 1);
        assert_eq!(stats.error_kinds.get("timeout"), Some(&1));
    }

    #[test]
    fn latency_samples_stay_bounded_after_the_reservoir_fills() {
        let mut stats = RunStats::new(1);
        for sample in 0..=RESERVOIR_SIZE {
            stats.record("chat", "short-interactive", sample as f64, None);
        }

        assert_eq!(stats.latency_reservoir.len(), RESERVOIR_SIZE);
        assert_eq!(stats.latency_count, RESERVOIR_SIZE as u64 + 1);
    }

    #[test]
    fn summary_reports_all_failed_gates() {
        let mut stats = RunStats::new(1);
        stats.total_successes = 998;
        stats.total_failures = 2;
        stats.health_failures = 1;
        stats.metrics_failures = 1;
        stats.process_failures = 1;
        stats.canary_failures = 1;
        stats.server_restarts = 1;
        stats.rss_samples = vec![100.0, 700.0];

        let summary = build_summary(
            &stats,
            10.0,
            0.001,
            Some(512.0),
            0,
            0,
            &["reporter failed: boom".to_string()],
        );

        assert!(!summary.passed);
        for reason in [
            "run stopped",
            "error rate",
            "liveness",
            "metrics",
            "process",
            "invalid-request",
            "counters reset",
            "RSS grew",
            "reporter failed",
        ] {
            assert!(
                summary
                    .failure_reasons
                    .iter()
                    .any(|message| message.contains(reason)),
                "missing {reason:?}: {:?}",
                summary.failure_reasons,
            );
        }
        assert_eq!(summary.rss_growth_mib, Some(600.0));
    }
}