Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust
26.0 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Thread-safe stats accumulator and serializable snapshot schema.

use std::collections::{BTreeMap, HashSet};
use std::sync::Arc;

use parking_lot::{Mutex, MutexGuard};
use prometheus::Registry;
use serde::Serialize;

use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};
use super::cache_eligibility::PrefixProbe;
use switchyard_protocol::ModelId;
use switchyard_runner::{DecisionTarget, Route};

const MAX_LATENCY_SAMPLES: usize = 10_000;

/// Normalized token counters recorded by the server.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) struct TokenUsage {
    pub prompt_tokens: u64,
    pub completion_tokens: u64,
    pub cached_tokens: u64,
    pub cache_creation_tokens: u64,
    pub cacheable_prompt_tokens: u64,
    pub reasoning_tokens: u64,
}

/// Thread-safe process-local stats store.
#[derive(Clone)]
pub(crate) struct StatsAccumulator {
    inner: Arc<Mutex<StatsAccumulatorInner>>,
    route: Option<Route>,
}

impl Default for StatsAccumulator {
    fn default() -> Self {
        Self::new(Registry::new(), std::iter::empty())
    }
}

impl StatsAccumulator {
    /// Creates a stats store for the supplied algorithm names.
    pub(crate) fn new<'a>(
        registry: Registry,
        algorithms: impl IntoIterator<Item = &'a str>,
    ) -> Self {
        Self {
            inner: Arc::new(Mutex::new(StatsAccumulatorInner::new(registry, algorithms))),
            route: None,
        }
    }

    /// Shares counters while resolving this route's target names to provider models.
    pub(crate) fn for_route(&self, route: Route) -> Self {
        Self {
            inner: Arc::clone(&self.inner),
            route: Some(route),
        }
    }

    /// Resolves a stats key to the model used by existing metric labels.
    pub(crate) fn provider_model(&self, target: &ModelId) -> ModelId {
        self.route
            .as_ref()
            .map_or(target, |route| route.provider_model(target))
            .clone()
    }

    /// Records one successful routed backend call.
    pub(crate) fn record_success(&self, model: impl Into<ModelId>, backend_latency_ms: f64) {
        let mut inner = self.lock();
        inner.total_requests = inner.total_requests.saturating_add(1);
        for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
            stats.calls = stats.calls.saturating_add(1);
            stats.model_call_latency.record(backend_latency_ms);
        }
    }

    /// Records one failed routed backend call.
    pub(crate) fn record_error(&self, model: impl Into<ModelId>) {
        let mut inner = self.lock();
        inner.total_requests = inner.total_requests.saturating_add(1);
        inner.total_errors = inner.total_errors.saturating_add(1);
        for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
            stats.errors = stats.errors.saturating_add(1);
        }
    }

    /// Record a failure while processing a buffered response or stream after its routed call
    /// was already counted.
    pub(crate) fn record_response_error(&self, model: impl Into<ModelId>) {
        let mut inner = self.lock();
        inner.total_errors = inner.total_errors.saturating_add(1);
        for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
            stats.errors = stats.errors.saturating_add(1);
        }
    }

    /// Records usage and terminal latency after a successful routed call.
    pub(crate) fn record_usage(
        &self,
        model: impl Into<ModelId>,
        usage: TokenUsage,
        total_latency_ms: f64,
    ) {
        let mut inner = self.lock();
        for stats in inner.model_stats_mut(model.into(), self.route.as_ref()) {
            stats.add_usage(usage);
            stats.total_latency.record(total_latency_ms);
        }
    }

    /// Records routing time for one completed algorithm run.
    pub(crate) fn record_routing_overhead(&self, routing_overhead_ms: f64) {
        self.lock().routing_overhead.record(routing_overhead_ms);
    }

    /// Records one successful classifier or judge call.
    pub(crate) fn record_classifier_success(
        &self,
        model: impl Into<ModelId>,
        usage: Option<TokenUsage>,
        latency_ms: f64,
    ) {
        let mut inner = self.lock();
        inner.classifier_requests = inner.classifier_requests.saturating_add(1);
        for stats in inner.classifier_stats_mut(model.into(), self.route.as_ref()) {
            stats.calls = stats.calls.saturating_add(1);
            if let Some(usage) = usage {
                stats.add_usage(usage);
            }
            stats.model_call_latency.record(latency_ms);
            stats.total_latency.record(latency_ms);
        }
    }

    /// Records one failed classifier or judge call.
    pub(crate) fn record_classifier_error(&self, model: impl Into<ModelId>) {
        let mut inner = self.lock();
        inner.classifier_requests = inner.classifier_requests.saturating_add(1);
        inner.classifier_errors = inner.classifier_errors.saturating_add(1);
        for stats in inner.classifier_stats_mut(model.into(), self.route.as_ref()) {
            stats.errors = stats.errors.saturating_add(1);
        }
    }

    /// Returns the cache-eligible fraction for `model` and records the prefix as seen.
    pub(crate) fn prefix_eligibility(&self, model: &ModelId, probe: &PrefixProbe) -> f64 {
        let mut inner = self.lock();
        // Cache eligibility stays target-specific when prompts or clients differ.
        inner
            .model_stats_mut(model.clone(), self.route.as_ref())
            .fold(0.0, |_, stats| {
                let fraction = probe.eligible_fraction(&stats.seen_prefixes);
                if let Some(hash) = probe.full_hash() {
                    stats.seen_prefixes.insert(hash);
                }
                fraction
            })
    }

    /// Returns a serializable point-in-time snapshot.
    pub(crate) fn snapshot(&self) -> StatsSnapshot {
        self.lock().snapshot()
    }

    /// Clears all accumulated stats.
    pub(crate) fn reset(&self) {
        self.lock().reset();
    }

    fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {
        self.inner.lock()
    }
}

struct StatsAccumulatorInner {
    by_model: BTreeMap<ModelId, ModelStats>,
    by_target: BTreeMap<ModelId, TargetStats>,
    total_requests: u64,
    total_errors: u64,
    routing_overhead: LatencyHistogram,
    routing_fallbacks: RoutingFallbackStats,
    by_classifier: BTreeMap<ModelId, ModelStats>,
    by_classifier_target: BTreeMap<ModelId, TargetStats>,
    classifier_requests: u64,
    classifier_errors: u64,
    algorithm_stats: AlgorithmStats,
}

impl StatsAccumulatorInner {
    fn new<'a>(registry: Registry, algorithms: impl IntoIterator<Item = &'a str>) -> Self {
        Self {
            by_model: BTreeMap::new(),
            by_target: BTreeMap::new(),
            total_requests: 0,
            total_errors: 0,
            routing_overhead: LatencyHistogram::default(),
            routing_fallbacks: RoutingFallbackStats::default(),
            by_classifier: BTreeMap::new(),
            by_classifier_target: BTreeMap::new(),
            classifier_requests: 0,
            classifier_errors: 0,
            algorithm_stats: AlgorithmStats::new(registry, algorithms),
        }
    }

    fn model_stats_mut(
        &mut self,
        selected: ModelId,
        route: Option<&Route>,
    ) -> impl Iterator<Item = &mut ModelStats> {
        stats_entries(&mut self.by_model, &mut self.by_target, selected, route)
    }

    fn classifier_stats_mut(
        &mut self,
        selected: ModelId,
        route: Option<&Route>,
    ) -> impl Iterator<Item = &mut ModelStats> {
        stats_entries(
            &mut self.by_classifier,
            &mut self.by_classifier_target,
            selected,
            route,
        )
    }

    fn snapshot(&self) -> StatsSnapshot {
        let (models, total_tokens) =
            build_model_snapshots(self.by_model.iter(), self.total_requests);
        let mut classifier = build_classifier_snapshot(
            &self.by_classifier,
            self.classifier_requests,
            self.classifier_errors,
        );
        classifier.targets =
            build_target_snapshots(&self.by_classifier_target, self.classifier_requests);
        StatsSnapshot {
            targets: build_target_snapshots(&self.by_target, self.total_requests),
            total_requests: self.total_requests,
            total_errors: self.total_errors,
            total_tokens,
            models,
            routing_overhead: self.routing_overhead.snapshot(),
            routing_fallbacks: self.routing_fallbacks,
            classifier,
            algorithm_stats: self.algorithm_stats.snapshot(),
        }
    }

    fn reset(&mut self) {
        self.by_model.clear();
        self.by_target.clear();
        self.total_requests = 0;
        self.total_errors = 0;
        self.routing_overhead = LatencyHistogram::default();
        self.routing_fallbacks = RoutingFallbackStats::default();
        self.by_classifier.clear();
        self.by_classifier_target.clear();
        self.classifier_requests = 0;
        self.classifier_errors = 0;
        self.algorithm_stats.reset();
    }
}

/// One configured target's counters and the provider settings that distinguish it.
struct TargetStats {
    model: ModelId,
    reasoning_effort: Option<String>,
    stats: ModelStats,
}

#[derive(Clone, Debug, PartialEq, Serialize)]
pub(crate) struct TargetStatsSnapshot {
    model: ModelId,
    reasoning_effort: Option<String>,
    #[serde(flatten)]
    stats: ModelStatsSnapshot,
}

/// Updates the legacy model aggregate and the independent target breakdown together.
fn stats_entries<'a>(
    models: &'a mut BTreeMap<ModelId, ModelStats>,
    targets: &'a mut BTreeMap<ModelId, TargetStats>,
    selected: ModelId,
    route: Option<&Route>,
) -> impl Iterator<Item = &'a mut ModelStats> + use<'a> {
    let target = route.and_then(|route| route.target(&selected));
    let model = target.map_or(&selected, |target| &target.model).clone();
    let model_stats = models.entry(model).or_default();
    let target_stats = target.map(|target: &DecisionTarget| {
        &mut targets
            .entry(selected)
            .or_insert_with(|| TargetStats {
                model: target.model.clone(),
                reasoning_effort: target.reasoning_effort.clone(),
                stats: ModelStats::default(),
            })
            .stats
    });
    std::iter::once(model_stats).chain(target_stats)
}

fn build_target_snapshots(
    targets: &BTreeMap<ModelId, TargetStats>,
    requests: u64,
) -> BTreeMap<ModelId, TargetStatsSnapshot> {
    build_model_snapshots(
        targets.iter().map(|(name, target)| (name, &target.stats)),
        requests,
    )
    .0
    .into_iter()
    .map(|(name, stats)| {
        let target = &targets[&name];
        (
            name,
            TargetStatsSnapshot {
                model: target.model.clone(),
                reasoning_effort: target.reasoning_effort.clone(),
                stats,
            },
        )
    })
    .collect()
}

#[derive(Clone, Debug, Default)]
struct ModelStats {
    calls: u64,
    errors: u64,
    prompt_tokens: u64,
    max_observed_context_tokens: u64,
    completion_tokens: u64,
    cached_tokens: u64,
    cache_creation_tokens: u64,
    cacheable_prompt_tokens: u64,
    reasoning_tokens: u64,
    seen_prefixes: HashSet<u64>,
    model_call_latency: LatencyHistogram,
    total_latency: LatencyHistogram,
}

impl ModelStats {
    fn add_usage(&mut self, usage: TokenUsage) {
        self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);
        self.completion_tokens = self
            .completion_tokens
            .saturating_add(usage.completion_tokens);
        self.cached_tokens = self.cached_tokens.saturating_add(usage.cached_tokens);
        self.cache_creation_tokens = self
            .cache_creation_tokens
            .saturating_add(usage.cache_creation_tokens);
        self.cacheable_prompt_tokens = self
            .cacheable_prompt_tokens
            .saturating_add(usage.cacheable_prompt_tokens);
        self.reasoning_tokens = self.reasoning_tokens.saturating_add(usage.reasoning_tokens);
        self.max_observed_context_tokens = self
            .max_observed_context_tokens
            .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));
    }
}

#[derive(Clone, Debug)]
struct LatencyHistogram {
    count: u64,
    total_ms: f64,
    min_ms: f64,
    max_ms: f64,
    samples: Vec<f64>,
}

impl Default for LatencyHistogram {
    fn default() -> Self {
        Self {
            count: 0,
            total_ms: 0.0,
            min_ms: f64::INFINITY,
            max_ms: 0.0,
            samples: Vec::new(),
        }
    }
}

impl LatencyHistogram {
    fn record(&mut self, latency_ms: f64) {
        if !latency_ms.is_finite() {
            tracing::debug!(latency_ms, "dropping non-finite latency sample");
            return;
        }
        let latency_ms = latency_ms.max(0.0);
        self.count = self.count.saturating_add(1);
        self.total_ms += latency_ms;
        self.min_ms = self.min_ms.min(latency_ms);
        self.max_ms = self.max_ms.max(latency_ms);
        if self.samples.len() < MAX_LATENCY_SAMPLES {
            self.samples.push(latency_ms);
        } else {
            let index = self.count.saturating_sub(1) as usize % MAX_LATENCY_SAMPLES;
            self.samples[index] = latency_ms;
        }
    }

    fn snapshot(&self) -> LatencyHistogramSnapshot {
        if self.count == 0 {
            return LatencyHistogramSnapshot::default();
        }
        let mut samples = self.samples.to_vec();
        samples.sort_by(f64::total_cmp);
        let count = samples.len();
        let p99_index = count.saturating_sub(1).min((count as f64 * 0.99) as usize);
        LatencyHistogramSnapshot {
            count: self.count,
            total_ms: round2(self.total_ms),
            min_ms: round2(self.min_ms),
            max_ms: round2(self.max_ms),
            avg_ms: round2(self.total_ms / self.count as f64),
            p50_ms: samples.get(count / 2).copied().map(round2).unwrap_or(0.0),
            p99_ms: samples.get(p99_index).copied().map(round2).unwrap_or(0.0),
        }
    }
}

/// Full JSON stats snapshot.
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub(crate) struct StatsSnapshot {
    pub total_requests: u64,
    pub total_errors: u64,
    pub total_tokens: TokenTotals,
    pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
    pub targets: BTreeMap<ModelId, TargetStatsSnapshot>,
    pub routing_overhead: LatencyHistogramSnapshot,
    pub routing_fallbacks: RoutingFallbackStats,
    pub classifier: ClassifierStatsSnapshot,
    pub algorithm_stats: AlgorithmStatsSnapshot,
}

/// Legacy fallback counters retained in the stats response shape.
///
/// New fallback details are logged instead.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
pub(crate) struct RoutingFallbackStats {
    pub context_window: u64,
    pub unavailable: u64,
}

#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub(crate) struct ClassifierStatsSnapshot {
    pub total_requests: u64,
    pub total_errors: u64,
    pub total_tokens: TokenTotals,
    pub models: BTreeMap<ModelId, ModelStatsSnapshot>,
    pub targets: BTreeMap<ModelId, TargetStatsSnapshot>,
}

#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
pub(crate) struct TokenTotals {
    pub prompt: u64,
    pub completion: u64,
    pub cached: u64,
    pub cache_creation: u64,
    pub reasoning: u64,
    pub total: u64,
}

#[derive(Clone, Debug, PartialEq, Serialize)]
pub(crate) struct ModelStatsSnapshot {
    pub calls: u64,
    pub errors: u64,
    pub request_pct: f64,
    pub prompt_tokens: u64,
    pub max_observed_context_tokens: u64,
    pub completion_tokens: u64,
    pub total_tokens: u64,
    pub token_pct: f64,
    pub cached_tokens: u64,
    pub cache_creation_tokens: u64,
    pub reasoning_tokens: u64,
    pub avg_prompt_tokens: f64,
    pub avg_completion_tokens: f64,
    pub cache_hit_rate: f64,
    pub theoretical_cache_hit_rate: f64,
    pub model_call_latency: LatencyHistogramSnapshot,
    pub total_latency: LatencyHistogramSnapshot,
}

#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize)]
pub(crate) struct LatencyHistogramSnapshot {
    pub count: u64,
    pub total_ms: f64,
    pub min_ms: f64,
    pub max_ms: f64,
    pub avg_ms: f64,
    pub p50_ms: f64,
    pub p99_ms: f64,
}

fn build_model_snapshots<'a>(
    by_model: impl Iterator<Item = (&'a ModelId, &'a ModelStats)> + Clone,
    total_requests: u64,
) -> (BTreeMap<ModelId, ModelStatsSnapshot>, TokenTotals) {
    let mut totals = TokenTotals::default();
    for (_, stats) in by_model.clone() {
        totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens);
        totals.completion = totals.completion.saturating_add(stats.completion_tokens);
        totals.cached = totals.cached.saturating_add(stats.cached_tokens);
        totals.cache_creation = totals
            .cache_creation
            .saturating_add(stats.cache_creation_tokens);
        totals.reasoning = totals.reasoning.saturating_add(stats.reasoning_tokens);
    }
    totals.total = totals.prompt.saturating_add(totals.completion);

    let models = by_model
        .map(|(model, stats)| {
            let total_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);
            let snapshot = ModelStatsSnapshot {
                calls: stats.calls,
                errors: stats.errors,
                request_pct: percentage(stats.calls, total_requests),
                prompt_tokens: stats.prompt_tokens,
                max_observed_context_tokens: stats.max_observed_context_tokens,
                completion_tokens: stats.completion_tokens,
                total_tokens,
                token_pct: percentage(total_tokens, totals.total),
                cached_tokens: stats.cached_tokens,
                cache_creation_tokens: stats.cache_creation_tokens,
                reasoning_tokens: stats.reasoning_tokens,
                avg_prompt_tokens: average(stats.prompt_tokens, stats.calls),
                avg_completion_tokens: average(stats.completion_tokens, stats.calls),
                cache_hit_rate: ratio4(stats.cached_tokens, stats.prompt_tokens),
                theoretical_cache_hit_rate: ratio4(
                    stats.cacheable_prompt_tokens,
                    stats.prompt_tokens,
                ),
                model_call_latency: stats.model_call_latency.snapshot(),
                total_latency: stats.total_latency.snapshot(),
            };
            (model.clone(), snapshot)
        })
        .collect();
    (models, totals)
}

fn build_classifier_snapshot(
    models: &BTreeMap<ModelId, ModelStats>,
    total_requests: u64,
    total_errors: u64,
) -> ClassifierStatsSnapshot {
    let (models, total_tokens) = build_model_snapshots(models.iter(), total_requests);
    ClassifierStatsSnapshot {
        total_requests,
        total_errors,
        total_tokens,
        models,
        targets: BTreeMap::new(),
    }
}

fn percentage(numerator: u64, denominator: u64) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        round2(numerator as f64 / denominator as f64 * 100.0)
    }
}

fn average(total: u64, count: u64) -> f64 {
    if count == 0 {
        0.0
    } else {
        round2(total as f64 / count as f64)
    }
}

fn ratio4(numerator: u64, denominator: u64) -> f64 {
    if denominator == 0 {
        0.0
    } else {
        round4(numerator as f64 / denominator as f64)
    }
}

fn round2(value: f64) -> f64 {
    (value * 100.0).round() / 100.0
}

fn round4(value: f64) -> f64 {
    (value * 10_000.0).round() / 10_000.0
}

#[cfg(test)]
mod tests {
    use serde_json::json;

    use super::*;
    use crate::stats::cache_eligibility::prefix_probe;

    fn usage(prompt: u64, completion: u64) -> TokenUsage {
        TokenUsage {
            prompt_tokens: prompt,
            completion_tokens: completion,
            ..TokenUsage::default()
        }
    }

    /// Shared-model targets contribute once to totals, with independent errors and cache state.
    #[test]
    fn shared_model_targets_keep_independent_counters_and_reset_together() {
        let runner = switchyard_runner::Runner::from_toml(
            r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "http://localhost:1/v1"
[targets.efficient]
id = "shared"
llm_client = "upstream"
reasoning_effort = "medium"
[targets.capable]
id = "shared"
llm_client = "upstream"
reasoning_effort = "max"
[routes.test]
id = "route"
type = "random"
targets = ["efficient", "capable"]
"#,
        )
        .unwrap();
        let root = StatsAccumulator::default();
        let stats = root.for_route(runner.route("route").unwrap().clone());
        for target in ["efficient", "capable"] {
            stats.record_success(target, 10.0);
            stats.record_usage(target, usage(10, 2), 15.0);
            stats.record_classifier_success(target, Some(usage(5, 1)), 3.0);
        }
        stats.record_error("efficient");
        stats.record_response_error("capable");
        stats.record_classifier_error("capable");
        let snapshot = root.snapshot();
        assert_eq!(snapshot.total_requests, 3);
        assert_eq!(snapshot.total_errors, 2);
        assert_eq!(snapshot.total_tokens.prompt, 20);
        assert_eq!(snapshot.models["shared"].calls, 2);
        assert_eq!(snapshot.models["shared"].errors, 2);
        assert_eq!(snapshot.models["shared"].model_call_latency.count, 2);
        assert_eq!(snapshot.targets["efficient"].stats.calls, 1);
        assert_eq!(snapshot.targets["efficient"].stats.errors, 1);
        assert_eq!(snapshot.targets["capable"].stats.errors, 1);
        assert_eq!(snapshot.classifier.total_requests, 3);
        assert_eq!(snapshot.classifier.models["shared"].calls, 2);
        assert_eq!(snapshot.classifier.total_tokens.prompt, 10);
        assert_eq!(snapshot.classifier.targets["capable"].stats.errors, 1);
        assert_eq!(
            snapshot.targets["capable"].reasoning_effort.as_deref(),
            Some("max")
        );
        let probe = prefix_probe(&json!({"messages": [{"role": "user", "content": "repeated"}]}));
        assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 0.0);
        assert_eq!(stats.prefix_eligibility(&"capable".into(), &probe), 0.0);
        assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 1.0);
        root.reset();
        assert_eq!(root.snapshot(), StatsSnapshot::default());
        assert_eq!(stats.prefix_eligibility(&"efficient".into(), &probe), 0.0);
    }

    #[test]
    fn snapshot_aggregates_backend_and_classifier_stats() {
        let stats = StatsAccumulator::default();
        stats.record_success("model/strong", 10.0);
        stats.record_usage(
            "model/strong",
            TokenUsage {
                cached_tokens: 4,
                cache_creation_tokens: 2,
                reasoning_tokens: 3,
                ..usage(10, 5)
            },
            15.0,
        );
        stats.record_routing_overhead(5.0);
        stats.record_success("model/weak", 20.0);
        stats.record_usage("model/weak", usage(20, 10), 30.0);
        stats.record_routing_overhead(10.0);
        stats.record_classifier_success("gemini-3.5-flash", Some(usage(1_000_000, 0)), 8.0);
        let snapshot = stats.snapshot();
        assert_eq!(snapshot.total_tokens.prompt, 30);
        assert_eq!(snapshot.total_tokens.cached, 4);
        assert_eq!(
            snapshot.models["model/strong"].max_observed_context_tokens,
            15
        );
        assert_eq!(
            snapshot.models["model/strong"].theoretical_cache_hit_rate,
            0.0
        );
        assert_eq!(snapshot.routing_overhead.p50_ms, 10.0);
        assert_eq!(snapshot.routing_overhead.p99_ms, 10.0);
        assert_eq!(snapshot.classifier.total_requests, 1);
        assert_eq!(snapshot.classifier.total_tokens.prompt, 1_000_000);
    }

    #[test]
    fn reset_clears_backend_classifier_and_cache_eligibility_state() {
        let stats = StatsAccumulator::default();
        stats.record_success("model/a", 10.0);
        stats.record_usage("model/a", usage(10, 5), 15.0);
        stats.record_routing_overhead(5.0);
        stats.record_classifier_success("model/classifier", Some(usage(4, 1)), 2.0);
        let probe = prefix_probe(&json!({
            "messages": [{"role": "user", "content": "repeat me"}],
        }));
        stats.prefix_eligibility(&ModelId::from("model/a"), &probe);

        stats.reset();

        assert_eq!(stats.snapshot(), StatsSnapshot::default());
        assert_eq!(
            stats.prefix_eligibility(&ModelId::from("model/a"), &probe),
            0.0
        );
    }

    #[test]
    fn theoretical_cache_hit_rate_is_switch_aware() {
        let stats = StatsAccumulator::default();
        let first = prefix_probe(&json!({
            "messages": [{"role": "user", "content": "aaaa"}],
        }));
        let first_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &first);
        stats.record_usage(
            "model/a",
            TokenUsage {
                cacheable_prompt_tokens: (100.0 * first_eligible).round() as u64,
                ..usage(100, 4)
            },
            1.0,
        );

        let second = prefix_probe(&json!({
            "messages": [
                {"role": "user", "content": "aaaa"},
                {"role": "user", "content": "bbbb"},
            ],
        }));
        let second_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &second);
        stats.record_usage(
            "model/a",
            TokenUsage {
                cacheable_prompt_tokens: (100.0 * second_eligible).round() as u64,
                ..usage(100, 4)
            },
            1.0,
        );

        assert_eq!(first_eligible, 0.0);
        assert_eq!(second_eligible, 0.5);
        assert_eq!(
            stats.snapshot().models["model/a"].theoretical_cache_hit_rate,
            0.25
        );
        assert_eq!(
            stats.prefix_eligibility(&ModelId::from("model/b"), &second),
            0.0
        );
    }
}