Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): expose stage router stats (#344)
nachiketb-nvidia6df31a9c3d8e
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex 32fcf8f52..1976f6b94 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -62,6 +62,11 @@ const EXPLORING_METRIC: &str = "switchyard.stage_router.exploring";/// Distribution of production-oriented tool activity.const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_intensity";+// Histogram boundaries live with the instruments so every host exports the+// same stage-router distributions without duplicating algorithm knowledge.+const SCORE_BUCKETS: &[f64] = &[-1.0, -0.75, -0.5, -0.25, 0.0, 0.25, 0.5, 0.75, 1.0];+const UNIT_BUCKETS: &[f64] = &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0];+/// The two tiers a turn can route to.#[derive(Clone, Copy, Debug, Eq, PartialEq)]pub enum Tier {@@ -288,15 +293,23 @@ fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {};let dimensions = dimensions_from_signal(signal);let meter = meter();- for (name, value) in [- (SCORE_METRIC, score),- (CONFIDENCE_METRIC, confidence),- (SEVERITY_METRIC, dimensions.severity),- (SPINNING_METRIC, dimensions.spinning),- (EXPLORING_METRIC, dimensions.exploring),- (PRODUCTION_INTENSITY_METRIC, dimensions.production_intensity),+ for (name, value, boundaries) in [+ (SCORE_METRIC, score, SCORE_BUCKETS),+ (CONFIDENCE_METRIC, confidence, UNIT_BUCKETS),+ (SEVERITY_METRIC, dimensions.severity, UNIT_BUCKETS),+ (SPINNING_METRIC, dimensions.spinning, UNIT_BUCKETS),+ (EXPLORING_METRIC, dimensions.exploring, UNIT_BUCKETS),+ (+ PRODUCTION_INTENSITY_METRIC,+ dimensions.production_intensity,+ UNIT_BUCKETS,+ ),] {- meter.f64_histogram(name).build().record(value, &[]);+ meter+ .f64_histogram(name)+ .with_boundaries(boundaries.to_vec())+ .build()+ .record(value, &[]);}}diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex b5960eb71..dd57c57d7 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -122,7 +122,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_| `POST` | `/v1/responses` | OpenAI Responses || `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target || `GET` | `/v1/models` | Routes served by this deployment |-| `GET` | `/v1/stats` | Per-model request, token, and cost totals |+| `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats || `POST` | `/v1/stats/reset` | Clear accumulated stats || `GET` | `/metrics` | Prometheus text, see [Metrics](#metrics) || `GET` | `/health` | Liveness |@@ -131,6 +131,10 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":routes through the `[routes.general]` entry above. Any of the three request formats can address anyroute, and the server translates between them.+For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic+target and summarizes its score, confidence, and input-dimension histograms. These values reset+with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative.+Token counting selects an Anthropic-format completion target, preferring target names or model IDscontaining `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order.diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 1f69e0b02..7ad32d81b 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -219,10 +219,14 @@ impl ServerState {return Err(ServerError::new("at least one algorithm route is required"));}let metrics = metrics::registry().map_err(ServerError::new)?;+ let stats = StatsAccumulator::new(+ metrics.clone(),+ entries.values().map(|entry| entry.algorithm.name()),+ );Ok(Self {routes: Arc::new(entries),metrics,- stats: StatsAccumulator::default(),+ stats,routing_log: None,track_cache_eligibility: tracking_enabled_from_env(),})diff --git a/crates/switchyard-server/src/stats.rs b/crates/switchyard-server/src/stats.rsindex ebb644578..1a419ffd2 100644--- a/crates/switchyard-server/src/stats.rs+++ b/crates/switchyard-server/src/stats.rs@@ -4,6 +4,7 @@//! Process-local JSON stats accounting for the Rust server.mod accumulator;+mod algorithms;mod cache_eligibility;pub(crate) use accumulator::{StatsAccumulator, StatsSnapshot, TokenUsage};diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rsindex ee5561808..56d2df552 100644--- a/crates/switchyard-server/src/stats/accumulator.rs+++ b/crates/switchyard-server/src/stats/accumulator.rs@@ -7,8 +7,10 @@ use std::collections::{BTreeMap, HashSet};use std::sync::Arc;use parking_lot::{Mutex, MutexGuard};+use prometheus::Registry;use serde::Serialize;+use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};use super::cache_eligibility::PrefixProbe;const MAX_LATENCY_SAMPLES: usize = 10_000;@@ -25,12 +27,28 @@ pub(crate) struct TokenUsage {}/// Thread-safe process-local stats store.-#[derive(Clone, Debug, Default)]+#[derive(Clone)]pub(crate) struct StatsAccumulator {inner: Arc<Mutex<StatsAccumulatorInner>>,}+impl Default for StatsAccumulator {+ fn default() -> Self {+ Self::new(Registry::new(), std::iter::empty())+ }+}+impl StatsAccumulator {+ /// Creates a stats store for the supplied algorithm names.+ pub(crate) fn new<'a>(+ registry: Registry,+ algorithms: impl IntoIterator<Item = &'a str>,+ ) -> Self {+ Self {+ inner: Arc::new(Mutex::new(StatsAccumulatorInner::new(registry, algorithms))),+ }+ }+/// Records one successful routed backend call.pub(crate) fn record_success(&self, model: impl Into<String>, backend_latency_ms: f64) {let mut inner = self.lock();@@ -115,13 +133,12 @@ impl StatsAccumulator {/// Returns a serializable point-in-time snapshot.pub(crate) fn snapshot(&self) -> StatsSnapshot {- let inner = self.lock().clone();- inner.snapshot()+ self.lock().snapshot()}/// Clears all accumulated stats.pub(crate) fn reset(&self) {- *self.lock() = StatsAccumulatorInner::default();+ self.lock().reset();}fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {@@ -129,7 +146,6 @@ impl StatsAccumulator {}}-#[derive(Clone, Debug, Default)]struct StatsAccumulatorInner {by_model: BTreeMap<String, ModelStats>,total_requests: u64,@@ -139,9 +155,24 @@ struct StatsAccumulatorInner {by_classifier: BTreeMap<String, ModelStats>,classifier_requests: u64,classifier_errors: u64,+ algorithm_stats: AlgorithmStats,}impl StatsAccumulatorInner {+ fn new<'a>(registry: Registry, algorithms: impl IntoIterator<Item = &'a str>) -> Self {+ Self {+ by_model: BTreeMap::new(),+ total_requests: 0,+ total_errors: 0,+ routing_overhead: LatencyHistogram::default(),+ routing_fallbacks: RoutingFallbackStats::default(),+ by_classifier: BTreeMap::new(),+ classifier_requests: 0,+ classifier_errors: 0,+ algorithm_stats: AlgorithmStats::new(registry, algorithms),+ }+ }+fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {self.by_model.entry(model).or_default()}@@ -165,8 +196,21 @@ impl StatsAccumulatorInner {routing_overhead: self.routing_overhead.snapshot(),routing_fallbacks: self.routing_fallbacks,classifier,+ algorithm_stats: self.algorithm_stats.snapshot(),}}++ fn reset(&mut self) {+ self.by_model.clear();+ self.total_requests = 0;+ self.total_errors = 0;+ self.routing_overhead = LatencyHistogram::default();+ self.routing_fallbacks = RoutingFallbackStats::default();+ self.by_classifier.clear();+ self.classifier_requests = 0;+ self.classifier_errors = 0;+ self.algorithm_stats.reset();+ }}#[derive(Clone, Debug, Default)]@@ -275,6 +319,7 @@ pub(crate) struct StatsSnapshot {pub routing_overhead: LatencyHistogramSnapshot,pub routing_fallbacks: RoutingFallbackStats,pub classifier: ClassifierStatsSnapshot,+ pub algorithm_stats: AlgorithmStatsSnapshot,}/// Legacy fallback counters retained in the stats response shape.diff --git a/crates/switchyard-server/src/stats/algorithms.rs b/crates/switchyard-server/src/stats/algorithms.rsnew file mode 100644index 000000000..27bfc0ff6--- /dev/null+++ b/crates/switchyard-server/src/stats/algorithms.rs@@ -0,0 +1,60 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Server-owned projections of algorithm OpenTelemetry metrics.++mod stage_router;++use std::collections::HashSet;++use prometheus::Registry;+use serde::Serialize;++use stage_router::{StageRouterCumulative, StageRouterStatsSnapshot};++const STAGE_ROUTER: &str = "stage_router";++/// Owns algorithm metric baselines behind the generic server stats interface.+pub(super) struct AlgorithmStats {+ registry: Registry,+ stage_router_baseline: Option<StageRouterCumulative>,+}++/// Curated algorithm-specific data included in the JSON stats response.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct AlgorithmStatsSnapshot {+ #[serde(skip_serializing_if = "Option::is_none")]+ pub stage_router: Option<StageRouterStatsSnapshot>,+}++impl AlgorithmStats {+ pub(super) fn new<'a>(+ registry: Registry,+ algorithms: impl IntoIterator<Item = &'a str>,+ ) -> Self {+ let algorithms: HashSet<_> = algorithms.into_iter().collect();+ let families = registry.gather();+ Self {+ stage_router_baseline: algorithms+ .contains(STAGE_ROUTER)+ .then(|| StageRouterCumulative::collect(&families)),+ registry,+ }+ }++ pub(super) fn snapshot(&self) -> AlgorithmStatsSnapshot {+ let families = self.registry.gather();+ AlgorithmStatsSnapshot {+ stage_router: self+ .stage_router_baseline+ .as_ref()+ .map(|baseline| StageRouterCumulative::collect(&families).delta(baseline)),+ }+ }++ pub(super) fn reset(&mut self) {+ if let Some(baseline) = &mut self.stage_router_baseline {+ *baseline = StageRouterCumulative::collect(&self.registry.gather());+ }+ }+}diff --git a/crates/switchyard-server/src/stats/algorithms/stage_router.rs b/crates/switchyard-server/src/stats/algorithms/stage_router.rsnew file mode 100644index 000000000..9b3fc113f--- /dev/null+++ b/crates/switchyard-server/src/stats/algorithms/stage_router.rs@@ -0,0 +1,282 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Stage-router projection from cumulative Prometheus metric families.++use std::collections::BTreeMap;++use prometheus::proto::{Metric, MetricFamily};+use serde::Serialize;++const ROUTING_DECISIONS_METRIC: &str = "switchyard_stage_router_routing_decisions_total";+const SCORE_METRIC: &str = "switchyard_stage_router_score";+const CONFIDENCE_METRIC: &str = "switchyard_stage_router_confidence";+const SEVERITY_METRIC: &str = "switchyard_stage_router_severity";+const SPINNING_METRIC: &str = "switchyard_stage_router_spinning";+const EXPLORING_METRIC: &str = "switchyard_stage_router_exploring";+const PRODUCTION_INTENSITY_METRIC: &str = "switchyard_stage_router_production_intensity";++#[derive(Clone, Debug, Default)]+pub(super) struct StageRouterCumulative {+ decisions: BTreeMap<DecisionKey, u64>,+ score: HistogramTotal,+ confidence: HistogramTotal,+ severity: HistogramTotal,+ spinning: HistogramTotal,+ exploring: HistogramTotal,+ production_intensity: HistogramTotal,+}++#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]+struct DecisionKey {+ source: String,+ target: String,+}++#[derive(Clone, Copy, Debug, Default)]+struct HistogramTotal {+ count: u64,+ sum: f64,+}++/// Human-readable stage-router stats derived from its native metrics.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct StageRouterStatsSnapshot {+ pub routing_decisions: BTreeMap<String, DecisionStatsSnapshot>,+ pub scoring: ScoringStatsSnapshot,+}++/// One decision source, retaining the semantic targets it selected.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct DecisionStatsSnapshot {+ pub total: u64,+ pub targets: BTreeMap<String, u64>,+}++/// Stage scorer output and input-dimension summaries.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct ScoringStatsSnapshot {+ pub score: MetricSummary,+ pub confidence: MetricSummary,+ pub dimensions: DimensionStatsSnapshot,+}++/// Input dimensions evaluated by the stage scorer.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct DimensionStatsSnapshot {+ pub severity: MetricSummary,+ pub spinning: MetricSummary,+ pub exploring: MetricSummary,+ pub production_intensity: MetricSummary,+}++/// Exact count and mean since the last JSON stats reset.+#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize)]+pub(crate) struct MetricSummary {+ pub count: u64,+ pub mean: f64,+}++impl StageRouterCumulative {+ pub(super) fn collect(families: &[MetricFamily]) -> Self {+ Self {+ decisions: collect_decisions(families),+ score: collect_histogram(families, SCORE_METRIC),+ confidence: collect_histogram(families, CONFIDENCE_METRIC),+ severity: collect_histogram(families, SEVERITY_METRIC),+ spinning: collect_histogram(families, SPINNING_METRIC),+ exploring: collect_histogram(families, EXPLORING_METRIC),+ production_intensity: collect_histogram(families, PRODUCTION_INTENSITY_METRIC),+ }+ }++ pub(super) fn delta(&self, baseline: &Self) -> StageRouterStatsSnapshot {+ let mut routing_decisions: BTreeMap<String, DecisionStatsSnapshot> = BTreeMap::new();+ for (key, current) in &self.decisions {+ let count = current.saturating_sub(*baseline.decisions.get(key).unwrap_or(&0));+ if count == 0 {+ continue;+ }+ let source = routing_decisions.entry(key.source.clone()).or_default();+ source.total = source.total.saturating_add(count);+ source.targets.insert(key.target.clone(), count);+ }+ StageRouterStatsSnapshot {+ routing_decisions,+ scoring: ScoringStatsSnapshot {+ score: self.score.delta(baseline.score),+ confidence: self.confidence.delta(baseline.confidence),+ dimensions: DimensionStatsSnapshot {+ severity: self.severity.delta(baseline.severity),+ spinning: self.spinning.delta(baseline.spinning),+ exploring: self.exploring.delta(baseline.exploring),+ production_intensity: self+ .production_intensity+ .delta(baseline.production_intensity),+ },+ },+ }+ }+}++impl HistogramTotal {+ fn delta(self, baseline: Self) -> MetricSummary {+ let count = self.count.saturating_sub(baseline.count);+ if count == 0 {+ return MetricSummary::default();+ }+ let sum = round4(self.sum - baseline.sum);+ MetricSummary {+ count,+ mean: round4(sum / count as f64),+ }+ }+}++fn collect_decisions(families: &[MetricFamily]) -> BTreeMap<DecisionKey, u64> {+ let mut decisions = BTreeMap::new();+ for metric in metrics(families, ROUTING_DECISIONS_METRIC) {+ let Some(source) = label(metric, "decision_source") else {+ continue;+ };+ let Some(target) = label(metric, "target_name") else {+ continue;+ };+ let Some(counter) = metric.get_counter().as_ref() else {+ continue;+ };+ let value = counter.value();+ if value.is_finite() && value > 0.0 {+ let count = decisions+ .entry(DecisionKey {+ source: source.to_string(),+ target: target.to_string(),+ })+ .or_insert(0u64);+ *count = count.saturating_add(value as u64);+ }+ }+ decisions+}++fn collect_histogram(families: &[MetricFamily], name: &str) -> HistogramTotal {+ metrics(families, name).fold(HistogramTotal::default(), |mut total, metric| {+ let Some(histogram) = metric.get_histogram().as_ref() else {+ return total;+ };+ total.count = total.count.saturating_add(histogram.sample_count());+ total.sum += histogram.sample_sum();+ total+ })+}++fn metrics<'a>(families: &'a [MetricFamily], name: &'a str) -> impl Iterator<Item = &'a Metric> {+ families+ .iter()+ .filter(move |family| family.name() == name)+ .flat_map(|family| family.get_metric())+}++fn label<'a>(metric: &'a Metric, name: &str) -> Option<&'a str> {+ metric+ .get_label()+ .iter()+ .find(|label| label.name() == name)+ .map(|label| label.value())+}++fn round4(value: f64) -> f64 {+ let rounded = (value * 10_000.0).round() / 10_000.0;+ if rounded == 0.0 { 0.0 } else { rounded }+}++#[cfg(test)]+mod tests {+ use opentelemetry::KeyValue;+ use opentelemetry::metrics::MeterProvider as _;+ use opentelemetry_sdk::metrics::SdkMeterProvider;+ use prometheus::Registry;++ use super::*;+ use crate::stats::StatsAccumulator;++ #[test]+ fn stage_router_projection_preserves_decisions_scores_and_reset_baseline() {+ let registry = Registry::new();+ let exporter = opentelemetry_prometheus::exporter()+ .with_registry(registry.clone())+ .build()+ .unwrap_or_else(|error| panic!("failed to build metrics exporter: {error}"));+ let provider = SdkMeterProvider::builder().with_reader(exporter).build();+ let meter = provider.meter("switchyard");+ let stats = StatsAccumulator::new(registry, ["stage_router"]);++ meter+ .u64_counter("switchyard.stage_router.routing_decisions")+ .build()+ .add(+ 2,+ &[+ KeyValue::new("decision_source", "dimensions"),+ KeyValue::new("target_name", "model/efficient"),+ ],+ );+ meter+ .u64_counter("switchyard.stage_router.routing_decisions")+ .build()+ .add(+ 1,+ &[+ KeyValue::new("decision_source", "dimensions"),+ KeyValue::new("target_name", "model/capable"),+ ],+ );+ for value in [0.5, -0.25] {+ meter+ .f64_histogram("switchyard.stage_router.score")+ .build()+ .record(value, &[]);+ }+ meter+ .f64_histogram("switchyard.stage_router.confidence")+ .build()+ .record(0.75, &[]);++ let snapshot = stats.snapshot();+ let stage = snapshot+ .algorithm_stats+ .stage_router+ .unwrap_or_else(|| panic!("stage-router stats missing"));+ let dimensions = &stage.routing_decisions["dimensions"];+ assert_eq!(dimensions.total, 3);+ assert_eq!(dimensions.targets["model/efficient"], 2);+ assert_eq!(dimensions.targets["model/capable"], 1);+ assert_eq!(stage.scoring.score.count, 2);+ assert_eq!(stage.scoring.score.mean, 0.125);+ assert_eq!(stage.scoring.confidence.mean, 0.75);++ stats.reset();+ assert_eq!(+ stats.snapshot().algorithm_stats.stage_router,+ Some(StageRouterStatsSnapshot::default())+ );++ meter+ .u64_counter("switchyard.stage_router.routing_decisions")+ .build()+ .add(+ 1,+ &[+ KeyValue::new("decision_source", "override"),+ KeyValue::new("target_name", "model/capable"),+ ],+ );+ let after_reset = stats+ .snapshot()+ .algorithm_stats+ .stage_router+ .unwrap_or_else(|| panic!("stage-router stats missing after reset"));+ assert_eq!(after_reset.routing_decisions["override"].total, 1);+ assert!(!after_reset.routing_decisions.contains_key("dimensions"));+ }+}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 0e3272cb0..65bd5ab79 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -294,6 +294,7 @@ async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResul"total_tokens": empty_token_totals(),"models": {},},+ "algorithm_stats": {},}));assert_eq!(@@ -754,11 +755,11 @@ format = "openai_chat"base_url = "{base_url}"[targets.strong]-id = "model/strong"+id = "model/stats-strong"llm_client = "upstream"[targets.weak]-id = "model/weak"+id = "model/stats-weak"llm_client = "upstream"[routes.stage]@@ -798,9 +799,14 @@ confidence_threshold = 0.5.headers.get("x-model-router-selected-model").and_then(|value| value.to_str().ok()),- Some("model/strong"),+ Some("model/stats-strong"),"a critical error should escalate on the signals alone");+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(+ stats["algorithm_stats"]["stage_router"]["routing_decisions"]["override"]["targets"]["model/stats-strong"],+ 1+ );Ok(())}diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.mdindex 207586134..bba42fbd2 100644--- a/docs/routing_algorithms/stage_router_routing.md+++ b/docs/routing_algorithms/stage_router_routing.md@@ -109,8 +109,8 @@ staying on the default tier.)| `1.0` | yes (required) | Classifier-driven. Tool signals only apply hard overrides; other turns reach the classifier. |The signal-vs-classifier split is dataset-dependent. Measure it in-production: `/v1/stats` reports traffic by tier and model, while response headers-and structured decision logs explain individual selections.+production: `/v1/stats` reports stage-router decisions by source and semantic+target, while response headers and structured decision logs explain individual selections.### Calibrating the threshold from run data