Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(switchyard-server): Support /v1/stats endpoint (#206)
Graham King88a9fd3031f6
diff --git a/Cargo.lock b/Cargo.lockindex c15c0af6b..5cf8b253d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1806,6 +1806,7 @@ dependencies = ["opentelemetry","opentelemetry-prometheus","opentelemetry_sdk",+ "parking_lot","prometheus","rustls","serde",diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rsindex 0cc7af8b1..7f12b58e3 100644--- a/crates/libsy/examples/research_agent_core.rs+++ b/crates/libsy/examples/research_agent_core.rs@@ -68,7 +68,10 @@ impl ResearchAgent {raw_request: None,metadata: None,};- let stream = self.algo.clone().run_stream(Context::default(), request);+ let stream = self+ .algo+ .clone()+ .run_stream(Context::default(), request, None);tokio::pin!(stream);while let Some(update) = stream.next().await {match update? {diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rsindex b8140a84e..f39692ca7 100644--- a/crates/libsy/examples/streaming_agent.rs+++ b/crates/libsy/examples/streaming_agent.rs@@ -62,7 +62,7 @@ async fn main() -> Result<()> {raw_request: None,metadata: None,};- let stream = algo.run_stream(Context::default(), request);+ let stream = algo.run_stream(Context::default(), request, None);tokio::pin!(stream);while let Some(step) = stream.next().await {diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 0654cfe91..b9ea6b98e 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -357,7 +357,7 @@ mod tests {classifier.score(&mut state, &mut request, Some(&driver)),serve);- selected(classification?.0)+ selected(classification?)}#[tokio::test]diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 81135dab2..c99c1802f 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -33,6 +33,33 @@ use crate::{observability, DriverError, LibsyError, Result};/// `Arc<dyn Algorithm>` object-safe.pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;+/// One completed model call observed at the algorithm offload boundary.+#[derive(Clone, Debug)]+pub struct LlmCallObservation {+ pub selected_model: String,+ /// Routing tier attached to the selected model, when present.+ pub tier: Option<String>,+ /// Whether this was the routed backend call rather than classifier or judge overhead.+ pub is_routed: bool,+ pub is_success: bool,+ /// Time spent waiting for the model call to resolve.+ pub duration: Duration,+ /// Normalized usage for a buffered successful response.+ pub usage: Option<Usage>,+}++/// One request-scoped observation emitted by the algorithm runner.+#[derive(Clone, Debug)]+pub enum RunObservation {+ /// A completed model call.+ LlmCall(LlmCallObservation),+ /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.+ RoutingOverhead(Duration),+}++/// Request-scoped callback for algorithm-run observations.+pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;+/// A request paired with the routing [`Decision`] that produced it — the offload/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.///@@ -118,15 +145,21 @@ pub struct Driver {driver: TypeErasedDriver,// How long the call that served this run took. We need this to calculate routing overhead.routed_call: Arc<Mutex<Option<Duration>>>,+ observer: Option<RunObserver>,}impl Driver {/// Build an empty driver with its step channel ready. Created per call by/// [`run_stream`](Algorithm::run_stream).pub(crate) fn new() -> Self {+ Self::with_observer(None)+ }++ fn with_observer(observer: Option<RunObserver>) -> Self {Self {driver: TypeErasedDriver::new(),routed_call: Arc::new(Mutex::new(None)),+ observer,}}@@ -135,6 +168,14 @@ impl Driver {*self.routed_call.lock()}+ /// Records routing overhead (how long Switchyard added to the call) once+ /// per successful run. Called by `observe_run`.+ pub(crate) fn observe_routing_overhead(&self, duration: Duration) {+ if let Some(observer) = &self.observer {+ observer(RunObservation::RoutingOverhead(duration));+ }+ }+/// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the/// consumer's [`Response`]. The call's context travels inside/// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.@@ -178,6 +219,20 @@ impl Driver {&result,&tracing::Span::current(),);+ if let Some(observer) = &self.observer {+ observer(RunObservation::LlmCall(LlmCallObservation {+ selected_model,+ tier,+ is_routed,+ is_success: result.is_ok(),+ duration: elapsed,+ usage: result+ .as_ref()+ .ok()+ .and_then(|response| response.llm_response.as_agg())+ .map(|response| response.usage.clone()),+ }));+ }// Classifier and judge calls are routing overhead.// And don't record time for failed calls.if is_routed && result.is_ok() {@@ -424,7 +479,13 @@ pub trait Algorithm: Send + Sync + 'static {/// Process a request to completion, returning a stream of [`Step`]s./// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve./// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.- fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {+ /// Report each model call to `observer`.+ fn run_stream(+ self: Arc<Self>,+ ctx: Context,+ request: Request,+ observer: Option<RunObserver>,+ ) -> StepStream {// Stamp the algorithm's telemetry label into the request context; the// context rides on every driver call, so its telemetry is attributed.let mut ctx = ctx;@@ -432,7 +493,7 @@ pub trait Algorithm: Send + Sync + 'static {observability::ALGORITHM_KEY.to_string(),self.name().to_string(),);- let driver = Driver::new();+ let driver = Driver::with_observer(observer);let task_driver = driver.clone();let task_ctx = ctx.clone();let stream = task_driver.stream();@@ -483,6 +544,16 @@ pub trait Algorithm: Send + Sync + 'static {self: Arc<Self>,ctx: Context,request: Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {+ self.run_observed(ctx, request, None).await+ }++ /// Process a request to completion while reporting each model call to `observer`.+ async fn run_observed(+ self: Arc<Self>,+ ctx: Context,+ request: Request,+ observer: Option<RunObserver>,) -> Result<(Vec<Arc<dyn Decision>>, Response)> {// Serve one offloaded call with its target's default client. A failed *model*// call is forwarded to the algorithm via `respond`; this errors only on an@@ -518,7 +589,7 @@ pub trait Algorithm: Send + Sync + 'static {call.respond(result)}- let stream = self.run_stream(ctx, request);+ let stream = self.run_stream(ctx, request, observer);tokio::pin!(stream);let mut trace: Vec<Arc<dyn Decision>> = Vec::new();@@ -663,6 +734,34 @@ mod tests {LlmTargetSet::new(targets)}+ #[tokio::test]+ async fn observed_run_reports_one_successful_routed_call() -> Result<()> {+ let observations = Arc::new(Mutex::new(Vec::new()));+ let observed = observations.clone();+ let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));+ let (_, response) = orch(target_set(&[("direct/model", true)]))+ .run_observed(Context::default(), request(), Some(observer))+ .await?;+ assert_eq!(+ response.llm_response.as_agg().map(completion_text),+ Some("direct/model".to_string())+ );+ let observations = observations.lock();+ assert_eq!(observations.len(), 2);+ let RunObservation::LlmCall(observation) = &observations[0] else {+ return Err(test_error("expected an LLM call observation"));+ };+ assert_eq!(observation.selected_model, "direct/model");+ assert!(observation.is_routed);+ assert!(observation.is_success);+ assert!(observation.usage.is_some());+ assert!(matches!(+ observations[1],+ RunObservation::RoutingOverhead(_)+ ));+ Ok(())+ }+#[test]fn target_lookup_returns_the_missing_target() {let error = target_set(&[]).get_target("missing").err();@@ -764,8 +863,11 @@ mod tests {async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {// A client-less target -> its call is offloaded via a promise the// orchestrator surfaces as a `CallLlm` step for us to fulfill.- let stream =- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ let stream = orch(target_set(&[("offload/model", false)])).run_stream(+ Context::default(),+ request(),+ None,+ );tokio::pin!(stream);let mut saw_call = false;@@ -812,8 +914,11 @@ mod tests {async fn client_backed_target_offloads_with_a_default_client() -> Result<()> {// Every call now offloads to the stream; a client-backed target rides its// client along as `default_client` so the consumer can serve it by default.- let stream =- orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());+ let stream = orch(target_set(&[("direct/model", true)])).run_stream(+ Context::default(),+ request(),+ None,+ );tokio::pin!(stream);let mut final_completion = None;@@ -966,8 +1071,11 @@ mod tests {// A client-less target offloads its call; we fulfill the promise with an// Err, which must flow back through `call_llm_target` into the algorithm and// out as an error step — not a response.- let stream =- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());+ let stream = orch(target_set(&[("offload/model", false)])).run_stream(+ Context::default(),+ request(),+ None,+ );tokio::pin!(stream);let mut saw_error = false;@@ -1041,7 +1149,7 @@ mod tests {dropped: dropped.clone(),});- let stream = algo.run_stream(Context::default(), request());+ let stream = algo.run_stream(Context::default(), request(), None);started_rx.recv().await@@ -1079,7 +1187,7 @@ mod tests {}let algo: Arc<dyn Algorithm> = Arc::new(Panicky);- let stream = algo.run_stream(Context::default(), request());+ let stream = algo.run_stream(Context::default(), request(), None);tokio::pin!(stream);let mut saw_error = false;diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 8c5de7c12..fd5ef5839 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -147,7 +147,11 @@ pub(crate) async fn observe_run(let algorithm = algorithm_label(&ctx);record_run(algorithm, duration, &result, &Span::current());if result.is_ok() {- record_routing_overhead(algorithm, duration, driver.routed_call_duration());+ if let Some(overhead) =+ record_routing_overhead(algorithm, duration, driver.routed_call_duration())+ {+ driver.observe_routing_overhead(overhead);+ }}result}@@ -197,20 +201,23 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp/// Records what routing cost on top of the call that served the run: classifier/// calls, target resolution, decision publishing. A run with no routed call has/// nothing to subtract, so it records nothing.-fn record_routing_overhead(algorithm: &str, run: Duration, routed_call: Option<Duration>) {- let Some(routed_call) = routed_call else {- return;- };+fn record_routing_overhead(+ algorithm: &str,+ run: Duration,+ routed_call: Option<Duration>,+) -> Option<Duration> {+ let routed_call = routed_call?;// Saturating: the two clocks start a moment apart, so a run that is all// routed call can come out fractionally negative.- let overhead_ms = run.saturating_sub(routed_call).as_secs_f64() * 1000.0;+ let overhead = run.saturating_sub(routed_call);meter().f64_histogram("switchyard.routing_overhead_ms").build().record(- overhead_ms,+ overhead.as_secs_f64() * 1000.0,&[KeyValue::new("algorithm", algorithm.to_string())],);+ Some(overhead)}/// Records the resolution of one offloaded model call: the call counter anddiff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rsindex 50d738978..896929a48 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy/tests/observability.rs@@ -620,6 +620,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::let stream = algo(ALGO, MODEL, None).run_stream(Context::default(),request_with_metadata("obs-session-2", "obs-corr-2"),+ None,);tokio::pin!(stream);diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 753333d32..424d73a5f 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -23,6 +23,7 @@ libsy = { package = "switchyard-libsy", path = "../libsy" }opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }opentelemetry-prometheus = "0.32"opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }+parking_lot.workspace = trueprometheus = "0.14"serde.workspace = truetoml = "1.1"diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex edc6107bb..db84554db 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -7,6 +7,7 @@ pub mod config;mod metrics;mod response;mod sse;+mod stats;mod usage_metrics;use std::collections::BTreeMap;@@ -25,7 +26,10 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Extension, Json, Router};use axum_server::tls_rustls::RustlsConfig;-use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};+use libsy::{+ Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request, RunObservation,+ RunObserver,+};use serde_json::{json, Value};use tokio::net::{TcpListener, TcpSocket};use tracing::Level;@@ -33,6 +37,7 @@ use tracing::Level;use switchyard_translation::{decode_request, WireFormat};use crate::response::into_http_response;+use crate::stats::{prefix_probe, tracking_enabled_from_env, StatsAccumulator, StatsSnapshot};/// Default TCP listen backlog used by the Rust server.pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;@@ -75,6 +80,8 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;pub struct ServerState {routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,metrics: prometheus::Registry,+ stats: StatsAccumulator,+ track_cache_eligibility: bool,}impl ServerState {@@ -99,6 +106,8 @@ impl ServerState {Ok(Self {routes: Arc::new(entries),metrics,+ stats: StatsAccumulator::default(),+ track_cache_eligibility: tracking_enabled_from_env(),})}@@ -198,6 +207,34 @@ async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {#[derive(Clone, Copy)]struct RequestStart(Instant);+/// Maps routed call observations to backend stats, non-routed calls to classifier/judge+/// stats, and records routing overhead once the algorithm run completes.+fn stats_observer(stats: StatsAccumulator) -> RunObserver {+ Arc::new(move |observation| match observation {+ RunObservation::LlmCall(call) => {+ let latency_ms = call.duration.as_secs_f64() * 1_000.0;+ if call.is_routed {+ if call.is_success {+ stats.record_success(&call.selected_model, latency_ms, call.tier.as_deref());+ } else {+ stats.record_error(&call.selected_model, call.tier.as_deref());+ }+ } else if call.is_success {+ stats.record_classifier_success(+ call.selected_model,+ call.usage.as_ref().map(usage_metrics::token_usage),+ latency_ms,+ );+ } else {+ stats.record_classifier_error(call.selected_model);+ }+ }+ RunObservation::RoutingOverhead(duration) => {+ stats.record_routing_overhead(duration.as_secs_f64() * 1_000.0);+ }+ })+}+/// Stamps the ingress instant into request extensions. Runs as a router layer,/// so it executes before the handlers' `Json` extractor buffers the body —/// request-latency measurements therefore include body read and decode.@@ -216,6 +253,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.route("/v1/responses", post(openai_responses)).route("/v1/messages/count_tokens", post(anthropic_count_tokens)).route("/v1/models", get(models))+ .route("/v1/stats", get(get_stats)).route("/metrics", get(prometheus_metrics)).route("/health", get(health)).fallback(not_found)@@ -421,24 +459,40 @@ async fn handle_llm_request(body: Value,wire_format: WireFormat,) -> Response {+ let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));let (algorithm, request) = match resolve_route(&state, metadata, body, wire_format) {Ok(resolved) => resolved,Err(response) => return response,};- let (trace, response) = match algorithm.run(Context::default(), request).await {+ let observer = stats_observer(state.stats.clone());+ let (trace, response) = match algorithm+ .run_observed(Context::default(), request, Some(observer))+ .await+ {Ok(result) => result,Err(error) => return algorithm_error(error),};+// Metrics, response body, and routing header all read the same decision, so// the model they name can never disagree. An empty trace leaves the body with// the id the upstream reported.let decision = trace.last();let response = if let Some(decision) = decision {+ let cache_eligible = cache_probe+ .as_ref()+ .map(|probe| {+ state+ .stats+ .prefix_eligibility(decision.selected_model(), probe)+ })+ .unwrap_or(0.0);usage_metrics::observe(response,decision.selected_model(),decision.routing_tier(),started.0,+ state.stats,+ cache_eligible,)} else {response@@ -662,6 +716,10 @@ async fn models(State(state): State<ServerState>) -> Json<Value> {Json(model_list_payload(state.models()))}+async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {+ Json(state.stats.snapshot())+}+async fn health() -> Json<Value> {Json(json!({"status": "ok"}))}diff --git a/crates/switchyard-server/src/stats.rs b/crates/switchyard-server/src/stats.rsnew file mode 100644index 000000000..ebb644578--- /dev/null+++ b/crates/switchyard-server/src/stats.rs@@ -0,0 +1,10 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Process-local JSON stats accounting for the Rust server.++mod accumulator;+mod cache_eligibility;++pub(crate) use accumulator::{StatsAccumulator, StatsSnapshot, TokenUsage};+pub(crate) use cache_eligibility::{prefix_probe, tracking_enabled_from_env};diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rsnew file mode 100644index 000000000..2aa924ec7--- /dev/null+++ b/crates/switchyard-server/src/stats/accumulator.rs@@ -0,0 +1,607 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Thread-safe stats accumulator and serializable snapshot schema.++use std::collections::{BTreeMap, BTreeSet, HashSet};+use std::sync::Arc;++use parking_lot::{Mutex, MutexGuard};+use serde::Serialize;++use super::cache_eligibility::PrefixProbe;++const MAX_LATENCY_SAMPLES: usize = 10_000;++/// Normalized token counters recorded by the server.+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]+pub(crate) struct TokenUsage {+ pub prompt_tokens: u64,+ pub completion_tokens: u64,+ pub cached_tokens: u64,+ pub cache_creation_tokens: u64,+ pub cacheable_prompt_tokens: u64,+ pub reasoning_tokens: u64,+}++/// Thread-safe process-local stats store.+#[derive(Clone, Debug, Default)]+pub(crate) struct StatsAccumulator {+ inner: Arc<Mutex<StatsAccumulatorInner>>,+}++impl StatsAccumulator {+ /// Records one successful routed backend call.+ pub(crate) fn record_success(+ &self,+ model: impl Into<String>,+ backend_latency_ms: f64,+ tier: Option<&str>,+ ) {+ let mut inner = self.lock();+ inner.total_requests = inner.total_requests.saturating_add(1);+ let model = model.into();+ let tier = normalized_tier(tier);+ let stats = inner.model_stats_mut(model.clone());+ stats.calls = stats.calls.saturating_add(1);+ stats.model_call_latency.record(backend_latency_ms);+ if let Some(tier) = tier {+ stats.tiers.insert(tier.to_string());+ let tier_stats = inner.tier_stats_mut(tier, &model);+ tier_stats.calls = tier_stats.calls.saturating_add(1);+ }+ }++ /// Records one failed routed backend call.+ pub(crate) fn record_error(&self, model: impl Into<String>, tier: Option<&str>) {+ let mut inner = self.lock();+ inner.total_requests = inner.total_requests.saturating_add(1);+ inner.total_errors = inner.total_errors.saturating_add(1);+ let model = model.into();+ let stats = inner.model_stats_mut(model.clone());+ stats.errors = stats.errors.saturating_add(1);+ if let Some(tier) = normalized_tier(tier) {+ stats.tiers.insert(tier.to_string());+ inner.tier_stats_mut(tier, &model);+ }+ }++ /// Records usage and terminal latency after a successful routed call.+ pub(crate) fn record_usage(+ &self,+ model: impl Into<String>,+ usage: TokenUsage,+ total_latency_ms: f64,+ tier: Option<&str>,+ ) {+ let mut inner = self.lock();+ let model = model.into();+ let stats = inner.model_stats_mut(model.clone());+ stats.add_usage(usage);+ stats.total_latency.record(total_latency_ms);+ if let Some(tier) = normalized_tier(tier) {+ stats.tiers.insert(tier.to_string());+ inner.tier_stats_mut(tier, &model).add_usage(usage);+ }+ }++ /// Records routing time for one completed algorithm run.+ pub(crate) fn record_routing_overhead(&self, routing_overhead_ms: f64) {+ self.lock().routing_overhead.record(routing_overhead_ms);+ }++ /// Records one successful classifier or judge call.+ pub(crate) fn record_classifier_success(+ &self,+ model: impl Into<String>,+ usage: Option<TokenUsage>,+ latency_ms: f64,+ ) {+ let mut inner = self.lock();+ inner.classifier_requests = inner.classifier_requests.saturating_add(1);+ let stats = inner.classifier_stats_mut(model.into());+ stats.calls = stats.calls.saturating_add(1);+ if let Some(usage) = usage {+ stats.add_usage(usage);+ }+ stats.model_call_latency.record(latency_ms);+ stats.total_latency.record(latency_ms);+ }++ /// Records one failed classifier or judge call.+ pub(crate) fn record_classifier_error(&self, model: impl Into<String>) {+ let mut inner = self.lock();+ inner.classifier_requests = inner.classifier_requests.saturating_add(1);+ inner.classifier_errors = inner.classifier_errors.saturating_add(1);+ let stats = inner.classifier_stats_mut(model.into());+ stats.errors = stats.errors.saturating_add(1);+ }++ /// Returns the cache-eligible fraction for `model` and records the prefix as seen.+ pub(crate) fn prefix_eligibility(&self, model: &str, probe: &PrefixProbe) -> f64 {+ let mut inner = self.lock();+ let stats = inner.model_stats_mut(model.to_string());+ let fraction = probe.eligible_fraction(&stats.seen_prefixes);+ if let Some(hash) = probe.full_hash() {+ stats.seen_prefixes.insert(hash);+ }+ fraction+ }++ /// Returns a serializable point-in-time snapshot.+ pub(crate) fn snapshot(&self) -> StatsSnapshot {+ let inner = self.lock().clone();+ inner.snapshot()+ }++ fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {+ self.inner.lock()+ }+}++fn normalized_tier(tier: Option<&str>) -> Option<&str> {+ tier.map(str::trim).filter(|tier| !tier.is_empty())+}++#[derive(Clone, Debug, Default)]+struct StatsAccumulatorInner {+ by_model: BTreeMap<String, ModelStats>,+ by_tier: BTreeMap<String, TierStats>,+ total_requests: u64,+ total_errors: u64,+ routing_overhead: LatencyHistogram,+ by_classifier: BTreeMap<String, ModelStats>,+ classifier_requests: u64,+ classifier_errors: u64,+}++impl StatsAccumulatorInner {+ fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {+ self.by_model.entry(model).or_default()+ }++ fn classifier_stats_mut(&mut self, model: String) -> &mut ModelStats {+ self.by_classifier.entry(model).or_default()+ }++ fn tier_stats_mut(&mut self, tier: &str, model: &str) -> &mut TierStats {+ let stats = self.by_tier.entry(tier.to_string()).or_default();+ stats.models.insert(model.to_string());+ stats+ }++ fn snapshot(&self) -> StatsSnapshot {+ let (models, total_tokens) = build_model_snapshots(&self.by_model, self.total_requests);+ let classifier = build_classifier_snapshot(+ &self.by_classifier,+ self.classifier_requests,+ self.classifier_errors,+ );+ StatsSnapshot {+ total_requests: self.total_requests,+ total_errors: self.total_errors,+ total_tokens,+ models,+ tiers: tier_snapshots(&self.by_tier, total_tokens.total, self.total_requests),+ routing_overhead: self.routing_overhead.snapshot(),+ classifier,+ }+ }+}++#[derive(Clone, Debug, Default)]+struct ModelStats {+ calls: u64,+ errors: u64,+ prompt_tokens: u64,+ max_observed_context_tokens: u64,+ completion_tokens: u64,+ cached_tokens: u64,+ cache_creation_tokens: u64,+ cacheable_prompt_tokens: u64,+ reasoning_tokens: u64,+ seen_prefixes: HashSet<u64>,+ model_call_latency: LatencyHistogram,+ total_latency: LatencyHistogram,+ tiers: BTreeSet<String>,+}++impl ModelStats {+ fn add_usage(&mut self, usage: TokenUsage) {+ self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);+ self.completion_tokens = self+ .completion_tokens+ .saturating_add(usage.completion_tokens);+ self.cached_tokens = self.cached_tokens.saturating_add(usage.cached_tokens);+ self.cache_creation_tokens = self+ .cache_creation_tokens+ .saturating_add(usage.cache_creation_tokens);+ self.cacheable_prompt_tokens = self+ .cacheable_prompt_tokens+ .saturating_add(usage.cacheable_prompt_tokens);+ self.reasoning_tokens = self.reasoning_tokens.saturating_add(usage.reasoning_tokens);+ self.max_observed_context_tokens = self+ .max_observed_context_tokens+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));+ }+}++#[derive(Clone, Debug, Default)]+struct TierStats {+ models: BTreeSet<String>,+ calls: u64,+ prompt_tokens: u64,+ completion_tokens: u64,+}++impl TierStats {+ fn add_usage(&mut self, usage: TokenUsage) {+ self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);+ self.completion_tokens = self+ .completion_tokens+ .saturating_add(usage.completion_tokens);+ }+}++#[derive(Clone, Debug)]+struct LatencyHistogram {+ count: u64,+ total_ms: f64,+ min_ms: f64,+ max_ms: f64,+ samples: Vec<f64>,+}++impl Default for LatencyHistogram {+ fn default() -> Self {+ Self {+ count: 0,+ total_ms: 0.0,+ min_ms: f64::INFINITY,+ max_ms: 0.0,+ samples: Vec::new(),+ }+ }+}++impl LatencyHistogram {+ fn record(&mut self, latency_ms: f64) {+ if !latency_ms.is_finite() {+ tracing::debug!(latency_ms, "dropping non-finite latency sample");+ return;+ }+ let latency_ms = latency_ms.max(0.0);+ self.count = self.count.saturating_add(1);+ self.total_ms += latency_ms;+ self.min_ms = self.min_ms.min(latency_ms);+ self.max_ms = self.max_ms.max(latency_ms);+ if self.samples.len() < MAX_LATENCY_SAMPLES {+ self.samples.push(latency_ms);+ } else {+ let index = self.count.saturating_sub(1) as usize % MAX_LATENCY_SAMPLES;+ self.samples[index] = latency_ms;+ }+ }++ fn snapshot(&self) -> LatencyHistogramSnapshot {+ if self.count == 0 {+ return LatencyHistogramSnapshot::default();+ }+ let mut samples = self.samples.to_vec();+ samples.sort_by(f64::total_cmp);+ let count = samples.len();+ let p99_index = count.saturating_sub(1).min((count as f64 * 0.99) as usize);+ LatencyHistogramSnapshot {+ count: self.count,+ total_ms: round2(self.total_ms),+ min_ms: round2(self.min_ms),+ max_ms: round2(self.max_ms),+ avg_ms: round2(self.total_ms / self.count as f64),+ p50_ms: samples.get(count / 2).copied().map(round2).unwrap_or(0.0),+ p99_ms: samples.get(p99_index).copied().map(round2).unwrap_or(0.0),+ }+ }+}++/// Full JSON stats snapshot.+#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct StatsSnapshot {+ pub total_requests: u64,+ pub total_errors: u64,+ pub total_tokens: TokenTotals,+ pub models: BTreeMap<String, ModelStatsSnapshot>,+ pub tiers: BTreeMap<String, TierStatsSnapshot>,+ pub routing_overhead: LatencyHistogramSnapshot,+ pub classifier: ClassifierStatsSnapshot,+}++#[derive(Clone, Debug, Default, PartialEq, Serialize)]+pub(crate) struct ClassifierStatsSnapshot {+ pub total_requests: u64,+ pub total_errors: u64,+ pub total_tokens: TokenTotals,+ pub models: BTreeMap<String, ModelStatsSnapshot>,+}++#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]+pub(crate) struct TokenTotals {+ pub prompt: u64,+ pub completion: u64,+ pub cached: u64,+ pub cache_creation: u64,+ pub reasoning: u64,+ pub total: u64,+}++#[derive(Clone, Debug, PartialEq, Serialize)]+pub(crate) struct ModelStatsSnapshot {+ pub calls: u64,+ pub errors: u64,+ pub request_pct: f64,+ pub prompt_tokens: u64,+ pub max_observed_context_tokens: u64,+ pub completion_tokens: u64,+ pub total_tokens: u64,+ pub token_pct: f64,+ pub cached_tokens: u64,+ pub cache_creation_tokens: u64,+ pub reasoning_tokens: u64,+ pub avg_prompt_tokens: f64,+ pub avg_completion_tokens: f64,+ pub cache_hit_rate: f64,+ pub theoretical_cache_hit_rate: f64,+ pub model_call_latency: LatencyHistogramSnapshot,+ pub total_latency: LatencyHistogramSnapshot,+ pub tiers: BTreeSet<String>,+}++#[derive(Clone, Debug, PartialEq, Serialize)]+pub(crate) struct TierStatsSnapshot {+ pub models: BTreeSet<String>,+ pub calls: u64,+ pub request_pct: f64,+ pub prompt_tokens: u64,+ pub completion_tokens: u64,+ pub total_tokens: u64,+ pub token_pct: f64,+}++#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize)]+pub(crate) struct LatencyHistogramSnapshot {+ pub count: u64,+ pub total_ms: f64,+ pub min_ms: f64,+ pub max_ms: f64,+ pub avg_ms: f64,+ pub p50_ms: f64,+ pub p99_ms: f64,+}++fn build_model_snapshots(+ by_model: &BTreeMap<String, ModelStats>,+ total_requests: u64,+) -> (BTreeMap<String, ModelStatsSnapshot>, TokenTotals) {+ let mut totals = TokenTotals::default();+ for stats in by_model.values() {+ totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens);+ totals.completion = totals.completion.saturating_add(stats.completion_tokens);+ totals.cached = totals.cached.saturating_add(stats.cached_tokens);+ totals.cache_creation = totals+ .cache_creation+ .saturating_add(stats.cache_creation_tokens);+ totals.reasoning = totals.reasoning.saturating_add(stats.reasoning_tokens);+ }+ totals.total = totals.prompt.saturating_add(totals.completion);++ let models = by_model+ .iter()+ .map(|(model, stats)| {+ let total_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);+ let snapshot = ModelStatsSnapshot {+ calls: stats.calls,+ errors: stats.errors,+ request_pct: percentage(stats.calls, total_requests),+ prompt_tokens: stats.prompt_tokens,+ max_observed_context_tokens: stats.max_observed_context_tokens,+ completion_tokens: stats.completion_tokens,+ total_tokens,+ token_pct: percentage(total_tokens, totals.total),+ cached_tokens: stats.cached_tokens,+ cache_creation_tokens: stats.cache_creation_tokens,+ reasoning_tokens: stats.reasoning_tokens,+ avg_prompt_tokens: average(stats.prompt_tokens, stats.calls),+ avg_completion_tokens: average(stats.completion_tokens, stats.calls),+ cache_hit_rate: ratio4(stats.cached_tokens, stats.prompt_tokens),+ theoretical_cache_hit_rate: ratio4(+ stats.cacheable_prompt_tokens,+ stats.prompt_tokens,+ ),+ model_call_latency: stats.model_call_latency.snapshot(),+ total_latency: stats.total_latency.snapshot(),+ tiers: stats.tiers.clone(),+ };+ (model.clone(), snapshot)+ })+ .collect();+ (models, totals)+}++fn build_classifier_snapshot(+ models: &BTreeMap<String, ModelStats>,+ total_requests: u64,+ total_errors: u64,+) -> ClassifierStatsSnapshot {+ let (models, total_tokens) = build_model_snapshots(models, total_requests);+ ClassifierStatsSnapshot {+ total_requests,+ total_errors,+ total_tokens,+ models,+ }+}++fn tier_snapshots(+ tiers: &BTreeMap<String, TierStats>,+ total_tokens: u64,+ total_requests: u64,+) -> BTreeMap<String, TierStatsSnapshot> {+ tiers+ .iter()+ .map(|(tier, stats)| {+ let tier_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);+ (+ tier.clone(),+ TierStatsSnapshot {+ models: stats.models.clone(),+ calls: stats.calls,+ request_pct: percentage(stats.calls, total_requests),+ prompt_tokens: stats.prompt_tokens,+ completion_tokens: stats.completion_tokens,+ total_tokens: tier_tokens,+ token_pct: percentage(tier_tokens, total_tokens),+ },+ )+ })+ .collect()+}++fn percentage(numerator: u64, denominator: u64) -> f64 {+ if denominator == 0 {+ 0.0+ } else {+ round2(numerator as f64 / denominator as f64 * 100.0)+ }+}++fn average(total: u64, count: u64) -> f64 {+ if count == 0 {+ 0.0+ } else {+ round2(total as f64 / count as f64)+ }+}++fn ratio4(numerator: u64, denominator: u64) -> f64 {+ if denominator == 0 {+ 0.0+ } else {+ round4(numerator as f64 / denominator as f64)+ }+}++fn round2(value: f64) -> f64 {+ (value * 100.0).round() / 100.0+}++fn round4(value: f64) -> f64 {+ (value * 10_000.0).round() / 10_000.0+}++#[cfg(test)]+mod tests {+ use serde_json::json;++ use super::*;+ use crate::stats::cache_eligibility::prefix_probe;++ fn usage(prompt: u64, completion: u64) -> TokenUsage {+ TokenUsage {+ prompt_tokens: prompt,+ completion_tokens: completion,+ ..TokenUsage::default()+ }+ }++ #[test]+ fn snapshot_aggregates_backend_and_classifier_stats() {+ let stats = StatsAccumulator::default();+ stats.record_success("model/strong", 10.0, Some("strong"));+ stats.record_usage(+ "model/strong",+ TokenUsage {+ cached_tokens: 4,+ cache_creation_tokens: 2,+ reasoning_tokens: 3,+ ..usage(10, 5)+ },+ 15.0,+ Some("strong"),+ );+ stats.record_routing_overhead(5.0);+ stats.record_success("model/weak", 20.0, Some("weak"));+ stats.record_usage("model/weak", usage(20, 10), 30.0, Some("weak"));+ stats.record_routing_overhead(10.0);+ stats.record_classifier_success("gemini-3.5-flash", Some(usage(1_000_000, 0)), 8.0);+ let snapshot = stats.snapshot();+ assert_eq!(snapshot.total_tokens.prompt, 30);+ assert_eq!(snapshot.total_tokens.cached, 4);+ assert_eq!(+ snapshot.models["model/strong"].max_observed_context_tokens,+ 15+ );+ assert_eq!(+ snapshot.models["model/strong"].theoretical_cache_hit_rate,+ 0.0+ );+ assert_eq!(snapshot.tiers["strong"].request_pct, 50.0);+ assert_eq!(snapshot.routing_overhead.p50_ms, 10.0);+ assert_eq!(snapshot.routing_overhead.p99_ms, 10.0);+ assert_eq!(snapshot.classifier.total_requests, 1);+ assert_eq!(snapshot.classifier.total_tokens.prompt, 1_000_000);++ stats.record_success("model/strong", 1.0, Some("weak"));+ stats.record_success("model/other", 1.0, Some("strong"));+ let snapshot = stats.snapshot();+ assert_eq!(+ snapshot.models["model/strong"].tiers,+ BTreeSet::from(["strong".to_string(), "weak".to_string()])+ );+ assert_eq!(+ snapshot.tiers["strong"].models,+ BTreeSet::from(["model/other".to_string(), "model/strong".to_string()])+ );+ }++ #[test]+ fn theoretical_cache_hit_rate_is_switch_aware() {+ let stats = StatsAccumulator::default();+ let first = prefix_probe(&json!({+ "messages": [{"role": "user", "content": "aaaa"}],+ }));+ let first_eligible = stats.prefix_eligibility("model/a", &first);+ stats.record_usage(+ "model/a",+ TokenUsage {+ cacheable_prompt_tokens: (100.0 * first_eligible).round() as u64,+ ..usage(100, 4)+ },+ 1.0,+ None,+ );++ let second = prefix_probe(&json!({+ "messages": [+ {"role": "user", "content": "aaaa"},+ {"role": "user", "content": "bbbb"},+ ],+ }));+ let second_eligible = stats.prefix_eligibility("model/a", &second);+ stats.record_usage(+ "model/a",+ TokenUsage {+ cacheable_prompt_tokens: (100.0 * second_eligible).round() as u64,+ ..usage(100, 4)+ },+ 1.0,+ None,+ );++ assert_eq!(first_eligible, 0.0);+ assert_eq!(second_eligible, 0.5);+ assert_eq!(+ stats.snapshot().models["model/a"].theoretical_cache_hit_rate,+ 0.25+ );+ assert_eq!(stats.prefix_eligibility("model/b", &second), 0.0);+ }+}diff --git a/crates/switchyard-server/src/stats/cache_eligibility.rs b/crates/switchyard-server/src/stats/cache_eligibility.rsnew file mode 100644index 000000000..030bdd71e--- /dev/null+++ b/crates/switchyard-server/src/stats/cache_eligibility.rs@@ -0,0 +1,136 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Switch-aware cache eligibility for the Rust server.++use std::collections::hash_map::DefaultHasher;+use std::collections::HashSet;+use std::hash::{Hash, Hasher};++use serde_json::Value;++/// Set this environment variable (to =1) to enable tracking cache prefixes to estimate+/// kv-cache hit as stat `theoretical_cache_hit_rate`.+/// Memory will grow forever so this is only for evals.+const TRACK_ENV: &str = "SWITCHYARD_THEORETICAL_CACHE";++/// Whether theoretical cache-hit tracking is enabled via the environment.+pub(crate) fn tracking_enabled_from_env() -> bool {+ env_opts_in(std::env::var(TRACK_ENV).ok().as_deref())+}++fn env_opts_in(value: Option<&str>) -> bool {+ matches!(+ value+ .map(|value| value.trim().to_ascii_lowercase())+ .as_deref(),+ Some("1" | "true" | "yes" | "on")+ )+}++/// Cumulative prefix fingerprints of a request, in message order.+#[derive(Clone, Debug, Default)]+pub(crate) struct PrefixProbe {+ boundaries: Vec<(u64, u64)>,+ total_len: u64,+}++impl PrefixProbe {+ /// Returns the fraction belonging to the longest prefix previously seen by a model.+ pub(crate) fn eligible_fraction(&self, seen: &HashSet<u64>) -> f64 {+ if self.total_len == 0 {+ return 0.0;+ }+ let eligible = self+ .boundaries+ .iter()+ .filter(|(_, hash)| seen.contains(hash))+ .map(|(len, _)| *len)+ .max()+ .unwrap_or(0);+ eligible as f64 / self.total_len as f64+ }++ /// Returns the full prompt fingerprint to retain after routing it to a model.+ pub(crate) fn full_hash(&self) -> Option<u64> {+ self.boundaries.last().map(|(_, hash)| *hash)+ }+}++/// Builds format-agnostic prefix fingerprints from a request body.+pub(crate) fn prefix_probe(body: &Value) -> PrefixProbe {+ let mut boundaries = Vec::new();+ let mut total_len = 0u64;+ let mut hasher = DefaultHasher::new();++ let system = body.get("system");+ let instructions = body.get("instructions");+ let prefix_len = system.map(text_len).unwrap_or(0) + instructions.map(text_len).unwrap_or(0);+ if prefix_len > 0 {+ total_len += prefix_len;+ for value in [system, instructions].into_iter().flatten() {+ hash_text_into(value, &mut hasher);+ }+ boundaries.push((total_len, hasher.finish()));+ }++ let turns = body+ .get("messages")+ .or_else(|| body.get("input"))+ .and_then(Value::as_array);+ if let Some(turns) = turns {+ for turn in turns {+ total_len += text_len(turn);+ hash_text_into(turn, &mut hasher);+ boundaries.push((total_len, hasher.finish()));+ }+ }+ PrefixProbe {+ boundaries,+ total_len,+ }+}++fn text_len(value: &Value) -> u64 {+ match value {+ Value::String(value) => value.len() as u64,+ Value::Array(items) => items.iter().map(text_len).sum(),+ Value::Object(map) => map.values().map(text_len).sum(),+ _ => 0,+ }+}++fn hash_text_into(value: &Value, hasher: &mut DefaultHasher) {+ match value {+ Value::String(value) => value.hash(hasher),+ Value::Number(value) => value.to_string().hash(hasher),+ Value::Bool(value) => value.hash(hasher),+ Value::Array(items) => items.iter().for_each(|item| hash_text_into(item, hasher)),+ Value::Object(map) => map.iter().for_each(|(key, value)| {+ key.hash(hasher);+ hash_text_into(value, hasher);+ }),+ Value::Null => {}+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn environment_opt_in_values_are_explicit() {+ assert!(env_opts_in(Some(" TRUE ")));+ assert!(env_opts_in(Some("on")));+ assert!(!env_opts_in(Some("false")));+ assert!(!env_opts_in(None));+ }++ #[test]+ fn object_field_names_affect_the_fingerprint() {+ let left = prefix_probe(&serde_json::json!({"messages": [{"left": "same"}]}));+ let right = prefix_probe(&serde_json::json!({"messages": [{"right": "same"}]}));++ assert_ne!(left.full_hash(), right.full_hash());+ }+}diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rsindex 4e9e6cd9d..341e23e66 100644--- a/crates/switchyard-server/src/usage_metrics.rs+++ b/crates/switchyard-server/src/usage_metrics.rs@@ -3,18 +3,22 @@//! Response usage and full-turn latency metrics for routed requests.-use std::time::Instant;+use std::time::{Duration, Instant};use futures_util::StreamExt;use libsy::{LlmResponse, LlmResponseChunk, Response, Usage};use opentelemetry::{global, KeyValue};+use crate::stats::{StatsAccumulator, TokenUsage};+/// Observes a routed response without changing its aggregate or streaming contents.pub(crate) fn observe(response: Response,model: &str,tier: Option<&str>,started: Instant,+ stats: StatsAccumulator,+ cache_eligible: f64,) -> Response {let Response {llm_response,@@ -25,8 +29,14 @@ pub(crate) fn observe(let llm_response = match llm_response {LlmResponse::Agg(agg) => {- record_usage(&agg.usage, &model, tier.as_deref());- record_latency(&model, tier.as_deref(), started);+ record_terminal(+ &stats,+ &agg.usage,+ &model,+ tier.as_deref(),+ started,+ cache_eligible,+ );LlmResponse::Agg(agg)}LlmResponse::Stream(mut stream) => {@@ -49,10 +59,14 @@ pub(crate) fn observe(return;}}- if let Some(usage) = latest_usage {- record_usage(&usage, &model, tier.as_deref());- }- record_latency(&model, tier.as_deref(), started);+ record_terminal(+ &stats,+ &latest_usage.unwrap_or_default(),+ &model,+ tier.as_deref(),+ started,+ cache_eligible,+ );};LlmResponse::Stream(Box::pin(wrapped))}@@ -64,6 +78,46 @@ pub(crate) fn observe(}}+pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {+ let cached_tokens = usage.cached_input_tokens().unwrap_or(0);+ let cache_creation_tokens = usage.cache_creation_input_tokens().unwrap_or(0);+ TokenUsage {+ prompt_tokens: usage+ .input_tokens+ .unwrap_or(0)+ .saturating_add(cached_tokens)+ .saturating_add(cache_creation_tokens),+ completion_tokens: usage.output_tokens.unwrap_or(0),+ cached_tokens,+ cache_creation_tokens,+ cacheable_prompt_tokens: 0,+ reasoning_tokens: usage.reasoning_tokens.unwrap_or(0),+ }+}++/// Records final usage and latency in both OpenTelemetry metrics and JSON stats.+fn record_terminal(+ stats: &StatsAccumulator,+ usage: &Usage,+ model: &str,+ tier: Option<&str>,+ started: Instant,+ cache_eligible: f64,+) {+ let total_latency = started.elapsed();+ record_usage(usage, model, tier);+ record_latency(model, tier, total_latency);+ let mut token_usage = token_usage(usage);+ token_usage.cacheable_prompt_tokens =+ (token_usage.prompt_tokens as f64 * cache_eligible).round() as u64;+ stats.record_usage(+ model,+ token_usage,+ total_latency.as_secs_f64() * 1_000.0,+ tier,+ );+}+fn attributes(model: &str, tier: Option<&str>) -> Vec<KeyValue> {let mut attributes = vec![KeyValue::new("model", model.to_string())];if let Some(tier) = tier {@@ -98,12 +152,9 @@ fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {}}-fn record_latency(model: &str, tier: Option<&str>, started: Instant) {+fn record_latency(model: &str, tier: Option<&str>, latency: Duration) {global::meter("switchyard").f64_histogram("switchyard.total_latency_ms").build()- .record(- started.elapsed().as_secs_f64() * 1000.0,- &attributes(model, tier),- );+ .record(latency.as_secs_f64() * 1000.0, &attributes(model, tier));}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex b6ee689c6..0581ee153 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -189,6 +189,113 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, RouteOk((upstream, app))}+fn empty_token_totals() -> Value {+ json!({+ "prompt": 0,+ "completion": 0,+ "cached": 0,+ "cache_creation": 0,+ "reasoning": 0,+ "total": 0+ })+}++#[tokio::test]+async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResult {+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;+ let response = send(&app, "GET", "/v1/stats", None).await?;+ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(+ response.json()?,+ json!({+ "total_requests": 0,+ "total_errors": 0,+ "total_tokens": empty_token_totals(),+ "models": {},+ "tiers": {},+ "routing_overhead": {+ "count": 0,+ "total_ms": 0.0,+ "min_ms": 0.0,+ "max_ms": 0.0,+ "avg_ms": 0.0,+ "p50_ms": 0.0,+ "p99_ms": 0.0+ },+ "classifier": {+ "total_requests": 0,+ "total_errors": 0,+ "total_tokens": empty_token_totals(),+ "models": {},+ },+ })+ );+ assert_eq!(+ send(&app, "GET", "/v1/routing/stats", None).await?.status,+ StatusCode::NOT_FOUND+ );+ Ok(())+}++#[tokio::test]+async fn stats_accumulates_buffered_success_error_and_shared_routes() -> TestResult {+ let (_upstream, app) = test_app(&[+ ("switchyard/one", &["gemini-3.5-flash"]),+ ("switchyard/two", &["model/unknown"]),+ ])+ .await?;+ for route in ["switchyard/one", "switchyard/two"] {+ assert_eq!(+ send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": route,+ "messages": [{"role": "user", "content": "hello"}]+ })),+ )+ .await?+ .status,+ StatusCode::OK+ );+ }+ assert_eq!(+ send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": "switchyard/one",+ "messages": [{"role": "user", "content": "fail"}]+ })),+ )+ .await?+ .status,+ StatusCode::IM_A_TEAPOT+ );++ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["total_requests"], 3);+ assert_eq!(stats["total_errors"], 1);+ assert_eq!(+ stats["total_tokens"],+ json!({+ "prompt": 20,+ "completion": 4,+ "cached": 14,+ "cache_creation": 0,+ "reasoning": 0,+ "total": 24+ })+ );+ assert_eq!(stats["models"]["gemini-3.5-flash"]["calls"], 1);+ assert_eq!(stats["models"]["gemini-3.5-flash"]["errors"], 1);+ assert_eq!(stats["models"]["model/unknown"]["calls"], 1);+ assert_eq!(stats["routing_overhead"]["count"], 2);+ Ok(())+}+#[tokio::test]async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {const MODEL: &str = "model/metrics-buffered";@@ -565,6 +672,18 @@ base_threshold = 0.5assert_eq!(calls[1]["model"], "model/classifier");assert_eq!(calls[2]["model"], "model/weak");assert_eq!(calls[3]["model"], "model/weak");+ drop(calls);++ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["total_requests"], 3);+ assert_eq!(stats["models"]["model/weak"]["calls"], 3);+ assert_eq!(stats["tiers"]["weak"]["calls"], 1);+ assert_eq!(stats["classifier"]["total_requests"], 1);+ assert_eq!(+ stats["classifier"]["models"]["model/classifier"]["calls"],+ 1+ );+ assert_eq!(stats["classifier"]["total_tokens"]["prompt"], 10);Ok(())}@@ -982,6 +1101,17 @@ async fn streaming_success_records_only_final_usage_and_one_latency() -> TestRes"unexpected delta for {name}");}+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["total_requests"], 1);+ assert_eq!(+ stats["total_tokens"],+ json!({+ "prompt": 12, "completion": 5, "cached": 7,+ "cache_creation": 2, "reasoning": 3, "total": 17+ })+ );+ assert_eq!(stats["models"][MODEL]["model_call_latency"]["count"], 1);+ assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 1);Ok(())}@@ -1026,6 +1156,12 @@ async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {"{name} changed after a failed stream");}+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;+ assert_eq!(stats["total_requests"], 1);+ assert_eq!(stats["total_tokens"], empty_token_totals());+ assert_eq!(stats["models"][MODEL]["calls"], 1);+ assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 0);+ assert_eq!(stats["routing_overhead"]["count"], 1);Ok(())}