Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(switchyard-server): Support /v1/stats endpoint (#206)

Graham King88a9fd3031f6
diff --git a/Cargo.lock b/Cargo.lock
index c15c0af6b..5cf8b253d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1806,6 +1806,7 @@ dependencies = [
"opentelemetry",
"opentelemetry-prometheus",
"opentelemetry_sdk",
+ "parking_lot",
"prometheus",
"rustls",
"serde",
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index 0cc7af8b1..7f12b58e3 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -68,7 +68,10 @@ impl ResearchAgent {
raw_request: None,
metadata: None,
};
- let stream = self.algo.clone().run_stream(Context::default(), request);
+ let stream = self
+ .algo
+ .clone()
+ .run_stream(Context::default(), request, None);
tokio::pin!(stream);
while let Some(update) = stream.next().await {
match update? {
diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs
index b8140a84e..f39692ca7 100644
--- a/crates/libsy/examples/streaming_agent.rs
+++ b/crates/libsy/examples/streaming_agent.rs
@@ -62,7 +62,7 @@ async fn main() -> Result<()> {
raw_request: None,
metadata: None,
};
- let stream = algo.run_stream(Context::default(), request);
+ let stream = algo.run_stream(Context::default(), request, None);
tokio::pin!(stream);
while let Some(step) = stream.next().await {
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 0654cfe91..b9ea6b98e 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -357,7 +357,7 @@ mod tests {
classifier.score(&mut state, &mut request, Some(&driver)),
serve
);
- selected(classification?.0)
+ selected(classification?)
}
#[tokio::test]
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 81135dab2..c99c1802f 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -33,6 +33,33 @@ use crate::{observability, DriverError, LibsyError, Result};
/// `Arc<dyn Algorithm>` object-safe.
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
+/// One completed model call observed at the algorithm offload boundary.
+#[derive(Clone, Debug)]
+pub struct LlmCallObservation {
+ pub selected_model: String,
+ /// Routing tier attached to the selected model, when present.
+ pub tier: Option<String>,
+ /// Whether this was the routed backend call rather than classifier or judge overhead.
+ pub is_routed: bool,
+ pub is_success: bool,
+ /// Time spent waiting for the model call to resolve.
+ pub duration: Duration,
+ /// Normalized usage for a buffered successful response.
+ pub usage: Option<Usage>,
+}
+
+/// One request-scoped observation emitted by the algorithm runner.
+#[derive(Clone, Debug)]
+pub enum RunObservation {
+ /// A completed model call.
+ LlmCall(LlmCallObservation),
+ /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
+ RoutingOverhead(Duration),
+}
+
+/// Request-scoped callback for algorithm-run observations.
+pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;
+
/// A request paired with the routing [`Decision`] that produced it — the offload
/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
///
@@ -118,15 +145,21 @@ pub struct Driver {
driver: TypeErasedDriver,
// How long the call that served this run took. We need this to calculate routing overhead.
routed_call: Arc<Mutex<Option<Duration>>>,
+ observer: Option<RunObserver>,
}
impl Driver {
/// Build an empty driver with its step channel ready. Created per call by
/// [`run_stream`](Algorithm::run_stream).
pub(crate) fn new() -> Self {
+ Self::with_observer(None)
+ }
+
+ fn with_observer(observer: Option<RunObserver>) -> Self {
Self {
driver: TypeErasedDriver::new(),
routed_call: Arc::new(Mutex::new(None)),
+ observer,
}
}
@@ -135,6 +168,14 @@ impl Driver {
*self.routed_call.lock()
}
+ /// Records routing overhead (how long Switchyard added to the call) once
+ /// per successful run. Called by `observe_run`.
+ pub(crate) fn observe_routing_overhead(&self, duration: Duration) {
+ if let Some(observer) = &self.observer {
+ observer(RunObservation::RoutingOverhead(duration));
+ }
+ }
+
/// Offload a model call: publish `routed` as a [`Step::CallLlm`] and await the
/// consumer's [`Response`]. The call's context travels inside
/// [`routed.ctx`](RoutedRequest::ctx). Errors if the stream is closed or the call failed.
@@ -178,6 +219,20 @@ impl Driver {
&result,
&tracing::Span::current(),
);
+ if let Some(observer) = &self.observer {
+ observer(RunObservation::LlmCall(LlmCallObservation {
+ selected_model,
+ tier,
+ is_routed,
+ is_success: result.is_ok(),
+ duration: elapsed,
+ usage: result
+ .as_ref()
+ .ok()
+ .and_then(|response| response.llm_response.as_agg())
+ .map(|response| response.usage.clone()),
+ }));
+ }
// Classifier and judge calls are routing overhead.
// And don't record time for failed calls.
if is_routed && result.is_ok() {
@@ -424,7 +479,13 @@ pub trait Algorithm: Send + Sync + 'static {
/// Process a request to completion, returning a stream of [`Step`]s.
/// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve.
/// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.
- fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
+ /// Report each model call to `observer`.
+ fn run_stream(
+ self: Arc<Self>,
+ ctx: Context,
+ request: Request,
+ observer: Option<RunObserver>,
+ ) -> StepStream {
// Stamp the algorithm's telemetry label into the request context; the
// context rides on every driver call, so its telemetry is attributed.
let mut ctx = ctx;
@@ -432,7 +493,7 @@ pub trait Algorithm: Send + Sync + 'static {
observability::ALGORITHM_KEY.to_string(),
self.name().to_string(),
);
- let driver = Driver::new();
+ let driver = Driver::with_observer(observer);
let task_driver = driver.clone();
let task_ctx = ctx.clone();
let stream = task_driver.stream();
@@ -483,6 +544,16 @@ pub trait Algorithm: Send + Sync + 'static {
self: Arc<Self>,
ctx: Context,
request: Request,
+ ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+ self.run_observed(ctx, request, None).await
+ }
+
+ /// Process a request to completion while reporting each model call to `observer`.
+ async fn run_observed(
+ self: Arc<Self>,
+ ctx: Context,
+ request: Request,
+ observer: Option<RunObserver>,
) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
// Serve one offloaded call with its target's default client. A failed *model*
// call is forwarded to the algorithm via `respond`; this errors only on an
@@ -518,7 +589,7 @@ pub trait Algorithm: Send + Sync + 'static {
call.respond(result)
}
- let stream = self.run_stream(ctx, request);
+ let stream = self.run_stream(ctx, request, observer);
tokio::pin!(stream);
let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
@@ -663,6 +734,34 @@ mod tests {
LlmTargetSet::new(targets)
}
+ #[tokio::test]
+ async fn observed_run_reports_one_successful_routed_call() -> Result<()> {
+ let observations = Arc::new(Mutex::new(Vec::new()));
+ let observed = observations.clone();
+ let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
+ let (_, response) = orch(target_set(&[("direct/model", true)]))
+ .run_observed(Context::default(), request(), Some(observer))
+ .await?;
+ assert_eq!(
+ response.llm_response.as_agg().map(completion_text),
+ Some("direct/model".to_string())
+ );
+ let observations = observations.lock();
+ assert_eq!(observations.len(), 2);
+ let RunObservation::LlmCall(observation) = &observations[0] else {
+ return Err(test_error("expected an LLM call observation"));
+ };
+ assert_eq!(observation.selected_model, "direct/model");
+ assert!(observation.is_routed);
+ assert!(observation.is_success);
+ assert!(observation.usage.is_some());
+ assert!(matches!(
+ observations[1],
+ RunObservation::RoutingOverhead(_)
+ ));
+ Ok(())
+ }
+
#[test]
fn target_lookup_returns_the_missing_target() {
let error = target_set(&[]).get_target("missing").err();
@@ -764,8 +863,11 @@ mod tests {
async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
// A client-less target -> its call is offloaded via a promise the
// orchestrator surfaces as a `CallLlm` step for us to fulfill.
- let stream =
- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
+ let stream = orch(target_set(&[("offload/model", false)])).run_stream(
+ Context::default(),
+ request(),
+ None,
+ );
tokio::pin!(stream);
let mut saw_call = false;
@@ -812,8 +914,11 @@ mod tests {
async fn client_backed_target_offloads_with_a_default_client() -> Result<()> {
// Every call now offloads to the stream; a client-backed target rides its
// client along as `default_client` so the consumer can serve it by default.
- let stream =
- orch(target_set(&[("direct/model", true)])).run_stream(Context::default(), request());
+ let stream = orch(target_set(&[("direct/model", true)])).run_stream(
+ Context::default(),
+ request(),
+ None,
+ );
tokio::pin!(stream);
let mut final_completion = None;
@@ -966,8 +1071,11 @@ mod tests {
// A client-less target offloads its call; we fulfill the promise with an
// Err, which must flow back through `call_llm_target` into the algorithm and
// out as an error step — not a response.
- let stream =
- orch(target_set(&[("offload/model", false)])).run_stream(Context::default(), request());
+ let stream = orch(target_set(&[("offload/model", false)])).run_stream(
+ Context::default(),
+ request(),
+ None,
+ );
tokio::pin!(stream);
let mut saw_error = false;
@@ -1041,7 +1149,7 @@ mod tests {
dropped: dropped.clone(),
});
- let stream = algo.run_stream(Context::default(), request());
+ let stream = algo.run_stream(Context::default(), request(), None);
started_rx
.recv()
.await
@@ -1079,7 +1187,7 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- let stream = algo.run_stream(Context::default(), request());
+ let stream = algo.run_stream(Context::default(), request(), None);
tokio::pin!(stream);
let mut saw_error = false;
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 8c5de7c12..fd5ef5839 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -147,7 +147,11 @@ pub(crate) async fn observe_run(
let algorithm = algorithm_label(&ctx);
record_run(algorithm, duration, &result, &Span::current());
if result.is_ok() {
- record_routing_overhead(algorithm, duration, driver.routed_call_duration());
+ if let Some(overhead) =
+ record_routing_overhead(algorithm, duration, driver.routed_call_duration())
+ {
+ driver.observe_routing_overhead(overhead);
+ }
}
result
}
@@ -197,20 +201,23 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
/// Records what routing cost on top of the call that served the run: classifier
/// calls, target resolution, decision publishing. A run with no routed call has
/// nothing to subtract, so it records nothing.
-fn record_routing_overhead(algorithm: &str, run: Duration, routed_call: Option<Duration>) {
- let Some(routed_call) = routed_call else {
- return;
- };
+fn record_routing_overhead(
+ algorithm: &str,
+ run: Duration,
+ routed_call: Option<Duration>,
+) -> Option<Duration> {
+ let routed_call = routed_call?;
// Saturating: the two clocks start a moment apart, so a run that is all
// routed call can come out fractionally negative.
- let overhead_ms = run.saturating_sub(routed_call).as_secs_f64() * 1000.0;
+ let overhead = run.saturating_sub(routed_call);
meter()
.f64_histogram("switchyard.routing_overhead_ms")
.build()
.record(
- overhead_ms,
+ overhead.as_secs_f64() * 1000.0,
&[KeyValue::new("algorithm", algorithm.to_string())],
);
+ Some(overhead)
}
/// Records the resolution of one offloaded model call: the call counter and
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 50d738978..896929a48 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -620,6 +620,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
let stream = algo(ALGO, MODEL, None).run_stream(
Context::default(),
request_with_metadata("obs-session-2", "obs-corr-2"),
+ None,
);
tokio::pin!(stream);
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 753333d32..424d73a5f 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -23,6 +23,7 @@ libsy = { package = "switchyard-libsy", path = "../libsy" }
opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
opentelemetry-prometheus = "0.32"
opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }
+parking_lot.workspace = true
prometheus = "0.14"
serde.workspace = true
toml = "1.1"
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index edc6107bb..db84554db 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -7,6 +7,7 @@ pub mod config;
mod metrics;
mod response;
mod sse;
+mod stats;
mod usage_metrics;
use std::collections::BTreeMap;
@@ -25,7 +26,10 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};
+use libsy::{
+ Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request, RunObservation,
+ RunObserver,
+};
use serde_json::{json, Value};
use tokio::net::{TcpListener, TcpSocket};
use tracing::Level;
@@ -33,6 +37,7 @@ use tracing::Level;
use switchyard_translation::{decode_request, WireFormat};
use crate::response::into_http_response;
+use crate::stats::{prefix_probe, tracking_enabled_from_env, StatsAccumulator, StatsSnapshot};
/// Default TCP listen backlog used by the Rust server.
pub const DEFAULT_LISTEN_BACKLOG: u32 = 65_535;
@@ -75,6 +80,8 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;
pub struct ServerState {
routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
metrics: prometheus::Registry,
+ stats: StatsAccumulator,
+ track_cache_eligibility: bool,
}
impl ServerState {
@@ -99,6 +106,8 @@ impl ServerState {
Ok(Self {
routes: Arc::new(entries),
metrics,
+ stats: StatsAccumulator::default(),
+ track_cache_eligibility: tracking_enabled_from_env(),
})
}
@@ -198,6 +207,34 @@ async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {
#[derive(Clone, Copy)]
struct RequestStart(Instant);
+/// Maps routed call observations to backend stats, non-routed calls to classifier/judge
+/// stats, and records routing overhead once the algorithm run completes.
+fn stats_observer(stats: StatsAccumulator) -> RunObserver {
+ Arc::new(move |observation| match observation {
+ RunObservation::LlmCall(call) => {
+ let latency_ms = call.duration.as_secs_f64() * 1_000.0;
+ if call.is_routed {
+ if call.is_success {
+ stats.record_success(&call.selected_model, latency_ms, call.tier.as_deref());
+ } else {
+ stats.record_error(&call.selected_model, call.tier.as_deref());
+ }
+ } else if call.is_success {
+ stats.record_classifier_success(
+ call.selected_model,
+ call.usage.as_ref().map(usage_metrics::token_usage),
+ latency_ms,
+ );
+ } else {
+ stats.record_classifier_error(call.selected_model);
+ }
+ }
+ RunObservation::RoutingOverhead(duration) => {
+ stats.record_routing_overhead(duration.as_secs_f64() * 1_000.0);
+ }
+ })
+}
+
/// Stamps the ingress instant into request extensions. Runs as a router layer,
/// so it executes before the handlers' `Json` extractor buffers the body —
/// request-latency measurements therefore include body read and decode.
@@ -216,6 +253,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/v1/responses", post(openai_responses))
.route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/models", get(models))
+ .route("/v1/stats", get(get_stats))
.route("/metrics", get(prometheus_metrics))
.route("/health", get(health))
.fallback(not_found)
@@ -421,24 +459,40 @@ async fn handle_llm_request(
body: Value,
wire_format: WireFormat,
) -> Response {
+ let cache_probe = state.track_cache_eligibility.then(|| prefix_probe(&body));
let (algorithm, request) = match resolve_route(&state, metadata, body, wire_format) {
Ok(resolved) => resolved,
Err(response) => return response,
};
- let (trace, response) = match algorithm.run(Context::default(), request).await {
+ let observer = stats_observer(state.stats.clone());
+ let (trace, response) = match algorithm
+ .run_observed(Context::default(), request, Some(observer))
+ .await
+ {
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
+
// Metrics, response body, and routing header all read the same decision, so
// the model they name can never disagree. An empty trace leaves the body with
// the id the upstream reported.
let decision = trace.last();
let response = if let Some(decision) = decision {
+ let cache_eligible = cache_probe
+ .as_ref()
+ .map(|probe| {
+ state
+ .stats
+ .prefix_eligibility(decision.selected_model(), probe)
+ })
+ .unwrap_or(0.0);
usage_metrics::observe(
response,
decision.selected_model(),
decision.routing_tier(),
started.0,
+ state.stats,
+ cache_eligible,
)
} else {
response
@@ -662,6 +716,10 @@ async fn models(State(state): State<ServerState>) -> Json<Value> {
Json(model_list_payload(state.models()))
}
+async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
+ Json(state.stats.snapshot())
+}
+
async fn health() -> Json<Value> {
Json(json!({"status": "ok"}))
}
diff --git a/crates/switchyard-server/src/stats.rs b/crates/switchyard-server/src/stats.rs
new file mode 100644
index 000000000..ebb644578
--- /dev/null
+++ b/crates/switchyard-server/src/stats.rs
@@ -0,0 +1,10 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Process-local JSON stats accounting for the Rust server.
+
+mod accumulator;
+mod cache_eligibility;
+
+pub(crate) use accumulator::{StatsAccumulator, StatsSnapshot, TokenUsage};
+pub(crate) use cache_eligibility::{prefix_probe, tracking_enabled_from_env};
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
new file mode 100644
index 000000000..2aa924ec7
--- /dev/null
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -0,0 +1,607 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Thread-safe stats accumulator and serializable snapshot schema.
+
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::sync::Arc;
+
+use parking_lot::{Mutex, MutexGuard};
+use serde::Serialize;
+
+use super::cache_eligibility::PrefixProbe;
+
+const MAX_LATENCY_SAMPLES: usize = 10_000;
+
+/// Normalized token counters recorded by the server.
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub(crate) struct TokenUsage {
+ pub prompt_tokens: u64,
+ pub completion_tokens: u64,
+ pub cached_tokens: u64,
+ pub cache_creation_tokens: u64,
+ pub cacheable_prompt_tokens: u64,
+ pub reasoning_tokens: u64,
+}
+
+/// Thread-safe process-local stats store.
+#[derive(Clone, Debug, Default)]
+pub(crate) struct StatsAccumulator {
+ inner: Arc<Mutex<StatsAccumulatorInner>>,
+}
+
+impl StatsAccumulator {
+ /// Records one successful routed backend call.
+ pub(crate) fn record_success(
+ &self,
+ model: impl Into<String>,
+ backend_latency_ms: f64,
+ tier: Option<&str>,
+ ) {
+ let mut inner = self.lock();
+ inner.total_requests = inner.total_requests.saturating_add(1);
+ let model = model.into();
+ let tier = normalized_tier(tier);
+ let stats = inner.model_stats_mut(model.clone());
+ stats.calls = stats.calls.saturating_add(1);
+ stats.model_call_latency.record(backend_latency_ms);
+ if let Some(tier) = tier {
+ stats.tiers.insert(tier.to_string());
+ let tier_stats = inner.tier_stats_mut(tier, &model);
+ tier_stats.calls = tier_stats.calls.saturating_add(1);
+ }
+ }
+
+ /// Records one failed routed backend call.
+ pub(crate) fn record_error(&self, model: impl Into<String>, tier: Option<&str>) {
+ let mut inner = self.lock();
+ inner.total_requests = inner.total_requests.saturating_add(1);
+ inner.total_errors = inner.total_errors.saturating_add(1);
+ let model = model.into();
+ let stats = inner.model_stats_mut(model.clone());
+ stats.errors = stats.errors.saturating_add(1);
+ if let Some(tier) = normalized_tier(tier) {
+ stats.tiers.insert(tier.to_string());
+ inner.tier_stats_mut(tier, &model);
+ }
+ }
+
+ /// Records usage and terminal latency after a successful routed call.
+ pub(crate) fn record_usage(
+ &self,
+ model: impl Into<String>,
+ usage: TokenUsage,
+ total_latency_ms: f64,
+ tier: Option<&str>,
+ ) {
+ let mut inner = self.lock();
+ let model = model.into();
+ let stats = inner.model_stats_mut(model.clone());
+ stats.add_usage(usage);
+ stats.total_latency.record(total_latency_ms);
+ if let Some(tier) = normalized_tier(tier) {
+ stats.tiers.insert(tier.to_string());
+ inner.tier_stats_mut(tier, &model).add_usage(usage);
+ }
+ }
+
+ /// Records routing time for one completed algorithm run.
+ pub(crate) fn record_routing_overhead(&self, routing_overhead_ms: f64) {
+ self.lock().routing_overhead.record(routing_overhead_ms);
+ }
+
+ /// Records one successful classifier or judge call.
+ pub(crate) fn record_classifier_success(
+ &self,
+ model: impl Into<String>,
+ usage: Option<TokenUsage>,
+ latency_ms: f64,
+ ) {
+ let mut inner = self.lock();
+ inner.classifier_requests = inner.classifier_requests.saturating_add(1);
+ let stats = inner.classifier_stats_mut(model.into());
+ stats.calls = stats.calls.saturating_add(1);
+ if let Some(usage) = usage {
+ stats.add_usage(usage);
+ }
+ stats.model_call_latency.record(latency_ms);
+ stats.total_latency.record(latency_ms);
+ }
+
+ /// Records one failed classifier or judge call.
+ pub(crate) fn record_classifier_error(&self, model: impl Into<String>) {
+ let mut inner = self.lock();
+ inner.classifier_requests = inner.classifier_requests.saturating_add(1);
+ inner.classifier_errors = inner.classifier_errors.saturating_add(1);
+ let stats = inner.classifier_stats_mut(model.into());
+ stats.errors = stats.errors.saturating_add(1);
+ }
+
+ /// Returns the cache-eligible fraction for `model` and records the prefix as seen.
+ pub(crate) fn prefix_eligibility(&self, model: &str, probe: &PrefixProbe) -> f64 {
+ let mut inner = self.lock();
+ let stats = inner.model_stats_mut(model.to_string());
+ let fraction = probe.eligible_fraction(&stats.seen_prefixes);
+ if let Some(hash) = probe.full_hash() {
+ stats.seen_prefixes.insert(hash);
+ }
+ fraction
+ }
+
+ /// Returns a serializable point-in-time snapshot.
+ pub(crate) fn snapshot(&self) -> StatsSnapshot {
+ let inner = self.lock().clone();
+ inner.snapshot()
+ }
+
+ fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {
+ self.inner.lock()
+ }
+}
+
+fn normalized_tier(tier: Option<&str>) -> Option<&str> {
+ tier.map(str::trim).filter(|tier| !tier.is_empty())
+}
+
+#[derive(Clone, Debug, Default)]
+struct StatsAccumulatorInner {
+ by_model: BTreeMap<String, ModelStats>,
+ by_tier: BTreeMap<String, TierStats>,
+ total_requests: u64,
+ total_errors: u64,
+ routing_overhead: LatencyHistogram,
+ by_classifier: BTreeMap<String, ModelStats>,
+ classifier_requests: u64,
+ classifier_errors: u64,
+}
+
+impl StatsAccumulatorInner {
+ fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {
+ self.by_model.entry(model).or_default()
+ }
+
+ fn classifier_stats_mut(&mut self, model: String) -> &mut ModelStats {
+ self.by_classifier.entry(model).or_default()
+ }
+
+ fn tier_stats_mut(&mut self, tier: &str, model: &str) -> &mut TierStats {
+ let stats = self.by_tier.entry(tier.to_string()).or_default();
+ stats.models.insert(model.to_string());
+ stats
+ }
+
+ fn snapshot(&self) -> StatsSnapshot {
+ let (models, total_tokens) = build_model_snapshots(&self.by_model, self.total_requests);
+ let classifier = build_classifier_snapshot(
+ &self.by_classifier,
+ self.classifier_requests,
+ self.classifier_errors,
+ );
+ StatsSnapshot {
+ total_requests: self.total_requests,
+ total_errors: self.total_errors,
+ total_tokens,
+ models,
+ tiers: tier_snapshots(&self.by_tier, total_tokens.total, self.total_requests),
+ routing_overhead: self.routing_overhead.snapshot(),
+ classifier,
+ }
+ }
+}
+
+#[derive(Clone, Debug, Default)]
+struct ModelStats {
+ calls: u64,
+ errors: u64,
+ prompt_tokens: u64,
+ max_observed_context_tokens: u64,
+ completion_tokens: u64,
+ cached_tokens: u64,
+ cache_creation_tokens: u64,
+ cacheable_prompt_tokens: u64,
+ reasoning_tokens: u64,
+ seen_prefixes: HashSet<u64>,
+ model_call_latency: LatencyHistogram,
+ total_latency: LatencyHistogram,
+ tiers: BTreeSet<String>,
+}
+
+impl ModelStats {
+ fn add_usage(&mut self, usage: TokenUsage) {
+ self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);
+ self.completion_tokens = self
+ .completion_tokens
+ .saturating_add(usage.completion_tokens);
+ self.cached_tokens = self.cached_tokens.saturating_add(usage.cached_tokens);
+ self.cache_creation_tokens = self
+ .cache_creation_tokens
+ .saturating_add(usage.cache_creation_tokens);
+ self.cacheable_prompt_tokens = self
+ .cacheable_prompt_tokens
+ .saturating_add(usage.cacheable_prompt_tokens);
+ self.reasoning_tokens = self.reasoning_tokens.saturating_add(usage.reasoning_tokens);
+ self.max_observed_context_tokens = self
+ .max_observed_context_tokens
+ .max(usage.prompt_tokens.saturating_add(usage.completion_tokens));
+ }
+}
+
+#[derive(Clone, Debug, Default)]
+struct TierStats {
+ models: BTreeSet<String>,
+ calls: u64,
+ prompt_tokens: u64,
+ completion_tokens: u64,
+}
+
+impl TierStats {
+ fn add_usage(&mut self, usage: TokenUsage) {
+ self.prompt_tokens = self.prompt_tokens.saturating_add(usage.prompt_tokens);
+ self.completion_tokens = self
+ .completion_tokens
+ .saturating_add(usage.completion_tokens);
+ }
+}
+
+#[derive(Clone, Debug)]
+struct LatencyHistogram {
+ count: u64,
+ total_ms: f64,
+ min_ms: f64,
+ max_ms: f64,
+ samples: Vec<f64>,
+}
+
+impl Default for LatencyHistogram {
+ fn default() -> Self {
+ Self {
+ count: 0,
+ total_ms: 0.0,
+ min_ms: f64::INFINITY,
+ max_ms: 0.0,
+ samples: Vec::new(),
+ }
+ }
+}
+
+impl LatencyHistogram {
+ fn record(&mut self, latency_ms: f64) {
+ if !latency_ms.is_finite() {
+ tracing::debug!(latency_ms, "dropping non-finite latency sample");
+ return;
+ }
+ let latency_ms = latency_ms.max(0.0);
+ self.count = self.count.saturating_add(1);
+ self.total_ms += latency_ms;
+ self.min_ms = self.min_ms.min(latency_ms);
+ self.max_ms = self.max_ms.max(latency_ms);
+ if self.samples.len() < MAX_LATENCY_SAMPLES {
+ self.samples.push(latency_ms);
+ } else {
+ let index = self.count.saturating_sub(1) as usize % MAX_LATENCY_SAMPLES;
+ self.samples[index] = latency_ms;
+ }
+ }
+
+ fn snapshot(&self) -> LatencyHistogramSnapshot {
+ if self.count == 0 {
+ return LatencyHistogramSnapshot::default();
+ }
+ let mut samples = self.samples.to_vec();
+ samples.sort_by(f64::total_cmp);
+ let count = samples.len();
+ let p99_index = count.saturating_sub(1).min((count as f64 * 0.99) as usize);
+ LatencyHistogramSnapshot {
+ count: self.count,
+ total_ms: round2(self.total_ms),
+ min_ms: round2(self.min_ms),
+ max_ms: round2(self.max_ms),
+ avg_ms: round2(self.total_ms / self.count as f64),
+ p50_ms: samples.get(count / 2).copied().map(round2).unwrap_or(0.0),
+ p99_ms: samples.get(p99_index).copied().map(round2).unwrap_or(0.0),
+ }
+ }
+}
+
+/// Full JSON stats snapshot.
+#[derive(Clone, Debug, Default, PartialEq, Serialize)]
+pub(crate) struct StatsSnapshot {
+ pub total_requests: u64,
+ pub total_errors: u64,
+ pub total_tokens: TokenTotals,
+ pub models: BTreeMap<String, ModelStatsSnapshot>,
+ pub tiers: BTreeMap<String, TierStatsSnapshot>,
+ pub routing_overhead: LatencyHistogramSnapshot,
+ pub classifier: ClassifierStatsSnapshot,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Serialize)]
+pub(crate) struct ClassifierStatsSnapshot {
+ pub total_requests: u64,
+ pub total_errors: u64,
+ pub total_tokens: TokenTotals,
+ pub models: BTreeMap<String, ModelStatsSnapshot>,
+}
+
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)]
+pub(crate) struct TokenTotals {
+ pub prompt: u64,
+ pub completion: u64,
+ pub cached: u64,
+ pub cache_creation: u64,
+ pub reasoning: u64,
+ pub total: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub(crate) struct ModelStatsSnapshot {
+ pub calls: u64,
+ pub errors: u64,
+ pub request_pct: f64,
+ pub prompt_tokens: u64,
+ pub max_observed_context_tokens: u64,
+ pub completion_tokens: u64,
+ pub total_tokens: u64,
+ pub token_pct: f64,
+ pub cached_tokens: u64,
+ pub cache_creation_tokens: u64,
+ pub reasoning_tokens: u64,
+ pub avg_prompt_tokens: f64,
+ pub avg_completion_tokens: f64,
+ pub cache_hit_rate: f64,
+ pub theoretical_cache_hit_rate: f64,
+ pub model_call_latency: LatencyHistogramSnapshot,
+ pub total_latency: LatencyHistogramSnapshot,
+ pub tiers: BTreeSet<String>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize)]
+pub(crate) struct TierStatsSnapshot {
+ pub models: BTreeSet<String>,
+ pub calls: u64,
+ pub request_pct: f64,
+ pub prompt_tokens: u64,
+ pub completion_tokens: u64,
+ pub total_tokens: u64,
+ pub token_pct: f64,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize)]
+pub(crate) struct LatencyHistogramSnapshot {
+ pub count: u64,
+ pub total_ms: f64,
+ pub min_ms: f64,
+ pub max_ms: f64,
+ pub avg_ms: f64,
+ pub p50_ms: f64,
+ pub p99_ms: f64,
+}
+
+fn build_model_snapshots(
+ by_model: &BTreeMap<String, ModelStats>,
+ total_requests: u64,
+) -> (BTreeMap<String, ModelStatsSnapshot>, TokenTotals) {
+ let mut totals = TokenTotals::default();
+ for stats in by_model.values() {
+ totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens);
+ totals.completion = totals.completion.saturating_add(stats.completion_tokens);
+ totals.cached = totals.cached.saturating_add(stats.cached_tokens);
+ totals.cache_creation = totals
+ .cache_creation
+ .saturating_add(stats.cache_creation_tokens);
+ totals.reasoning = totals.reasoning.saturating_add(stats.reasoning_tokens);
+ }
+ totals.total = totals.prompt.saturating_add(totals.completion);
+
+ let models = by_model
+ .iter()
+ .map(|(model, stats)| {
+ let total_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);
+ let snapshot = ModelStatsSnapshot {
+ calls: stats.calls,
+ errors: stats.errors,
+ request_pct: percentage(stats.calls, total_requests),
+ prompt_tokens: stats.prompt_tokens,
+ max_observed_context_tokens: stats.max_observed_context_tokens,
+ completion_tokens: stats.completion_tokens,
+ total_tokens,
+ token_pct: percentage(total_tokens, totals.total),
+ cached_tokens: stats.cached_tokens,
+ cache_creation_tokens: stats.cache_creation_tokens,
+ reasoning_tokens: stats.reasoning_tokens,
+ avg_prompt_tokens: average(stats.prompt_tokens, stats.calls),
+ avg_completion_tokens: average(stats.completion_tokens, stats.calls),
+ cache_hit_rate: ratio4(stats.cached_tokens, stats.prompt_tokens),
+ theoretical_cache_hit_rate: ratio4(
+ stats.cacheable_prompt_tokens,
+ stats.prompt_tokens,
+ ),
+ model_call_latency: stats.model_call_latency.snapshot(),
+ total_latency: stats.total_latency.snapshot(),
+ tiers: stats.tiers.clone(),
+ };
+ (model.clone(), snapshot)
+ })
+ .collect();
+ (models, totals)
+}
+
+fn build_classifier_snapshot(
+ models: &BTreeMap<String, ModelStats>,
+ total_requests: u64,
+ total_errors: u64,
+) -> ClassifierStatsSnapshot {
+ let (models, total_tokens) = build_model_snapshots(models, total_requests);
+ ClassifierStatsSnapshot {
+ total_requests,
+ total_errors,
+ total_tokens,
+ models,
+ }
+}
+
+fn tier_snapshots(
+ tiers: &BTreeMap<String, TierStats>,
+ total_tokens: u64,
+ total_requests: u64,
+) -> BTreeMap<String, TierStatsSnapshot> {
+ tiers
+ .iter()
+ .map(|(tier, stats)| {
+ let tier_tokens = stats.prompt_tokens.saturating_add(stats.completion_tokens);
+ (
+ tier.clone(),
+ TierStatsSnapshot {
+ models: stats.models.clone(),
+ calls: stats.calls,
+ request_pct: percentage(stats.calls, total_requests),
+ prompt_tokens: stats.prompt_tokens,
+ completion_tokens: stats.completion_tokens,
+ total_tokens: tier_tokens,
+ token_pct: percentage(tier_tokens, total_tokens),
+ },
+ )
+ })
+ .collect()
+}
+
+fn percentage(numerator: u64, denominator: u64) -> f64 {
+ if denominator == 0 {
+ 0.0
+ } else {
+ round2(numerator as f64 / denominator as f64 * 100.0)
+ }
+}
+
+fn average(total: u64, count: u64) -> f64 {
+ if count == 0 {
+ 0.0
+ } else {
+ round2(total as f64 / count as f64)
+ }
+}
+
+fn ratio4(numerator: u64, denominator: u64) -> f64 {
+ if denominator == 0 {
+ 0.0
+ } else {
+ round4(numerator as f64 / denominator as f64)
+ }
+}
+
+fn round2(value: f64) -> f64 {
+ (value * 100.0).round() / 100.0
+}
+
+fn round4(value: f64) -> f64 {
+ (value * 10_000.0).round() / 10_000.0
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+
+ use super::*;
+ use crate::stats::cache_eligibility::prefix_probe;
+
+ fn usage(prompt: u64, completion: u64) -> TokenUsage {
+ TokenUsage {
+ prompt_tokens: prompt,
+ completion_tokens: completion,
+ ..TokenUsage::default()
+ }
+ }
+
+ #[test]
+ fn snapshot_aggregates_backend_and_classifier_stats() {
+ let stats = StatsAccumulator::default();
+ stats.record_success("model/strong", 10.0, Some("strong"));
+ stats.record_usage(
+ "model/strong",
+ TokenUsage {
+ cached_tokens: 4,
+ cache_creation_tokens: 2,
+ reasoning_tokens: 3,
+ ..usage(10, 5)
+ },
+ 15.0,
+ Some("strong"),
+ );
+ stats.record_routing_overhead(5.0);
+ stats.record_success("model/weak", 20.0, Some("weak"));
+ stats.record_usage("model/weak", usage(20, 10), 30.0, Some("weak"));
+ stats.record_routing_overhead(10.0);
+ stats.record_classifier_success("gemini-3.5-flash", Some(usage(1_000_000, 0)), 8.0);
+ let snapshot = stats.snapshot();
+ assert_eq!(snapshot.total_tokens.prompt, 30);
+ assert_eq!(snapshot.total_tokens.cached, 4);
+ assert_eq!(
+ snapshot.models["model/strong"].max_observed_context_tokens,
+ 15
+ );
+ assert_eq!(
+ snapshot.models["model/strong"].theoretical_cache_hit_rate,
+ 0.0
+ );
+ assert_eq!(snapshot.tiers["strong"].request_pct, 50.0);
+ assert_eq!(snapshot.routing_overhead.p50_ms, 10.0);
+ assert_eq!(snapshot.routing_overhead.p99_ms, 10.0);
+ assert_eq!(snapshot.classifier.total_requests, 1);
+ assert_eq!(snapshot.classifier.total_tokens.prompt, 1_000_000);
+
+ stats.record_success("model/strong", 1.0, Some("weak"));
+ stats.record_success("model/other", 1.0, Some("strong"));
+ let snapshot = stats.snapshot();
+ assert_eq!(
+ snapshot.models["model/strong"].tiers,
+ BTreeSet::from(["strong".to_string(), "weak".to_string()])
+ );
+ assert_eq!(
+ snapshot.tiers["strong"].models,
+ BTreeSet::from(["model/other".to_string(), "model/strong".to_string()])
+ );
+ }
+
+ #[test]
+ fn theoretical_cache_hit_rate_is_switch_aware() {
+ let stats = StatsAccumulator::default();
+ let first = prefix_probe(&json!({
+ "messages": [{"role": "user", "content": "aaaa"}],
+ }));
+ let first_eligible = stats.prefix_eligibility("model/a", &first);
+ stats.record_usage(
+ "model/a",
+ TokenUsage {
+ cacheable_prompt_tokens: (100.0 * first_eligible).round() as u64,
+ ..usage(100, 4)
+ },
+ 1.0,
+ None,
+ );
+
+ let second = prefix_probe(&json!({
+ "messages": [
+ {"role": "user", "content": "aaaa"},
+ {"role": "user", "content": "bbbb"},
+ ],
+ }));
+ let second_eligible = stats.prefix_eligibility("model/a", &second);
+ stats.record_usage(
+ "model/a",
+ TokenUsage {
+ cacheable_prompt_tokens: (100.0 * second_eligible).round() as u64,
+ ..usage(100, 4)
+ },
+ 1.0,
+ None,
+ );
+
+ assert_eq!(first_eligible, 0.0);
+ assert_eq!(second_eligible, 0.5);
+ assert_eq!(
+ stats.snapshot().models["model/a"].theoretical_cache_hit_rate,
+ 0.25
+ );
+ assert_eq!(stats.prefix_eligibility("model/b", &second), 0.0);
+ }
+}
diff --git a/crates/switchyard-server/src/stats/cache_eligibility.rs b/crates/switchyard-server/src/stats/cache_eligibility.rs
new file mode 100644
index 000000000..030bdd71e
--- /dev/null
+++ b/crates/switchyard-server/src/stats/cache_eligibility.rs
@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Switch-aware cache eligibility for the Rust server.
+
+use std::collections::hash_map::DefaultHasher;
+use std::collections::HashSet;
+use std::hash::{Hash, Hasher};
+
+use serde_json::Value;
+
+/// Set this environment variable (to =1) to enable tracking cache prefixes to estimate
+/// kv-cache hit as stat `theoretical_cache_hit_rate`.
+/// Memory will grow forever so this is only for evals.
+const TRACK_ENV: &str = "SWITCHYARD_THEORETICAL_CACHE";
+
+/// Whether theoretical cache-hit tracking is enabled via the environment.
+pub(crate) fn tracking_enabled_from_env() -> bool {
+ env_opts_in(std::env::var(TRACK_ENV).ok().as_deref())
+}
+
+fn env_opts_in(value: Option<&str>) -> bool {
+ matches!(
+ value
+ .map(|value| value.trim().to_ascii_lowercase())
+ .as_deref(),
+ Some("1" | "true" | "yes" | "on")
+ )
+}
+
+/// Cumulative prefix fingerprints of a request, in message order.
+#[derive(Clone, Debug, Default)]
+pub(crate) struct PrefixProbe {
+ boundaries: Vec<(u64, u64)>,
+ total_len: u64,
+}
+
+impl PrefixProbe {
+ /// Returns the fraction belonging to the longest prefix previously seen by a model.
+ pub(crate) fn eligible_fraction(&self, seen: &HashSet<u64>) -> f64 {
+ if self.total_len == 0 {
+ return 0.0;
+ }
+ let eligible = self
+ .boundaries
+ .iter()
+ .filter(|(_, hash)| seen.contains(hash))
+ .map(|(len, _)| *len)
+ .max()
+ .unwrap_or(0);
+ eligible as f64 / self.total_len as f64
+ }
+
+ /// Returns the full prompt fingerprint to retain after routing it to a model.
+ pub(crate) fn full_hash(&self) -> Option<u64> {
+ self.boundaries.last().map(|(_, hash)| *hash)
+ }
+}
+
+/// Builds format-agnostic prefix fingerprints from a request body.
+pub(crate) fn prefix_probe(body: &Value) -> PrefixProbe {
+ let mut boundaries = Vec::new();
+ let mut total_len = 0u64;
+ let mut hasher = DefaultHasher::new();
+
+ let system = body.get("system");
+ let instructions = body.get("instructions");
+ let prefix_len = system.map(text_len).unwrap_or(0) + instructions.map(text_len).unwrap_or(0);
+ if prefix_len > 0 {
+ total_len += prefix_len;
+ for value in [system, instructions].into_iter().flatten() {
+ hash_text_into(value, &mut hasher);
+ }
+ boundaries.push((total_len, hasher.finish()));
+ }
+
+ let turns = body
+ .get("messages")
+ .or_else(|| body.get("input"))
+ .and_then(Value::as_array);
+ if let Some(turns) = turns {
+ for turn in turns {
+ total_len += text_len(turn);
+ hash_text_into(turn, &mut hasher);
+ boundaries.push((total_len, hasher.finish()));
+ }
+ }
+ PrefixProbe {
+ boundaries,
+ total_len,
+ }
+}
+
+fn text_len(value: &Value) -> u64 {
+ match value {
+ Value::String(value) => value.len() as u64,
+ Value::Array(items) => items.iter().map(text_len).sum(),
+ Value::Object(map) => map.values().map(text_len).sum(),
+ _ => 0,
+ }
+}
+
+fn hash_text_into(value: &Value, hasher: &mut DefaultHasher) {
+ match value {
+ Value::String(value) => value.hash(hasher),
+ Value::Number(value) => value.to_string().hash(hasher),
+ Value::Bool(value) => value.hash(hasher),
+ Value::Array(items) => items.iter().for_each(|item| hash_text_into(item, hasher)),
+ Value::Object(map) => map.iter().for_each(|(key, value)| {
+ key.hash(hasher);
+ hash_text_into(value, hasher);
+ }),
+ Value::Null => {}
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn environment_opt_in_values_are_explicit() {
+ assert!(env_opts_in(Some(" TRUE ")));
+ assert!(env_opts_in(Some("on")));
+ assert!(!env_opts_in(Some("false")));
+ assert!(!env_opts_in(None));
+ }
+
+ #[test]
+ fn object_field_names_affect_the_fingerprint() {
+ let left = prefix_probe(&serde_json::json!({"messages": [{"left": "same"}]}));
+ let right = prefix_probe(&serde_json::json!({"messages": [{"right": "same"}]}));
+
+ assert_ne!(left.full_hash(), right.full_hash());
+ }
+}
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index 4e9e6cd9d..341e23e66 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -3,18 +3,22 @@
//! Response usage and full-turn latency metrics for routed requests.
-use std::time::Instant;
+use std::time::{Duration, Instant};
use futures_util::StreamExt;
use libsy::{LlmResponse, LlmResponseChunk, Response, Usage};
use opentelemetry::{global, KeyValue};
+use crate::stats::{StatsAccumulator, TokenUsage};
+
/// Observes a routed response without changing its aggregate or streaming contents.
pub(crate) fn observe(
response: Response,
model: &str,
tier: Option<&str>,
started: Instant,
+ stats: StatsAccumulator,
+ cache_eligible: f64,
) -> Response {
let Response {
llm_response,
@@ -25,8 +29,14 @@ pub(crate) fn observe(
let llm_response = match llm_response {
LlmResponse::Agg(agg) => {
- record_usage(&agg.usage, &model, tier.as_deref());
- record_latency(&model, tier.as_deref(), started);
+ record_terminal(
+ &stats,
+ &agg.usage,
+ &model,
+ tier.as_deref(),
+ started,
+ cache_eligible,
+ );
LlmResponse::Agg(agg)
}
LlmResponse::Stream(mut stream) => {
@@ -49,10 +59,14 @@ pub(crate) fn observe(
return;
}
}
- if let Some(usage) = latest_usage {
- record_usage(&usage, &model, tier.as_deref());
- }
- record_latency(&model, tier.as_deref(), started);
+ record_terminal(
+ &stats,
+ &latest_usage.unwrap_or_default(),
+ &model,
+ tier.as_deref(),
+ started,
+ cache_eligible,
+ );
};
LlmResponse::Stream(Box::pin(wrapped))
}
@@ -64,6 +78,46 @@ pub(crate) fn observe(
}
}
+pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {
+ let cached_tokens = usage.cached_input_tokens().unwrap_or(0);
+ let cache_creation_tokens = usage.cache_creation_input_tokens().unwrap_or(0);
+ TokenUsage {
+ prompt_tokens: usage
+ .input_tokens
+ .unwrap_or(0)
+ .saturating_add(cached_tokens)
+ .saturating_add(cache_creation_tokens),
+ completion_tokens: usage.output_tokens.unwrap_or(0),
+ cached_tokens,
+ cache_creation_tokens,
+ cacheable_prompt_tokens: 0,
+ reasoning_tokens: usage.reasoning_tokens.unwrap_or(0),
+ }
+}
+
+/// Records final usage and latency in both OpenTelemetry metrics and JSON stats.
+fn record_terminal(
+ stats: &StatsAccumulator,
+ usage: &Usage,
+ model: &str,
+ tier: Option<&str>,
+ started: Instant,
+ cache_eligible: f64,
+) {
+ let total_latency = started.elapsed();
+ record_usage(usage, model, tier);
+ record_latency(model, tier, total_latency);
+ let mut token_usage = token_usage(usage);
+ token_usage.cacheable_prompt_tokens =
+ (token_usage.prompt_tokens as f64 * cache_eligible).round() as u64;
+ stats.record_usage(
+ model,
+ token_usage,
+ total_latency.as_secs_f64() * 1_000.0,
+ tier,
+ );
+}
+
fn attributes(model: &str, tier: Option<&str>) -> Vec<KeyValue> {
let mut attributes = vec![KeyValue::new("model", model.to_string())];
if let Some(tier) = tier {
@@ -98,12 +152,9 @@ fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {
}
}
-fn record_latency(model: &str, tier: Option<&str>, started: Instant) {
+fn record_latency(model: &str, tier: Option<&str>, latency: Duration) {
global::meter("switchyard")
.f64_histogram("switchyard.total_latency_ms")
.build()
- .record(
- started.elapsed().as_secs_f64() * 1000.0,
- &attributes(model, tier),
- );
+ .record(latency.as_secs_f64() * 1000.0, &attributes(model, tier));
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index b6ee689c6..0581ee153 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -189,6 +189,113 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
Ok((upstream, app))
}
+fn empty_token_totals() -> Value {
+ json!({
+ "prompt": 0,
+ "completion": 0,
+ "cached": 0,
+ "cache_creation": 0,
+ "reasoning": 0,
+ "total": 0
+ })
+}
+
+#[tokio::test]
+async fn stats_exposes_the_exact_empty_schema_and_no_legacy_alias() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+ let response = send(&app, "GET", "/v1/stats", None).await?;
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response.json()?,
+ json!({
+ "total_requests": 0,
+ "total_errors": 0,
+ "total_tokens": empty_token_totals(),
+ "models": {},
+ "tiers": {},
+ "routing_overhead": {
+ "count": 0,
+ "total_ms": 0.0,
+ "min_ms": 0.0,
+ "max_ms": 0.0,
+ "avg_ms": 0.0,
+ "p50_ms": 0.0,
+ "p99_ms": 0.0
+ },
+ "classifier": {
+ "total_requests": 0,
+ "total_errors": 0,
+ "total_tokens": empty_token_totals(),
+ "models": {},
+ },
+ })
+ );
+ assert_eq!(
+ send(&app, "GET", "/v1/routing/stats", None).await?.status,
+ StatusCode::NOT_FOUND
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn stats_accumulates_buffered_success_error_and_shared_routes() -> TestResult {
+ let (_upstream, app) = test_app(&[
+ ("switchyard/one", &["gemini-3.5-flash"]),
+ ("switchyard/two", &["model/unknown"]),
+ ])
+ .await?;
+ for route in ["switchyard/one", "switchyard/two"] {
+ assert_eq!(
+ send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": route,
+ "messages": [{"role": "user", "content": "hello"}]
+ })),
+ )
+ .await?
+ .status,
+ StatusCode::OK
+ );
+ }
+ assert_eq!(
+ send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/one",
+ "messages": [{"role": "user", "content": "fail"}]
+ })),
+ )
+ .await?
+ .status,
+ StatusCode::IM_A_TEAPOT
+ );
+
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["total_requests"], 3);
+ assert_eq!(stats["total_errors"], 1);
+ assert_eq!(
+ stats["total_tokens"],
+ json!({
+ "prompt": 20,
+ "completion": 4,
+ "cached": 14,
+ "cache_creation": 0,
+ "reasoning": 0,
+ "total": 24
+ })
+ );
+ assert_eq!(stats["models"]["gemini-3.5-flash"]["calls"], 1);
+ assert_eq!(stats["models"]["gemini-3.5-flash"]["errors"], 1);
+ assert_eq!(stats["models"]["model/unknown"]["calls"], 1);
+ assert_eq!(stats["routing_overhead"]["count"], 2);
+ Ok(())
+}
+
#[tokio::test]
async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
const MODEL: &str = "model/metrics-buffered";
@@ -565,6 +672,18 @@ base_threshold = 0.5
assert_eq!(calls[1]["model"], "model/classifier");
assert_eq!(calls[2]["model"], "model/weak");
assert_eq!(calls[3]["model"], "model/weak");
+ drop(calls);
+
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["total_requests"], 3);
+ assert_eq!(stats["models"]["model/weak"]["calls"], 3);
+ assert_eq!(stats["tiers"]["weak"]["calls"], 1);
+ assert_eq!(stats["classifier"]["total_requests"], 1);
+ assert_eq!(
+ stats["classifier"]["models"]["model/classifier"]["calls"],
+ 1
+ );
+ assert_eq!(stats["classifier"]["total_tokens"]["prompt"], 10);
Ok(())
}
@@ -982,6 +1101,17 @@ async fn streaming_success_records_only_final_usage_and_one_latency() -> TestRes
"unexpected delta for {name}"
);
}
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["total_requests"], 1);
+ assert_eq!(
+ stats["total_tokens"],
+ json!({
+ "prompt": 12, "completion": 5, "cached": 7,
+ "cache_creation": 2, "reasoning": 3, "total": 17
+ })
+ );
+ assert_eq!(stats["models"][MODEL]["model_call_latency"]["count"], 1);
+ assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 1);
Ok(())
}
@@ -1026,6 +1156,12 @@ async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {
"{name} changed after a failed stream"
);
}
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["total_requests"], 1);
+ assert_eq!(stats["total_tokens"], empty_token_totals());
+ assert_eq!(stats["models"][MODEL]["calls"], 1);
+ assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 0);
+ assert_eq!(stats["routing_overhead"]["count"], 1);
Ok(())
}