Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(switchyard-server): Metrics for tokens and latency (#179)
Graham King36b96362fd78
diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 4f622649f..5ac6ac835 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -287,11 +287,9 @@ boundary — so algorithms carry no telemetry code beyond `Algorithm::name()`, t- **Structured logs** (`tracing`, target `libsy`): an info event per published`Decision` (selected model + reasoning), warn events for failed calls and runs.- **Metrics** (OpenTelemetry, scope `switchyard`, via the global meter provider): counters- `switchyard.runs`, `switchyard.llm_calls`, `switchyard.decisions`,- `switchyard.input_tokens`, `switchyard.output_tokens`, `switchyard.total_tokens`,- `switchyard.reasoning_tokens`; histograms `switchyard.run_duration_ms` and- `switchyard.llm_call_duration_ms`. Attributes are `algorithm`, `selected_model`, and- `outcome` (`ok`/`error`) — failure rates are the+ `switchyard.runs`, `switchyard.llm_calls`, and `switchyard.decisions`; histograms+ `switchyard.run_duration_ms` and `switchyard.llm_call_duration_ms`. Attributes are+ `algorithm`, `selected_model`, and `outcome` (`ok`/`error`) — failure rates are the`outcome="error"` share of runs/calls.libsy also records compatibility metrics for final routed calls:diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 0c54c3015..9f29db910 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -61,8 +61,8 @@//! (fulfillment time as the algorithm observes it) plus a `libsy.client_call`//! span around the actual API call when [`run`](Algorithm::run) serves it;//! each [`Driver::info`] decision is logged with its reasoning; and OpenTelemetry-//! metrics record run/call counts, latency, token usage, and published-//! decisions, keyed by [`Algorithm::name`] plus `selected_model` and `outcome`.+//! metrics record run/call counts, latency, and published decisions, keyed by+//! [`Algorithm::name`] plus `selected_model` and `outcome`.//! Metrics use the global meter provider and spans/logs the `tracing` facade: a//! host that installs an OTel SDK and a `tracing` subscriber (bridged with//! `tracing-opentelemetry` for OTLP spans) gets the full signal set; withdiff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 09f61a793..2fbf79c7c 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -193,9 +193,8 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp}/// Records the resolution of one offloaded model call: the call counter and-/// latency histogram, token counters from the response usage (absent fields are-/// skipped, not recorded as zero), the `outcome`/`error`/token fields on-/// `span`, and a warn log when the call failed.+/// latency histogram, the `outcome`/`error`/token fields on `span`, and a warn+/// log when the call failed.pub(crate) fn record_llm_call(algorithm: &str,selected_model: &str,@@ -254,38 +253,14 @@ pub(crate) fn record_llm_call(let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {return;};- let token_attributes = [- KeyValue::new("algorithm", algorithm.to_string()),- KeyValue::new("selected_model", selected_model.to_string()),- ];- for (counter, field, value) in [- (- "switchyard.input_tokens",- "input_tokens",- usage.input_tokens,- ),- (- "switchyard.output_tokens",- "output_tokens",- usage.output_tokens,- ),- (- "switchyard.total_tokens",- "total_tokens",- usage.total_tokens,- ),- (- "switchyard.reasoning_tokens",- "reasoning_tokens",- usage.reasoning_tokens,- ),+ for (field, value) in [+ ("input_tokens", usage.input_tokens),+ ("output_tokens", usage.output_tokens),+ ("total_tokens", usage.total_tokens),+ ("reasoning_tokens", usage.reasoning_tokens),] {if let Some(value) = value {span.record(field, value);- meter- .u64_counter(counter)- .build()- .add(value, &token_attributes);}}}diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rsindex e22cef8eb..3d540827f 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy/tests/observability.rs@@ -433,7 +433,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lassert_eq!(trace.len(), 1);// Metrics: run/call counters and latency histograms keyed by algorithm,- // token counters from the response usage, one published decision.+ // plus one published decision.let snapshots = flushed_metrics(exporter, provider);let run_attrs = [("algorithm", ALGO), ("outcome", "ok")];let call_attrs = [@@ -458,22 +458,6 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lf64_histogram_count(&snapshots, "switchyard.llm_call_duration_ms", &call_attrs),Some(1));- assert_eq!(- u64_counter_value(&snapshots, "switchyard.input_tokens", &token_attrs),- Some(11)- );- assert_eq!(- u64_counter_value(&snapshots, "switchyard.output_tokens", &token_attrs),- Some(7)- );- assert_eq!(- u64_counter_value(&snapshots, "switchyard.total_tokens", &token_attrs),- Some(18)- );- assert_eq!(- u64_counter_value(&snapshots, "switchyard.reasoning_tokens", &token_attrs),- Some(2)- );assert_eq!(u64_counter_value(&snapshots, "switchyard.decisions", &token_attrs),Some(1)@@ -625,8 +609,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::"expected an error step from the failed call");- // Metrics: the call and the run both count under outcome=error, and no- // token counters exist for the failed model.+ // Metrics: the call and the run both count under outcome=error.let snapshots = flushed_metrics(exporter, provider);let run_attrs = [("algorithm", ALGO), ("outcome", "error")];let call_attrs = [@@ -642,14 +625,6 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),Some(1));- assert_eq!(- u64_counter_value(- &snapshots,- "switchyard.input_tokens",- &[("algorithm", ALGO), ("selected_model", MODEL)],- ),- None- );assert_eq!(u64_counter_value(&snapshots, "switchyard.errors", &[("model", MODEL)]),Some(1)diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex f83760b0b..b88424bdd 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -94,6 +94,12 @@ Routed-call compatibility metrics are:| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls || `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls || `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |+| `switchyard_prompt_tokens_total` | counter | `model`, optional `tier` | Input tokens, including cached and cache-creation tokens |+| `switchyard_completion_tokens_total` | counter | `model`, optional `tier` | Output tokens |+| `switchyard_cached_tokens_total` | counter | `model`, optional `tier` | Cached input tokens |+| `switchyard_cache_creation_tokens_total` | counter | `model`, optional `tier` | Cache-creation input tokens |+| `switchyard_reasoning_tokens_total` | counter | `model`, optional `tier` | Reasoning output tokens |+| `switchyard_total_latency_ms` | histogram | `model`, optional `tier` | Full-turn latency for successful routed responses || `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses || `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts || `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |@@ -101,4 +107,12 @@ Routed-call compatibility metrics are:The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision andis omitted for untiered algorithms. Classifier calls are excluded from these families.+`switchyard_total_latency_ms` observes an aggregate when it becomes available or a stream when it+ends cleanly. Its clock starts in a router-wide middleware, before the request body is read and+decoded, so it covers the same span as the Python server's request-ingress-to-completion+measurement. It still excludes connection accept and TLS handshake, which hyper completes before+the server sees the request. The Rust server exports this metric as a histogram, while the Python+server exports its counterpart as a summary; this matches the existing histogram/summary difference+for model-call latency.+See [CONFIGURATION.md](CONFIGURATION.md) to add an LLM client, target, or algorithm.diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 6dcb5d2c5..aa2b4fd1d 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -7,6 +7,7 @@ pub mod config;mod metrics;mod response;mod sse;+mod usage_metrics;use std::collections::BTreeMap;use std::error::Error;@@ -16,12 +17,13 @@ use std::path::PathBuf;use std::sync::Arc;use std::time::{Duration, Instant};-use axum::extract::{rejection::JsonRejection, DefaultBodyLimit, State};+use axum::extract::{rejection::JsonRejection, DefaultBodyLimit, Request as HttpRequest, State};use axum::http::header::CONTENT_TYPE;use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};+use axum::middleware::Next;use axum::response::{IntoResponse, Response};use axum::routing::{get, post};-use axum::{Json, Router};+use axum::{Extension, Json, Router};use axum_server::tls_rustls::RustlsConfig;use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};use serde_json::{json, Value};@@ -192,6 +194,20 @@ async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {.map_err(server_io_error)}+/// Ingress timestamp for one request, taken before any body is read.+#[derive(Clone, Copy)]+struct RequestStart(Instant);++/// Stamps the ingress instant into request extensions. Runs as a router layer,+/// so it executes before the handlers' `Json` extractor buffers the body —+/// request-latency measurements therefore include body read and decode.+async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {+ request+ .extensions_mut()+ .insert(RequestStart(Instant::now()));+ next.run(request).await+}+/// Builds an Axum router for the supported LLM wire formats.pub fn build_switchyard_router(state: ServerState) -> Router {Router::new()@@ -204,6 +220,8 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.route("/health", get(health)).fallback(not_found).layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))+ // `layer` only wraps routes registered before it, so this stays last.+ .layer(axum::middleware::from_fn(stamp_request_start)).with_state(state)}@@ -236,26 +254,29 @@ async fn shutdown_signal() {async fn openai_chat_completions(State(state): State<ServerState>,+ Extension(started): Extension<RequestStart>,headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,) -> Response {- handle_endpoint(state, headers, body, WireFormat::OpenAiChat).await+ handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await}async fn anthropic_messages(State(state): State<ServerState>,+ Extension(started): Extension<RequestStart>,headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,) -> Response {- handle_endpoint(state, headers, body, WireFormat::AnthropicMessages).await+ handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await}async fn openai_responses(State(state): State<ServerState>,+ Extension(started): Extension<RequestStart>,headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,) -> Response {- handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await+ handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await}/// Anthropic token counting. Resolves the route named by `model`, then does a@@ -306,13 +327,14 @@ fn count_tokens_error(error: LibsyError) -> Response {async fn handle_endpoint(state: ServerState,+ started: RequestStart,headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,wire_format: WireFormat,) -> Response {let metadata = metadata_from_headers(&headers);let request_log = RequestLogContext {- started: Instant::now(),+ started: started.0,wire_format,requested_model: body.as_ref()@@ -331,7 +353,7 @@ async fn handle_endpoint(};let response = match llm_json_body(body) {- Ok(body) => handle_llm_request(state, metadata, body, wire_format).await,+ Ok(body) => handle_llm_request(state, started, metadata, body, wire_format).await,Err(message) => invalid_body_error(message),};metrics::record_client_response(response.status().as_u16());@@ -395,6 +417,7 @@ fn resolve_route(async fn handle_llm_request(state: ServerState,+ started: RequestStart,metadata: Metadata,body: Value,wire_format: WireFormat,@@ -408,6 +431,16 @@ async fn handle_llm_request(Ok(result) => result,Err(error) => return algorithm_error(error),};+ let response = if let Some(decision) = trace.last() {+ usage_metrics::observe(+ response,+ decision.selected_model(),+ decision.routing_tier(),+ started.0,+ )+ } else {+ response+ };let mut response = match into_http_response(response, wire_format, Some(requested_model)) {Ok(response) => response,diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rsnew file mode 100644index 000000000..4e9e6cd9d--- /dev/null+++ b/crates/switchyard-server/src/usage_metrics.rs@@ -0,0 +1,109 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Response usage and full-turn latency metrics for routed requests.++use std::time::Instant;++use futures_util::StreamExt;+use libsy::{LlmResponse, LlmResponseChunk, Response, Usage};+use opentelemetry::{global, KeyValue};++/// Observes a routed response without changing its aggregate or streaming contents.+pub(crate) fn observe(+ response: Response,+ model: &str,+ tier: Option<&str>,+ started: Instant,+) -> Response {+ let Response {+ llm_response,+ metadata,+ } = response;+ let model = model.to_string();+ let tier = tier.map(str::to_string);++ let llm_response = match llm_response {+ LlmResponse::Agg(agg) => {+ record_usage(&agg.usage, &model, tier.as_deref());+ record_latency(&model, tier.as_deref(), started);+ LlmResponse::Agg(agg)+ }+ LlmResponse::Stream(mut stream) => {+ let wrapped = async_stream::stream! {+ let mut latest_usage = None;+ while let Some(item) = stream.next().await {+ let failed = matches!(+ &item,+ Err(_)+ | Ok(+ LlmResponseChunk::StreamError { .. }+ | LlmResponseChunk::DecodeError { .. }+ )+ );+ if let Ok(LlmResponseChunk::Usage(usage)) = &item {+ latest_usage = Some(usage.clone());+ }+ yield item;+ if failed {+ return;+ }+ }+ if let Some(usage) = latest_usage {+ record_usage(&usage, &model, tier.as_deref());+ }+ record_latency(&model, tier.as_deref(), started);+ };+ LlmResponse::Stream(Box::pin(wrapped))+ }+ };++ Response {+ llm_response,+ metadata,+ }+}++fn attributes(model: &str, tier: Option<&str>) -> Vec<KeyValue> {+ let mut attributes = vec![KeyValue::new("model", model.to_string())];+ if let Some(tier) = tier {+ attributes.push(KeyValue::new("tier", tier.to_string()));+ }+ attributes+}++fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {+ let attributes = attributes(model, tier);+ let meter = global::meter("switchyard");+ let cached = usage.cached_input_tokens();+ let cache_creation = usage.cache_creation_input_tokens();++ if usage.input_tokens.is_some() || cached.is_some() || cache_creation.is_some() {+ let prompt =+ usage.input_tokens.unwrap_or(0) + cached.unwrap_or(0) + cache_creation.unwrap_or(0);+ meter+ .u64_counter("switchyard.prompt_tokens")+ .build()+ .add(prompt, &attributes);+ }+ for (name, value) in [+ ("switchyard.completion_tokens", usage.output_tokens),+ ("switchyard.cached_tokens", cached),+ ("switchyard.cache_creation_tokens", cache_creation),+ ("switchyard.reasoning_tokens", usage.reasoning_tokens),+ ] {+ if let Some(value) = value {+ meter.u64_counter(name).build().add(value, &attributes);+ }+ }+}++fn record_latency(model: &str, tier: Option<&str>, started: Instant) {+ global::meter("switchyard")+ .f64_histogram("switchyard.total_latency_ms")+ .build()+ .record(+ started.elapsed().as_secs_f64() * 1000.0,+ &attributes(model, tier),+ );+}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 9b5328931..c1c09201b 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -85,9 +85,25 @@ async fn upstream_chat(let model = body["model"].as_str().unwrap_or("unknown").to_string();if body["stream"].as_bool() == Some(true) {+ if body["messages"][0]["content"] == "stream-error" {+ let events = [+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "before"}}]}).to_string(),+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "still here"}}], "usage": {"prompt_tokens": 6, "completion_tokens": 2, "total_tokens": 8}}).to_string(),+ json!({"error": {"message": "upstream stream failed", "type": "server_error"}}).to_string(),+ ];+ let stream = futures_util::stream::iter(+ events+ .into_iter()+ .map(|data| Ok::<Event, Infallible>(Event::default().data(data))),+ );+ return Sse::new(stream).into_response();+ }let events = [json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(),+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-partial"}}], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "prompt_tokens_details": {"cached_tokens": 2, "cache_creation_tokens": 1}}}).to_string(),+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, "completion_tokens_details": {"reasoning_tokens": 3}}}).to_string(),json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(),"[DONE]".to_string(),];@@ -173,7 +189,8 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route#[tokio::test]async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {- let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;+ const MODEL: &str = "model/metrics-buffered";+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;let before = send(&app, "GET", "/metrics", None).await?;assert_eq!(before.status, StatusCode::OK);@@ -232,14 +249,52 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {"# TYPE switchyard_llm_calls_total counter","# TYPE switchyard_run_duration_ms histogram","# TYPE switchyard_llm_call_duration_ms histogram",+ "# TYPE switchyard_prompt_tokens_total counter",+ "# TYPE switchyard_completion_tokens_total counter",+ "# TYPE switchyard_cached_tokens_total counter",+ "# TYPE switchyard_total_latency_ms histogram","algorithm=\"random\"",- "selected_model=\"model/a\"",+ &format!("selected_model=\"{MODEL}\""),] {assert!(metrics.contains(expected),"missing {expected:?} in metrics:\n{metrics}");}+ for (name, expected_delta) in [+ ("switchyard_prompt_tokens_total", 10.0),+ ("switchyard_completion_tokens_total", 2.0),+ ("switchyard_cached_tokens_total", 7.0),+ ("switchyard_total_latency_ms_count", 1.0),+ ] {+ assert_eq!(+ metric_delta(seeded, metrics, name, &[("model", MODEL)]),+ Some(expected_delta),+ "unexpected delta for {name}"+ );+ }+ assert!(metric_line(+ metrics,+ "switchyard_cache_creation_tokens_total",+ &[("model", MODEL)]+ )+ .is_none());+ assert!(metric_line(+ metrics,+ "switchyard_reasoning_tokens_total",+ &[("model", MODEL)]+ )+ .is_none());+ for metric in [+ "switchyard_prompt_tokens_total",+ "switchyard_completion_tokens_total",+ "switchyard_cached_tokens_total",+ "switchyard_total_latency_ms_count",+ ] {+ let line = metric_line(metrics, metric, &[("model", MODEL)])+ .ok_or_else(|| format!("missing {metric} series for {MODEL}"))?;+ assert!(!line.contains("tier="), "unexpected tier label in {line}");+ }Ok(())}@@ -308,6 +363,38 @@ impl Response {}}+fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> {+ metrics.lines().find(|line| {+ line.starts_with(name)+ && labels+ .iter()+ .all(|(key, value)| line.contains(&format!("{key}=\"{value}\"")))+ })+}++fn metric_value(metrics: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {+ metric_line(metrics, name, labels)?+ .split_whitespace()+ .last()?+ .parse()+ .ok()+}++fn metric_delta(before: &str, after: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {+ metric_value(after, name, labels)+ .map(|after| after - metric_value(before, name, labels).unwrap_or_default())+}++fn assert_in_order(haystack: &str, needles: &[&str]) {+ let mut remainder = haystack;+ for needle in needles {+ let offset = remainder+ .find(needle)+ .unwrap_or_else(|| panic!("missing {needle:?} after prior events in:\n{haystack}"));+ remainder = &remainder[offset + needle.len()..];+ }+}+#[tokio::test]async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {let upstream = MockUpstream::start().await?;@@ -690,6 +777,100 @@ async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {Ok(())}+#[tokio::test]+async fn streaming_success_records_only_final_usage_and_one_latency() -> TestResult {+ const MODEL: &str = "model/stream-success";+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;+ let before = send(&app, "GET", "/metrics", None).await?;+ let before = before.text()?;++ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "stream-success"}],+ "stream": true+ })),+ )+ .await?;++ assert_eq!(response.status, StatusCode::OK);+ assert_in_order(+ response.text()?,+ &[+ "hello",+ "-partial",+ "-final",+ "\"finish_reason\":\"stop\"",+ "[DONE]",+ ],+ );++ let after = send(&app, "GET", "/metrics", None).await?;+ let after = after.text()?;+ for (name, expected_delta) in [+ ("switchyard_prompt_tokens_total", 12.0),+ ("switchyard_completion_tokens_total", 5.0),+ ("switchyard_cached_tokens_total", 7.0),+ ("switchyard_cache_creation_tokens_total", 2.0),+ ("switchyard_reasoning_tokens_total", 3.0),+ ("switchyard_total_latency_ms_count", 1.0),+ ] {+ assert_eq!(+ metric_delta(before, after, name, &[("model", MODEL)]),+ Some(expected_delta),+ "unexpected delta for {name}"+ );+ }+ Ok(())+}++#[tokio::test]+async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {+ const MODEL: &str = "model/stream-error";+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;+ let before = send(&app, "GET", "/metrics", None).await?;+ let before = before.text()?;++ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "stream-error"}],+ "stream": true+ })),+ )+ .await?;++ assert_eq!(response.status, StatusCode::OK);+ assert_in_order(+ response.text()?,+ &["before", "still here", "upstream stream failed"],+ );++ let after = send(&app, "GET", "/metrics", None).await?;+ let after = after.text()?;+ for name in [+ "switchyard_prompt_tokens_total",+ "switchyard_completion_tokens_total",+ "switchyard_cached_tokens_total",+ "switchyard_cache_creation_tokens_total",+ "switchyard_reasoning_tokens_total",+ "switchyard_total_latency_ms_count",+ ] {+ assert_eq!(+ metric_value(after, name, &[("model", MODEL)]),+ metric_value(before, name, &[("model", MODEL)]),+ "{name} changed after a failed stream"+ );+ }+ Ok(())+}+#[tokio::test]async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult {let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;