Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(switchyard-server): Metrics for tokens and latency (#179)

Graham King36b96362fd78
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 4f622649f..5ac6ac835 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -287,11 +287,9 @@ boundary — so algorithms carry no telemetry code beyond `Algorithm::name()`, t
- **Structured logs** (`tracing`, target `libsy`): an info event per published
`Decision` (selected model + reasoning), warn events for failed calls and runs.
- **Metrics** (OpenTelemetry, scope `switchyard`, via the global meter provider): counters
- `switchyard.runs`, `switchyard.llm_calls`, `switchyard.decisions`,
- `switchyard.input_tokens`, `switchyard.output_tokens`, `switchyard.total_tokens`,
- `switchyard.reasoning_tokens`; histograms `switchyard.run_duration_ms` and
- `switchyard.llm_call_duration_ms`. Attributes are `algorithm`, `selected_model`, and
- `outcome` (`ok`/`error`) — failure rates are the
+ `switchyard.runs`, `switchyard.llm_calls`, and `switchyard.decisions`; histograms
+ `switchyard.run_duration_ms` and `switchyard.llm_call_duration_ms`. Attributes are
+ `algorithm`, `selected_model`, and `outcome` (`ok`/`error`) — failure rates are the
`outcome="error"` share of runs/calls.
libsy also records compatibility metrics for final routed calls:
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 0c54c3015..9f29db910 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -61,8 +61,8 @@
//! (fulfillment time as the algorithm observes it) plus a `libsy.client_call`
//! span around the actual API call when [`run`](Algorithm::run) serves it;
//! each [`Driver::info`] decision is logged with its reasoning; and OpenTelemetry
-//! metrics record run/call counts, latency, token usage, and published
-//! decisions, keyed by [`Algorithm::name`] plus `selected_model` and `outcome`.
+//! metrics record run/call counts, latency, and published decisions, keyed by
+//! [`Algorithm::name`] plus `selected_model` and `outcome`.
//! Metrics use the global meter provider and spans/logs the `tracing` facade: a
//! host that installs an OTel SDK and a `tracing` subscriber (bridged with
//! `tracing-opentelemetry` for OTLP spans) gets the full signal set; with
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 09f61a793..2fbf79c7c 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -193,9 +193,8 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
}
/// Records the resolution of one offloaded model call: the call counter and
-/// latency histogram, token counters from the response usage (absent fields are
-/// skipped, not recorded as zero), the `outcome`/`error`/token fields on
-/// `span`, and a warn log when the call failed.
+/// latency histogram, the `outcome`/`error`/token fields on `span`, and a warn
+/// log when the call failed.
pub(crate) fn record_llm_call(
algorithm: &str,
selected_model: &str,
@@ -254,38 +253,14 @@ pub(crate) fn record_llm_call(
let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {
return;
};
- let token_attributes = [
- KeyValue::new("algorithm", algorithm.to_string()),
- KeyValue::new("selected_model", selected_model.to_string()),
- ];
- for (counter, field, value) in [
- (
- "switchyard.input_tokens",
- "input_tokens",
- usage.input_tokens,
- ),
- (
- "switchyard.output_tokens",
- "output_tokens",
- usage.output_tokens,
- ),
- (
- "switchyard.total_tokens",
- "total_tokens",
- usage.total_tokens,
- ),
- (
- "switchyard.reasoning_tokens",
- "reasoning_tokens",
- usage.reasoning_tokens,
- ),
+ for (field, value) in [
+ ("input_tokens", usage.input_tokens),
+ ("output_tokens", usage.output_tokens),
+ ("total_tokens", usage.total_tokens),
+ ("reasoning_tokens", usage.reasoning_tokens),
] {
if let Some(value) = value {
span.record(field, value);
- meter
- .u64_counter(counter)
- .build()
- .add(value, &token_attributes);
}
}
}
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index e22cef8eb..3d540827f 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -433,7 +433,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
assert_eq!(trace.len(), 1);
// Metrics: run/call counters and latency histograms keyed by algorithm,
- // token counters from the response usage, one published decision.
+ // plus one published decision.
let snapshots = flushed_metrics(exporter, provider);
let run_attrs = [("algorithm", ALGO), ("outcome", "ok")];
let call_attrs = [
@@ -458,22 +458,6 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
f64_histogram_count(&snapshots, "switchyard.llm_call_duration_ms", &call_attrs),
Some(1)
);
- assert_eq!(
- u64_counter_value(&snapshots, "switchyard.input_tokens", &token_attrs),
- Some(11)
- );
- assert_eq!(
- u64_counter_value(&snapshots, "switchyard.output_tokens", &token_attrs),
- Some(7)
- );
- assert_eq!(
- u64_counter_value(&snapshots, "switchyard.total_tokens", &token_attrs),
- Some(18)
- );
- assert_eq!(
- u64_counter_value(&snapshots, "switchyard.reasoning_tokens", &token_attrs),
- Some(2)
- );
assert_eq!(
u64_counter_value(&snapshots, "switchyard.decisions", &token_attrs),
Some(1)
@@ -625,8 +609,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
"expected an error step from the failed call"
);
- // Metrics: the call and the run both count under outcome=error, and no
- // token counters exist for the failed model.
+ // Metrics: the call and the run both count under outcome=error.
let snapshots = flushed_metrics(exporter, provider);
let run_attrs = [("algorithm", ALGO), ("outcome", "error")];
let call_attrs = [
@@ -642,14 +625,6 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),
Some(1)
);
- assert_eq!(
- u64_counter_value(
- &snapshots,
- "switchyard.input_tokens",
- &[("algorithm", ALGO), ("selected_model", MODEL)],
- ),
- None
- );
assert_eq!(
u64_counter_value(&snapshots, "switchyard.errors", &[("model", MODEL)]),
Some(1)
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index f83760b0b..b88424bdd 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -94,6 +94,12 @@ Routed-call compatibility metrics are:
| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls |
| `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls |
| `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |
+| `switchyard_prompt_tokens_total` | counter | `model`, optional `tier` | Input tokens, including cached and cache-creation tokens |
+| `switchyard_completion_tokens_total` | counter | `model`, optional `tier` | Output tokens |
+| `switchyard_cached_tokens_total` | counter | `model`, optional `tier` | Cached input tokens |
+| `switchyard_cache_creation_tokens_total` | counter | `model`, optional `tier` | Cache-creation input tokens |
+| `switchyard_reasoning_tokens_total` | counter | `model`, optional `tier` | Reasoning output tokens |
+| `switchyard_total_latency_ms` | histogram | `model`, optional `tier` | Full-turn latency for successful routed responses |
| `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses |
| `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |
| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |
@@ -101,4 +107,12 @@ Routed-call compatibility metrics are:
The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision and
is omitted for untiered algorithms. Classifier calls are excluded from these families.
+`switchyard_total_latency_ms` observes an aggregate when it becomes available or a stream when it
+ends cleanly. Its clock starts in a router-wide middleware, before the request body is read and
+decoded, so it covers the same span as the Python server's request-ingress-to-completion
+measurement. It still excludes connection accept and TLS handshake, which hyper completes before
+the server sees the request. The Rust server exports this metric as a histogram, while the Python
+server exports its counterpart as a summary; this matches the existing histogram/summary difference
+for model-call latency.
+
See [CONFIGURATION.md](CONFIGURATION.md) to add an LLM client, target, or algorithm.
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 6dcb5d2c5..aa2b4fd1d 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -7,6 +7,7 @@ pub mod config;
mod metrics;
mod response;
mod sse;
+mod usage_metrics;
use std::collections::BTreeMap;
use std::error::Error;
@@ -16,12 +17,13 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
-use axum::extract::{rejection::JsonRejection, DefaultBodyLimit, State};
+use axum::extract::{rejection::JsonRejection, DefaultBodyLimit, Request as HttpRequest, State};
use axum::http::header::CONTENT_TYPE;
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
+use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
-use axum::{Json, Router};
+use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};
use serde_json::{json, Value};
@@ -192,6 +194,20 @@ async fn serve(listener: TcpListener, router: Router) -> ServerResult<()> {
.map_err(server_io_error)
}
+/// Ingress timestamp for one request, taken before any body is read.
+#[derive(Clone, Copy)]
+struct RequestStart(Instant);
+
+/// Stamps the ingress instant into request extensions. Runs as a router layer,
+/// so it executes before the handlers' `Json` extractor buffers the body —
+/// request-latency measurements therefore include body read and decode.
+async fn stamp_request_start(mut request: HttpRequest, next: Next) -> Response {
+ request
+ .extensions_mut()
+ .insert(RequestStart(Instant::now()));
+ next.run(request).await
+}
+
/// Builds an Axum router for the supported LLM wire formats.
pub fn build_switchyard_router(state: ServerState) -> Router {
Router::new()
@@ -204,6 +220,8 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/health", get(health))
.fallback(not_found)
.layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES))
+ // `layer` only wraps routes registered before it, so this stays last.
+ .layer(axum::middleware::from_fn(stamp_request_start))
.with_state(state)
}
@@ -236,26 +254,29 @@ async fn shutdown_signal() {
async fn openai_chat_completions(
State(state): State<ServerState>,
+ Extension(started): Extension<RequestStart>,
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
- handle_endpoint(state, headers, body, WireFormat::OpenAiChat).await
+ handle_endpoint(state, started, headers, body, WireFormat::OpenAiChat).await
}
async fn anthropic_messages(
State(state): State<ServerState>,
+ Extension(started): Extension<RequestStart>,
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
- handle_endpoint(state, headers, body, WireFormat::AnthropicMessages).await
+ handle_endpoint(state, started, headers, body, WireFormat::AnthropicMessages).await
}
async fn openai_responses(
State(state): State<ServerState>,
+ Extension(started): Extension<RequestStart>,
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
) -> Response {
- handle_endpoint(state, headers, body, WireFormat::OpenAiResponses).await
+ handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await
}
/// Anthropic token counting. Resolves the route named by `model`, then does a
@@ -306,13 +327,14 @@ fn count_tokens_error(error: LibsyError) -> Response {
async fn handle_endpoint(
state: ServerState,
+ started: RequestStart,
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
wire_format: WireFormat,
) -> Response {
let metadata = metadata_from_headers(&headers);
let request_log = RequestLogContext {
- started: Instant::now(),
+ started: started.0,
wire_format,
requested_model: body
.as_ref()
@@ -331,7 +353,7 @@ async fn handle_endpoint(
};
let response = match llm_json_body(body) {
- Ok(body) => handle_llm_request(state, metadata, body, wire_format).await,
+ Ok(body) => handle_llm_request(state, started, metadata, body, wire_format).await,
Err(message) => invalid_body_error(message),
};
metrics::record_client_response(response.status().as_u16());
@@ -395,6 +417,7 @@ fn resolve_route(
async fn handle_llm_request(
state: ServerState,
+ started: RequestStart,
metadata: Metadata,
body: Value,
wire_format: WireFormat,
@@ -408,6 +431,16 @@ async fn handle_llm_request(
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
+ let response = if let Some(decision) = trace.last() {
+ usage_metrics::observe(
+ response,
+ decision.selected_model(),
+ decision.routing_tier(),
+ started.0,
+ )
+ } else {
+ response
+ };
let mut response = match into_http_response(response, wire_format, Some(requested_model)) {
Ok(response) => response,
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
new file mode 100644
index 000000000..4e9e6cd9d
--- /dev/null
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -0,0 +1,109 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Response usage and full-turn latency metrics for routed requests.
+
+use std::time::Instant;
+
+use futures_util::StreamExt;
+use libsy::{LlmResponse, LlmResponseChunk, Response, Usage};
+use opentelemetry::{global, KeyValue};
+
+/// Observes a routed response without changing its aggregate or streaming contents.
+pub(crate) fn observe(
+ response: Response,
+ model: &str,
+ tier: Option<&str>,
+ started: Instant,
+) -> Response {
+ let Response {
+ llm_response,
+ metadata,
+ } = response;
+ let model = model.to_string();
+ let tier = tier.map(str::to_string);
+
+ let llm_response = match llm_response {
+ LlmResponse::Agg(agg) => {
+ record_usage(&agg.usage, &model, tier.as_deref());
+ record_latency(&model, tier.as_deref(), started);
+ LlmResponse::Agg(agg)
+ }
+ LlmResponse::Stream(mut stream) => {
+ let wrapped = async_stream::stream! {
+ let mut latest_usage = None;
+ while let Some(item) = stream.next().await {
+ let failed = matches!(
+ &item,
+ Err(_)
+ | Ok(
+ LlmResponseChunk::StreamError { .. }
+ | LlmResponseChunk::DecodeError { .. }
+ )
+ );
+ if let Ok(LlmResponseChunk::Usage(usage)) = &item {
+ latest_usage = Some(usage.clone());
+ }
+ yield item;
+ if failed {
+ return;
+ }
+ }
+ if let Some(usage) = latest_usage {
+ record_usage(&usage, &model, tier.as_deref());
+ }
+ record_latency(&model, tier.as_deref(), started);
+ };
+ LlmResponse::Stream(Box::pin(wrapped))
+ }
+ };
+
+ Response {
+ llm_response,
+ metadata,
+ }
+}
+
+fn attributes(model: &str, tier: Option<&str>) -> Vec<KeyValue> {
+ let mut attributes = vec![KeyValue::new("model", model.to_string())];
+ if let Some(tier) = tier {
+ attributes.push(KeyValue::new("tier", tier.to_string()));
+ }
+ attributes
+}
+
+fn record_usage(usage: &Usage, model: &str, tier: Option<&str>) {
+ let attributes = attributes(model, tier);
+ let meter = global::meter("switchyard");
+ let cached = usage.cached_input_tokens();
+ let cache_creation = usage.cache_creation_input_tokens();
+
+ if usage.input_tokens.is_some() || cached.is_some() || cache_creation.is_some() {
+ let prompt =
+ usage.input_tokens.unwrap_or(0) + cached.unwrap_or(0) + cache_creation.unwrap_or(0);
+ meter
+ .u64_counter("switchyard.prompt_tokens")
+ .build()
+ .add(prompt, &attributes);
+ }
+ for (name, value) in [
+ ("switchyard.completion_tokens", usage.output_tokens),
+ ("switchyard.cached_tokens", cached),
+ ("switchyard.cache_creation_tokens", cache_creation),
+ ("switchyard.reasoning_tokens", usage.reasoning_tokens),
+ ] {
+ if let Some(value) = value {
+ meter.u64_counter(name).build().add(value, &attributes);
+ }
+ }
+}
+
+fn record_latency(model: &str, tier: Option<&str>, started: Instant) {
+ global::meter("switchyard")
+ .f64_histogram("switchyard.total_latency_ms")
+ .build()
+ .record(
+ started.elapsed().as_secs_f64() * 1000.0,
+ &attributes(model, tier),
+ );
+}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 9b5328931..c1c09201b 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -85,9 +85,25 @@ async fn upstream_chat(
let model = body["model"].as_str().unwrap_or("unknown").to_string();
if body["stream"].as_bool() == Some(true) {
+ if body["messages"][0]["content"] == "stream-error" {
+ let events = [
+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "before"}}]}).to_string(),
+ json!({"id": "chatcmpl-stream-error", "model": model, "choices": [{"index": 0, "delta": {"content": "still here"}}], "usage": {"prompt_tokens": 6, "completion_tokens": 2, "total_tokens": 8}}).to_string(),
+ json!({"error": {"message": "upstream stream failed", "type": "server_error"}}).to_string(),
+ ];
+ let stream = futures_util::stream::iter(
+ events
+ .into_iter()
+ .map(|data| Ok::<Event, Infallible>(Event::default().data(data))),
+ );
+ return Sse::new(stream).into_response();
+ }
let events = [
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"role": "assistant"}}]}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "hello"}}]}).to_string(),
+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-partial"}}], "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6, "prompt_tokens_details": {"cached_tokens": 2, "cache_creation_tokens": 1}}}).to_string(),
+ json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {"content": "-final"}}], "usage": {"prompt_tokens": 12, "completion_tokens": 5, "total_tokens": 17, "prompt_tokens_details": {"cached_tokens": 7, "cache_creation_tokens": 2}, "completion_tokens_details": {"reasoning_tokens": 3}}}).to_string(),
json!({"id": "chatcmpl-stream", "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}).to_string(),
"[DONE]".to_string(),
];
@@ -173,7 +189,8 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
#[tokio::test]
async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
- let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+ const MODEL: &str = "model/metrics-buffered";
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
let before = send(&app, "GET", "/metrics", None).await?;
assert_eq!(before.status, StatusCode::OK);
@@ -232,14 +249,52 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
"# TYPE switchyard_llm_calls_total counter",
"# TYPE switchyard_run_duration_ms histogram",
"# TYPE switchyard_llm_call_duration_ms histogram",
+ "# TYPE switchyard_prompt_tokens_total counter",
+ "# TYPE switchyard_completion_tokens_total counter",
+ "# TYPE switchyard_cached_tokens_total counter",
+ "# TYPE switchyard_total_latency_ms histogram",
"algorithm=\"random\"",
- "selected_model=\"model/a\"",
+ &format!("selected_model=\"{MODEL}\""),
] {
assert!(
metrics.contains(expected),
"missing {expected:?} in metrics:\n{metrics}"
);
}
+ for (name, expected_delta) in [
+ ("switchyard_prompt_tokens_total", 10.0),
+ ("switchyard_completion_tokens_total", 2.0),
+ ("switchyard_cached_tokens_total", 7.0),
+ ("switchyard_total_latency_ms_count", 1.0),
+ ] {
+ assert_eq!(
+ metric_delta(seeded, metrics, name, &[("model", MODEL)]),
+ Some(expected_delta),
+ "unexpected delta for {name}"
+ );
+ }
+ assert!(metric_line(
+ metrics,
+ "switchyard_cache_creation_tokens_total",
+ &[("model", MODEL)]
+ )
+ .is_none());
+ assert!(metric_line(
+ metrics,
+ "switchyard_reasoning_tokens_total",
+ &[("model", MODEL)]
+ )
+ .is_none());
+ for metric in [
+ "switchyard_prompt_tokens_total",
+ "switchyard_completion_tokens_total",
+ "switchyard_cached_tokens_total",
+ "switchyard_total_latency_ms_count",
+ ] {
+ let line = metric_line(metrics, metric, &[("model", MODEL)])
+ .ok_or_else(|| format!("missing {metric} series for {MODEL}"))?;
+ assert!(!line.contains("tier="), "unexpected tier label in {line}");
+ }
Ok(())
}
@@ -308,6 +363,38 @@ impl Response {
}
}
+fn metric_line<'a>(metrics: &'a str, name: &str, labels: &[(&str, &str)]) -> Option<&'a str> {
+ metrics.lines().find(|line| {
+ line.starts_with(name)
+ && labels
+ .iter()
+ .all(|(key, value)| line.contains(&format!("{key}=\"{value}\"")))
+ })
+}
+
+fn metric_value(metrics: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {
+ metric_line(metrics, name, labels)?
+ .split_whitespace()
+ .last()?
+ .parse()
+ .ok()
+}
+
+fn metric_delta(before: &str, after: &str, name: &str, labels: &[(&str, &str)]) -> Option<f64> {
+ metric_value(after, name, labels)
+ .map(|after| after - metric_value(before, name, labels).unwrap_or_default())
+}
+
+fn assert_in_order(haystack: &str, needles: &[&str]) {
+ let mut remainder = haystack;
+ for needle in needles {
+ let offset = remainder
+ .find(needle)
+ .unwrap_or_else(|| panic!("missing {needle:?} after prior events in:\n{haystack}"));
+ remainder = &remainder[offset + needle.len()..];
+ }
+}
+
#[tokio::test]
async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {
let upstream = MockUpstream::start().await?;
@@ -690,6 +777,100 @@ async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {
Ok(())
}
+#[tokio::test]
+async fn streaming_success_records_only_final_usage_and_one_latency() -> TestResult {
+ const MODEL: &str = "model/stream-success";
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
+ let before = send(&app, "GET", "/metrics", None).await?;
+ let before = before.text()?;
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "stream-success"}],
+ "stream": true
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ assert_in_order(
+ response.text()?,
+ &[
+ "hello",
+ "-partial",
+ "-final",
+ "\"finish_reason\":\"stop\"",
+ "[DONE]",
+ ],
+ );
+
+ let after = send(&app, "GET", "/metrics", None).await?;
+ let after = after.text()?;
+ for (name, expected_delta) in [
+ ("switchyard_prompt_tokens_total", 12.0),
+ ("switchyard_completion_tokens_total", 5.0),
+ ("switchyard_cached_tokens_total", 7.0),
+ ("switchyard_cache_creation_tokens_total", 2.0),
+ ("switchyard_reasoning_tokens_total", 3.0),
+ ("switchyard_total_latency_ms_count", 1.0),
+ ] {
+ assert_eq!(
+ metric_delta(before, after, name, &[("model", MODEL)]),
+ Some(expected_delta),
+ "unexpected delta for {name}"
+ );
+ }
+ Ok(())
+}
+
+#[tokio::test]
+async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {
+ const MODEL: &str = "model/stream-error";
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
+ let before = send(&app, "GET", "/metrics", None).await?;
+ let before = before.text()?;
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "stream-error"}],
+ "stream": true
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ assert_in_order(
+ response.text()?,
+ &["before", "still here", "upstream stream failed"],
+ );
+
+ let after = send(&app, "GET", "/metrics", None).await?;
+ let after = after.text()?;
+ for name in [
+ "switchyard_prompt_tokens_total",
+ "switchyard_completion_tokens_total",
+ "switchyard_cached_tokens_total",
+ "switchyard_cache_creation_tokens_total",
+ "switchyard_reasoning_tokens_total",
+ "switchyard_total_latency_ms_count",
+ ] {
+ assert_eq!(
+ metric_value(after, name, &[("model", MODEL)]),
+ metric_value(before, name, &[("model", MODEL)]),
+ "{name} changed after a failed stream"
+ );
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn request_and_upstream_errors_use_the_canonical_envelope() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;