Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy-llm-client): Metrics on HTTP traffic (#171)
Graham King2faf5f18f41a
diff --git a/Cargo.lock b/Cargo.lockindex 1cc8e7946..284c22e85 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1796,6 +1796,7 @@ dependencies = ["async-trait","futures","futures-util",+ "opentelemetry","reqwest","serde_json","switchyard-protocol",diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlindex b2054dfb0..19b2470fe 100644--- a/crates/libsy-llm-client/Cargo.toml+++ b/crates/libsy-llm-client/Cargo.toml@@ -17,6 +17,7 @@ switchyard-translation.workspace = truereqwest.workspace = trueasync-trait.workspace = truefutures-util.workspace = true+opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }serde_json.workspace = true[dev-dependencies]diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 5ca18cc16..058b66fd4 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -20,6 +20,7 @@ use switchyard_translation::{use crate::backend::Backend;use crate::error::{LlmClientError, Result};+use crate::metrics::record_upstream_attempt;use crate::raw::RawResponse;// TODO: Why is this here? What does it do?@@ -168,7 +169,16 @@ impl TranslatingLlmClient {let builder = apply_extra_headers(builder, backend);let builder = backend.apply_auth(builder);- let http_response = builder.send().await.map_err(convert_reqwest_error)?;+ let http_response = match builder.send().await {+ Ok(response) => {+ record_upstream_attempt(Some(response.status().as_u16()));+ response+ }+ Err(error) => {+ record_upstream_attempt(None);+ return Err(convert_reqwest_error(error));+ }+ };let status = http_response.status();if !status.is_success() {let body = match http_response.text().await {diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex 22a1c4726..79fb84e3a 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -14,6 +14,7 @@pub mod backend;pub mod client;pub mod error;+pub mod metrics;pub mod raw;pub use backend::{Backend, HttpBackendConfig};diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rsnew file mode 100644index 000000000..67202463d--- /dev/null+++ b/crates/libsy-llm-client/src/metrics.rs@@ -0,0 +1,55 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Metric labelling inherited from Python++use opentelemetry::{global, KeyValue};++pub const fn http_outcome_label(status: Option<u16>) -> &'static str {+ match status {+ Some(200..=299) => "success",+ Some(429 | 500 | 504) | None => "retryable_error",+ Some(_) => "other_error",+ }+}++/// Limit the cardinality of the HTTP status code.+/// This matches Python, but likely we should log the status code directly. Most of these never+/// appear.+pub const fn http_status_code_label(status: Option<u16>) -> &'static str {+ match status {+ None => "none",+ Some(200) => "200",+ Some(400) => "400",+ Some(401) => "401",+ Some(403) => "403",+ Some(404) => "404",+ Some(408) => "408",+ Some(409) => "409",+ Some(422) => "422",+ Some(429) => "429",+ Some(500) => "500",+ Some(502) => "502",+ Some(503) => "503",+ Some(504) => "504",+ Some(100..=199) => "1xx",+ Some(200..=299) => "2xx",+ Some(300..=399) => "3xx",+ Some(400..=499) => "4xx",+ Some(500..=599) => "5xx",+ Some(_) => "other",+ }+}++pub(crate) fn record_upstream_attempt(status: Option<u16>) {+ global::meter("switchyard")+ .u64_counter("switchyard.upstream_attempts")+ .build()+ .add(+ 1,+ &[+ KeyValue::new("outcome", http_outcome_label(status)),+ KeyValue::new("code", http_status_code_label(status)),+ ],+ );+}diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex f37767a53..8cfec5844 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -77,6 +77,9 @@ Routed-call compatibility metrics are:| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls || `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls || `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |+| `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses |+| `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |+| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision andis omitted for untiered algorithms. Classifier calls are excluded from these families.diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex bb0ae635a..dd3ae8ba3 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -194,8 +194,8 @@ pub fn build_switchyard_router(state: ServerState) -> Router {Router::new().route("/v1/chat/completions", post(openai_chat_completions)).route("/v1/messages", post(anthropic_messages))- .route("/v1/messages/count_tokens", post(anthropic_count_tokens)).route("/v1/responses", post(openai_responses))+ .route("/v1/messages/count_tokens", post(anthropic_count_tokens)).route("/v1/models", get(models)).route("/metrics", get(prometheus_metrics)).route("/health", get(health))@@ -330,6 +330,7 @@ async fn handle_endpoint(Ok(body) => handle_llm_request(state, metadata, body, wire_format).await,Err(message) => invalid_body_error(message),};+ metrics::record_client_response(response.status().as_u16());request_log.emit(&response);response}diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rsindex 7b75233c6..d27b95b9d 100644--- a/crates/switchyard-server/src/metrics.rs+++ b/crates/switchyard-server/src/metrics.rs@@ -8,6 +8,7 @@ use std::sync::OnceLock;use opentelemetry::{global, KeyValue};use opentelemetry_sdk::metrics::SdkMeterProvider;use prometheus::{Encoder, Registry, TextEncoder};+use switchyard_llm_client::metrics::{http_outcome_label, http_status_code_label};pub(crate) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";@@ -39,12 +40,49 @@ fn initialize() -> Result<Metrics, String> {.u64_gauge("switchyard.build_info").build().record(1, &[KeyValue::new("version", env!("CARGO_PKG_VERSION"))]);+ seed_outcome_metrics();Ok(Metrics {registry,_provider: provider,})}+/// Make the metrics exist before they get a hit. Nicer for dashboards but not really necessary.+/// The HTTP status codes we seed are somewhat arbitrary.+fn seed_outcome_metrics() {+ let meter = global::meter("switchyard");+ let upstream_attempts = meter.u64_counter("switchyard.upstream_attempts").build();+ for status in [Some(200), Some(404), Some(429), Some(500), Some(504), None] {+ upstream_attempts.add(+ 0,+ &[+ KeyValue::new("outcome", http_outcome_label(status)),+ KeyValue::new("code", http_status_code_label(status)),+ ],+ );+ }++ let client_responses = meter.u64_counter("switchyard.client_responses").build();+ for outcome in ["success", "retryable_error", "other_error"] {+ client_responses.add(0, &[KeyValue::new("outcome", outcome)]);+ }+ meter+ .u64_counter("switchyard.router_retry_recovered")+ .build()+ .add(0, &[]);+}++/// Records the final status returned by an LLM-serving route.+pub(crate) fn record_client_response(status: u16) {+ global::meter("switchyard")+ .u64_counter("switchyard.client_responses")+ .build()+ .add(+ 1,+ &[KeyValue::new("outcome", http_outcome_label(Some(status)))],+ );+}+/// Encodes the current cumulative metric values in Prometheus text format.pub(crate) fn encode(registry: &Registry) -> Result<Vec<u8>, String> {let mut body = Vec::new();diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex e75fe5445..c963b393b 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -183,6 +183,26 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {.and_then(|value| value.to_str().ok()),Some("text/plain; version=0.0.4; charset=utf-8"));+ let seeded = before.text()?;+ for expected in [+ "# TYPE switchyard_client_responses_total counter",+ "switchyard_client_responses_total{outcome=\"success\",",+ "switchyard_client_responses_total{outcome=\"retryable_error\",",+ "switchyard_client_responses_total{outcome=\"other_error\",",+ "# TYPE switchyard_upstream_attempts_total counter",+ "switchyard_upstream_attempts_total{code=\"200\",outcome=\"success\",",+ "switchyard_upstream_attempts_total{code=\"429\",outcome=\"retryable_error\",",+ "switchyard_upstream_attempts_total{code=\"500\",outcome=\"retryable_error\",",+ "switchyard_upstream_attempts_total{code=\"504\",outcome=\"retryable_error\",",+ "switchyard_upstream_attempts_total{code=\"none\",outcome=\"retryable_error\",",+ "# TYPE switchyard_router_retry_recovered_total counter",+ "switchyard_router_retry_recovered_total{otel_scope_name=\"switchyard\"} 0",+ ] {+ assert!(+ seeded.contains(expected),+ "missing seeded {expected:?} in metrics:\n{seeded}"+ );+ }let response = send(&app,@@ -205,6 +225,8 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {"# TYPE switchyard_total_errors gauge","# TYPE switchyard_requests_total counter","# TYPE switchyard_model_call_latency_ms histogram",+ "switchyard_client_responses_total{outcome=\"success\",",+ "switchyard_upstream_attempts_total{code=\"200\",outcome=\"success\",","# TYPE switchyard_runs_total counter","# TYPE switchyard_llm_calls_total counter","# TYPE switchyard_run_duration_ms histogram",