Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy-llm-client): Metrics on HTTP traffic (#171)

Graham King2faf5f18f41a
diff --git a/Cargo.lock b/Cargo.lock
index 1cc8e7946..284c22e85 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1796,6 +1796,7 @@ dependencies = [
"async-trait",
"futures",
"futures-util",
+ "opentelemetry",
"reqwest",
"serde_json",
"switchyard-protocol",
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml
index b2054dfb0..19b2470fe 100644
--- a/crates/libsy-llm-client/Cargo.toml
+++ b/crates/libsy-llm-client/Cargo.toml
@@ -17,6 +17,7 @@ switchyard-translation.workspace = true
reqwest.workspace = true
async-trait.workspace = true
futures-util.workspace = true
+opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
serde_json.workspace = true
[dev-dependencies]
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 5ca18cc16..058b66fd4 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -20,6 +20,7 @@ use switchyard_translation::{
use crate::backend::Backend;
use crate::error::{LlmClientError, Result};
+use crate::metrics::record_upstream_attempt;
use crate::raw::RawResponse;
// TODO: Why is this here? What does it do?
@@ -168,7 +169,16 @@ impl TranslatingLlmClient {
let builder = apply_extra_headers(builder, backend);
let builder = backend.apply_auth(builder);
- let http_response = builder.send().await.map_err(convert_reqwest_error)?;
+ let http_response = match builder.send().await {
+ Ok(response) => {
+ record_upstream_attempt(Some(response.status().as_u16()));
+ response
+ }
+ Err(error) => {
+ record_upstream_attempt(None);
+ return Err(convert_reqwest_error(error));
+ }
+ };
let status = http_response.status();
if !status.is_success() {
let body = match http_response.text().await {
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index 22a1c4726..79fb84e3a 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -14,6 +14,7 @@
pub mod backend;
pub mod client;
pub mod error;
+pub mod metrics;
pub mod raw;
pub use backend::{Backend, HttpBackendConfig};
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rs
new file mode 100644
index 000000000..67202463d
--- /dev/null
+++ b/crates/libsy-llm-client/src/metrics.rs
@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Metric labelling inherited from Python
+
+use opentelemetry::{global, KeyValue};
+
+pub const fn http_outcome_label(status: Option<u16>) -> &'static str {
+ match status {
+ Some(200..=299) => "success",
+ Some(429 | 500 | 504) | None => "retryable_error",
+ Some(_) => "other_error",
+ }
+}
+
+/// Limit the cardinality of the HTTP status code.
+/// This matches Python, but likely we should log the status code directly. Most of these never
+/// appear.
+pub const fn http_status_code_label(status: Option<u16>) -> &'static str {
+ match status {
+ None => "none",
+ Some(200) => "200",
+ Some(400) => "400",
+ Some(401) => "401",
+ Some(403) => "403",
+ Some(404) => "404",
+ Some(408) => "408",
+ Some(409) => "409",
+ Some(422) => "422",
+ Some(429) => "429",
+ Some(500) => "500",
+ Some(502) => "502",
+ Some(503) => "503",
+ Some(504) => "504",
+ Some(100..=199) => "1xx",
+ Some(200..=299) => "2xx",
+ Some(300..=399) => "3xx",
+ Some(400..=499) => "4xx",
+ Some(500..=599) => "5xx",
+ Some(_) => "other",
+ }
+}
+
+pub(crate) fn record_upstream_attempt(status: Option<u16>) {
+ global::meter("switchyard")
+ .u64_counter("switchyard.upstream_attempts")
+ .build()
+ .add(
+ 1,
+ &[
+ KeyValue::new("outcome", http_outcome_label(status)),
+ KeyValue::new("code", http_status_code_label(status)),
+ ],
+ );
+}
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index f37767a53..8cfec5844 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -77,6 +77,9 @@ Routed-call compatibility metrics are:
| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls |
| `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls |
| `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |
+| `switchyard_client_responses_total` | counter | `outcome` | Final LLM-route responses |
+| `switchyard_upstream_attempts_total` | counter | `outcome`, `code` | Actual upstream HTTP attempts |
+| `switchyard_router_retry_recovered_total` | counter | none | Retry recoveries (currently always zero) |
The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision and
is omitted for untiered algorithms. Classifier calls are excluded from these families.
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index bb0ae635a..dd3ae8ba3 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -194,8 +194,8 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
Router::new()
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
- .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/responses", post(openai_responses))
+ .route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/models", get(models))
.route("/metrics", get(prometheus_metrics))
.route("/health", get(health))
@@ -330,6 +330,7 @@ async fn handle_endpoint(
Ok(body) => handle_llm_request(state, metadata, body, wire_format).await,
Err(message) => invalid_body_error(message),
};
+ metrics::record_client_response(response.status().as_u16());
request_log.emit(&response);
response
}
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
index 7b75233c6..d27b95b9d 100644
--- a/crates/switchyard-server/src/metrics.rs
+++ b/crates/switchyard-server/src/metrics.rs
@@ -8,6 +8,7 @@ use std::sync::OnceLock;
use opentelemetry::{global, KeyValue};
use opentelemetry_sdk::metrics::SdkMeterProvider;
use prometheus::{Encoder, Registry, TextEncoder};
+use switchyard_llm_client::metrics::{http_outcome_label, http_status_code_label};
pub(crate) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
@@ -39,12 +40,49 @@ fn initialize() -> Result<Metrics, String> {
.u64_gauge("switchyard.build_info")
.build()
.record(1, &[KeyValue::new("version", env!("CARGO_PKG_VERSION"))]);
+ seed_outcome_metrics();
Ok(Metrics {
registry,
_provider: provider,
})
}
+/// Make the metrics exist before they get a hit. Nicer for dashboards but not really necessary.
+/// The HTTP status codes we seed are somewhat arbitrary.
+fn seed_outcome_metrics() {
+ let meter = global::meter("switchyard");
+ let upstream_attempts = meter.u64_counter("switchyard.upstream_attempts").build();
+ for status in [Some(200), Some(404), Some(429), Some(500), Some(504), None] {
+ upstream_attempts.add(
+ 0,
+ &[
+ KeyValue::new("outcome", http_outcome_label(status)),
+ KeyValue::new("code", http_status_code_label(status)),
+ ],
+ );
+ }
+
+ let client_responses = meter.u64_counter("switchyard.client_responses").build();
+ for outcome in ["success", "retryable_error", "other_error"] {
+ client_responses.add(0, &[KeyValue::new("outcome", outcome)]);
+ }
+ meter
+ .u64_counter("switchyard.router_retry_recovered")
+ .build()
+ .add(0, &[]);
+}
+
+/// Records the final status returned by an LLM-serving route.
+pub(crate) fn record_client_response(status: u16) {
+ global::meter("switchyard")
+ .u64_counter("switchyard.client_responses")
+ .build()
+ .add(
+ 1,
+ &[KeyValue::new("outcome", http_outcome_label(Some(status)))],
+ );
+}
+
/// Encodes the current cumulative metric values in Prometheus text format.
pub(crate) fn encode(registry: &Registry) -> Result<Vec<u8>, String> {
let mut body = Vec::new();
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index e75fe5445..c963b393b 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -183,6 +183,26 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
.and_then(|value| value.to_str().ok()),
Some("text/plain; version=0.0.4; charset=utf-8")
);
+ let seeded = before.text()?;
+ for expected in [
+ "# TYPE switchyard_client_responses_total counter",
+ "switchyard_client_responses_total{outcome=\"success\",",
+ "switchyard_client_responses_total{outcome=\"retryable_error\",",
+ "switchyard_client_responses_total{outcome=\"other_error\",",
+ "# TYPE switchyard_upstream_attempts_total counter",
+ "switchyard_upstream_attempts_total{code=\"200\",outcome=\"success\",",
+ "switchyard_upstream_attempts_total{code=\"429\",outcome=\"retryable_error\",",
+ "switchyard_upstream_attempts_total{code=\"500\",outcome=\"retryable_error\",",
+ "switchyard_upstream_attempts_total{code=\"504\",outcome=\"retryable_error\",",
+ "switchyard_upstream_attempts_total{code=\"none\",outcome=\"retryable_error\",",
+ "# TYPE switchyard_router_retry_recovered_total counter",
+ "switchyard_router_retry_recovered_total{otel_scope_name=\"switchyard\"} 0",
+ ] {
+ assert!(
+ seeded.contains(expected),
+ "missing seeded {expected:?} in metrics:\n{seeded}"
+ );
+ }
let response = send(
&app,
@@ -205,6 +225,8 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
"# TYPE switchyard_total_errors gauge",
"# TYPE switchyard_requests_total counter",
"# TYPE switchyard_model_call_latency_ms histogram",
+ "switchyard_client_responses_total{outcome=\"success\",",
+ "switchyard_upstream_attempts_total{code=\"200\",outcome=\"success\",",
"# TYPE switchyard_runs_total counter",
"# TYPE switchyard_llm_calls_total counter",
"# TYPE switchyard_run_duration_ms histogram",