Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(switchyard-server): Prometheus /metrics endpoint (#165)
Graham King50f6622fedfa
diff --git a/Cargo.lock b/Cargo.lockindex 26a48a2e9..d0082b8c0 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1020,7 +1020,20 @@ dependencies = ["futures-sink","js-sys","pin-project-lite",- "thiserror",+ "thiserror 2.0.18",+ "tracing",+]++[[package]]+name = "opentelemetry-prometheus"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2c0359983e7f79cf33c9abd89e5d7ddf67c46c419d0148598022d70e70c01aba"+dependencies = [+ "once_cell",+ "opentelemetry",+ "opentelemetry_sdk",+ "prometheus","tracing",]@@ -1037,7 +1050,7 @@ dependencies = ["percent-encoding","portable-atomic","rand 0.9.5",- "thiserror",+ "thiserror 2.0.18","tokio",]@@ -1125,6 +1138,41 @@ dependencies = ["unicode-ident",]+[[package]]+name = "prometheus"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"+dependencies = [+ "cfg-if",+ "fnv",+ "lazy_static",+ "memchr",+ "parking_lot",+ "protobuf",+ "thiserror 2.0.18",+]++[[package]]+name = "protobuf"+version = "3.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"+dependencies = [+ "once_cell",+ "protobuf-support",+ "thiserror 1.0.69",+]++[[package]]+name = "protobuf-support"+version = "3.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"+dependencies = [+ "thiserror 1.0.69",+]+[[package]]name = "pyo3"version = "0.28.3"@@ -1221,7 +1269,7 @@ dependencies = ["rustc-hash","rustls","socket2",- "thiserror",+ "thiserror 2.0.18","tokio","tracing","web-time",@@ -1243,7 +1291,7 @@ dependencies = ["rustls","rustls-pki-types","slab",- "thiserror",+ "thiserror 2.0.18","tinyvec","tracing","web-time",@@ -1706,7 +1754,7 @@ dependencies = ["rand 0.8.7","serde","serde_json",- "thiserror",+ "thiserror 2.0.18","tokio","tracing",]@@ -1724,7 +1772,7 @@ dependencies = ["serde_json","switchyard-llm-client","switchyard-protocol",- "thiserror",+ "thiserror 2.0.18","tokio","tokio-stream","tracing",@@ -1754,7 +1802,7 @@ dependencies = ["futures","serde","serde_json",- "thiserror",+ "thiserror 2.0.18",][[package]]@@ -1789,6 +1837,10 @@ dependencies = ["clap","futures-util","http-body-util",+ "opentelemetry",+ "opentelemetry-prometheus",+ "opentelemetry_sdk",+ "prometheus","rustls","serde","serde_json",@@ -1809,7 +1861,7 @@ dependencies = ["async-trait","serde","serde_json",- "thiserror",+ "thiserror 2.0.18","tokio",]@@ -1823,7 +1875,7 @@ dependencies = ["serde","serde_json","switchyard-protocol",- "thiserror",+ "thiserror 2.0.18",][[package]]@@ -1876,13 +1928,33 @@ dependencies = ["windows-sys 0.61.2",]+[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl 1.0.69",+]+[[package]]name = "thiserror"version = "2.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"dependencies = [- "thiserror-impl",+ "thiserror-impl 2.0.18",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",][[package]]diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 1c14ceab6..9bb4010e0 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -19,6 +19,10 @@ rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs",clap = { version = "4", features = ["derive", "env"] }futures-util.workspace = truelibsy = { package = "switchyard-libsy", path = "../libsy" }+opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }+opentelemetry-prometheus = "0.32"+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }+prometheus = "0.14"serde.workspace = truetoml = "1.1"switchyard-llm-client.workspace = truediff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex e9935adc3..2720e9235 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -4,6 +4,7 @@//! Rust HTTP server for libsy algorithms.pub mod config;+mod metrics;mod response;mod sse;@@ -16,6 +17,7 @@ use std::sync::Arc;use std::time::Duration;use axum::extract::{rejection::JsonRejection, State};+use axum::http::header::CONTENT_TYPE;use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};use axum::response::{IntoResponse, Response};use axum::routing::{get, post};@@ -66,6 +68,7 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;#[derive(Clone)]pub struct ServerState {routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,+ metrics: prometheus::Registry,}impl ServerState {@@ -86,8 +89,10 @@ impl ServerState {if entries.is_empty() {return Err(ServerError::new("at least one algorithm route is required"));}+ let metrics = metrics::registry().map_err(ServerError::new)?;Ok(Self {routes: Arc::new(entries),+ metrics,})}@@ -190,6 +195,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {.route("/v1/messages", post(anthropic_messages)).route("/v1/responses", post(openai_responses)).route("/v1/models", get(models))+ .route("/metrics", get(prometheus_metrics)).route("/health", get(health)).fallback(not_found).with_state(state)@@ -463,6 +469,13 @@ async fn health() -> Json<Value> {Json(json!({"status": "ok"}))}+async fn prometheus_metrics(State(state): State<ServerState>) -> Response {+ match metrics::encode(&state.metrics) {+ Ok(body) => ([(CONTENT_TYPE, metrics::CONTENT_TYPE)], body).into_response(),+ Err(error) => server_error(error),+ }+}+async fn not_found() -> Response {error_response(StatusCode::NOT_FOUND,diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rsnew file mode 100644index 000000000..de76d6221--- /dev/null+++ b/crates/switchyard-server/src/metrics.rs@@ -0,0 +1,50 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Process-wide Prometheus export for libsy's OpenTelemetry metrics.++use std::sync::OnceLock;++use opentelemetry::global;+use opentelemetry_sdk::metrics::SdkMeterProvider;+use prometheus::{Encoder, Registry, TextEncoder};++pub(crate) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";++struct Metrics {+ registry: Registry,+ _provider: SdkMeterProvider,+}++static METRICS: OnceLock<Result<Metrics, String>> = OnceLock::new();++/// Returns the registry shared by every server router in this process.+pub(crate) fn registry() -> Result<Registry, String> {+ match METRICS.get_or_init(initialize) {+ Ok(metrics) => Ok(metrics.registry.clone()),+ Err(error) => Err(error.clone()),+ }+}++fn initialize() -> Result<Metrics, String> {+ let registry = Registry::new();+ let exporter = opentelemetry_prometheus::exporter()+ .with_registry(registry.clone())+ .build()+ .map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;+ let provider = SdkMeterProvider::builder().with_reader(exporter).build();+ global::set_meter_provider(provider.clone());+ Ok(Metrics {+ registry,+ _provider: provider,+ })+}++/// Encodes the current cumulative metric values in Prometheus text format.+pub(crate) fn encode(registry: &Registry) -> Result<Vec<u8>, String> {+ let mut body = Vec::new();+ TextEncoder::new()+ .encode(®istry.gather(), &mut body)+ .map_err(|error| format!("failed to encode Prometheus metrics: {error}"))?;+ Ok(body)+}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex f3fe06667..b7a6abbc1 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -159,6 +159,50 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, RouteOk((upstream, app))}+#[tokio::test]+async fn metrics_exposes_libsy_otel_instruments() -> TestResult {+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;++ let before = send(&app, "GET", "/metrics", None).await?;+ assert_eq!(before.status, StatusCode::OK);+ assert_eq!(+ before+ .headers+ .get("content-type")+ .and_then(|value| value.to_str().ok()),+ Some("text/plain; version=0.0.4; charset=utf-8")+ );++ let response = send(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL,+ "messages": [{"role": "user", "content": "hello"}]+ })),+ )+ .await?;+ assert_eq!(response.status, StatusCode::OK);++ let after = send(&app, "GET", "/metrics", None).await?;+ let metrics = after.text()?;+ for expected in [+ "# TYPE libsy_runs_total counter",+ "# TYPE libsy_llm_calls_total counter",+ "# TYPE libsy_run_duration_ms histogram",+ "# TYPE libsy_llm_call_duration_ms histogram",+ "algorithm=\"random\"",+ "selected_model=\"model/a\"",+ ] {+ assert!(+ metrics.contains(expected),+ "missing {expected:?} in metrics:\n{metrics}"+ );+ }+ Ok(())+}+fn load_test_config(toml: &str) -> TestResult<ServerState> {let mut config = tempfile::Builder::new().prefix("switchyard-server-config-")