Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(switchyard-server): Prometheus /metrics endpoint (#165)

Graham King50f6622fedfa
diff --git a/Cargo.lock b/Cargo.lock
index 26a48a2e9..d0082b8c0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1020,7 +1020,20 @@ dependencies = [
"futures-sink",
"js-sys",
"pin-project-lite",
- "thiserror",
+ "thiserror 2.0.18",
+ "tracing",
+]
+
+[[package]]
+name = "opentelemetry-prometheus"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c0359983e7f79cf33c9abd89e5d7ddf67c46c419d0148598022d70e70c01aba"
+dependencies = [
+ "once_cell",
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prometheus",
"tracing",
]
@@ -1037,7 +1050,7 @@ dependencies = [
"percent-encoding",
"portable-atomic",
"rand 0.9.5",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
]
@@ -1125,6 +1138,41 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "prometheus"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
+dependencies = [
+ "cfg-if",
+ "fnv",
+ "lazy_static",
+ "memchr",
+ "parking_lot",
+ "protobuf",
+ "thiserror 2.0.18",
+]
+
+[[package]]
+name = "protobuf"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
+dependencies = [
+ "once_cell",
+ "protobuf-support",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
+dependencies = [
+ "thiserror 1.0.69",
+]
+
[[package]]
name = "pyo3"
version = "0.28.3"
@@ -1221,7 +1269,7 @@ dependencies = [
"rustc-hash",
"rustls",
"socket2",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
@@ -1243,7 +1291,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"slab",
- "thiserror",
+ "thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
@@ -1706,7 +1754,7 @@ dependencies = [
"rand 0.8.7",
"serde",
"serde_json",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tracing",
]
@@ -1724,7 +1772,7 @@ dependencies = [
"serde_json",
"switchyard-llm-client",
"switchyard-protocol",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
"tokio-stream",
"tracing",
@@ -1754,7 +1802,7 @@ dependencies = [
"futures",
"serde",
"serde_json",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -1789,6 +1837,10 @@ dependencies = [
"clap",
"futures-util",
"http-body-util",
+ "opentelemetry",
+ "opentelemetry-prometheus",
+ "opentelemetry_sdk",
+ "prometheus",
"rustls",
"serde",
"serde_json",
@@ -1809,7 +1861,7 @@ dependencies = [
"async-trait",
"serde",
"serde_json",
- "thiserror",
+ "thiserror 2.0.18",
"tokio",
]
@@ -1823,7 +1875,7 @@ dependencies = [
"serde",
"serde_json",
"switchyard-protocol",
- "thiserror",
+ "thiserror 2.0.18",
]
[[package]]
@@ -1876,13 +1928,33 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl 1.0.69",
+]
+
[[package]]
name = "thiserror"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
- "thiserror-impl",
+ "thiserror-impl 2.0.18",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 1c14ceab6..9bb4010e0 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -19,6 +19,10 @@ rustls = { version = "0.23", default-features = false, features = ["aws-lc-rs",
clap = { version = "4", features = ["derive", "env"] }
futures-util.workspace = true
libsy = { package = "switchyard-libsy", path = "../libsy" }
+opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
+opentelemetry-prometheus = "0.32"
+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics"] }
+prometheus = "0.14"
serde.workspace = true
toml = "1.1"
switchyard-llm-client.workspace = true
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index e9935adc3..2720e9235 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -4,6 +4,7 @@
//! Rust HTTP server for libsy algorithms.
pub mod config;
+mod metrics;
mod response;
mod sse;
@@ -16,6 +17,7 @@ use std::sync::Arc;
use std::time::Duration;
use axum::extract::{rejection::JsonRejection, State};
+use axum::http::header::CONTENT_TYPE;
use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
@@ -66,6 +68,7 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;
#[derive(Clone)]
pub struct ServerState {
routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
+ metrics: prometheus::Registry,
}
impl ServerState {
@@ -86,8 +89,10 @@ impl ServerState {
if entries.is_empty() {
return Err(ServerError::new("at least one algorithm route is required"));
}
+ let metrics = metrics::registry().map_err(ServerError::new)?;
Ok(Self {
routes: Arc::new(entries),
+ metrics,
})
}
@@ -190,6 +195,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
.route("/v1/models", get(models))
+ .route("/metrics", get(prometheus_metrics))
.route("/health", get(health))
.fallback(not_found)
.with_state(state)
@@ -463,6 +469,13 @@ async fn health() -> Json<Value> {
Json(json!({"status": "ok"}))
}
+async fn prometheus_metrics(State(state): State<ServerState>) -> Response {
+ match metrics::encode(&state.metrics) {
+ Ok(body) => ([(CONTENT_TYPE, metrics::CONTENT_TYPE)], body).into_response(),
+ Err(error) => server_error(error),
+ }
+}
+
async fn not_found() -> Response {
error_response(
StatusCode::NOT_FOUND,
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
new file mode 100644
index 000000000..de76d6221
--- /dev/null
+++ b/crates/switchyard-server/src/metrics.rs
@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Process-wide Prometheus export for libsy's OpenTelemetry metrics.
+
+use std::sync::OnceLock;
+
+use opentelemetry::global;
+use opentelemetry_sdk::metrics::SdkMeterProvider;
+use prometheus::{Encoder, Registry, TextEncoder};
+
+pub(crate) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
+
+struct Metrics {
+ registry: Registry,
+ _provider: SdkMeterProvider,
+}
+
+static METRICS: OnceLock<Result<Metrics, String>> = OnceLock::new();
+
+/// Returns the registry shared by every server router in this process.
+pub(crate) fn registry() -> Result<Registry, String> {
+ match METRICS.get_or_init(initialize) {
+ Ok(metrics) => Ok(metrics.registry.clone()),
+ Err(error) => Err(error.clone()),
+ }
+}
+
+fn initialize() -> Result<Metrics, String> {
+ let registry = Registry::new();
+ let exporter = opentelemetry_prometheus::exporter()
+ .with_registry(registry.clone())
+ .build()
+ .map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;
+ let provider = SdkMeterProvider::builder().with_reader(exporter).build();
+ global::set_meter_provider(provider.clone());
+ Ok(Metrics {
+ registry,
+ _provider: provider,
+ })
+}
+
+/// Encodes the current cumulative metric values in Prometheus text format.
+pub(crate) fn encode(registry: &Registry) -> Result<Vec<u8>, String> {
+ let mut body = Vec::new();
+ TextEncoder::new()
+ .encode(&registry.gather(), &mut body)
+ .map_err(|error| format!("failed to encode Prometheus metrics: {error}"))?;
+ Ok(body)
+}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index f3fe06667..b7a6abbc1 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -159,6 +159,50 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
Ok((upstream, app))
}
+#[tokio::test]
+async fn metrics_exposes_libsy_otel_instruments() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+
+ let before = send(&app, "GET", "/metrics", None).await?;
+ assert_eq!(before.status, StatusCode::OK);
+ assert_eq!(
+ before
+ .headers
+ .get("content-type")
+ .and_then(|value| value.to_str().ok()),
+ Some("text/plain; version=0.0.4; charset=utf-8")
+ );
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "hello"}]
+ })),
+ )
+ .await?;
+ assert_eq!(response.status, StatusCode::OK);
+
+ let after = send(&app, "GET", "/metrics", None).await?;
+ let metrics = after.text()?;
+ for expected in [
+ "# TYPE libsy_runs_total counter",
+ "# TYPE libsy_llm_calls_total counter",
+ "# TYPE libsy_run_duration_ms histogram",
+ "# TYPE libsy_llm_call_duration_ms histogram",
+ "algorithm=\"random\"",
+ "selected_model=\"model/a\"",
+ ] {
+ assert!(
+ metrics.contains(expected),
+ "missing {expected:?} in metrics:\n{metrics}"
+ );
+ }
+ Ok(())
+}
+
fn load_test_config(toml: &str) -> TestResult<ServerState> {
let mut config = tempfile::Builder::new()
.prefix("switchyard-server-config-")