Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(switchyard-server): Add some metrics (#167)

Graham King4a39c5f93603
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index d1ec9081b..9a0a6821f 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -262,13 +262,29 @@ boundary — so algorithms carry no telemetry code beyond `Algorithm::name()`, t
their own transport should span their `RoutedLlmClient` equivalently.
- **Structured logs** (`tracing`, target `libsy`): an info event per published
`Decision` (selected model + reasoning), warn events for failed calls and runs.
-- **Metrics** (OpenTelemetry, scope `libsy`, via the global meter provider): counters
- `libsy.runs`, `libsy.llm_calls`, `libsy.decisions`, `libsy.input_tokens`,
- `libsy.output_tokens`, `libsy.total_tokens`, `libsy.reasoning_tokens`; histograms
- `libsy.run_duration_ms`, `libsy.llm_call_duration_ms`. Attributes are `algorithm`,
- `selected_model`, and `outcome` (`ok`/`error`) — failure rates are the
+- **Metrics** (OpenTelemetry, scope `switchyard`, via the global meter provider): counters
+ `switchyard.runs`, `switchyard.llm_calls`, `switchyard.decisions`,
+ `switchyard.input_tokens`, `switchyard.output_tokens`, `switchyard.total_tokens`,
+ `switchyard.reasoning_tokens`; histograms `switchyard.run_duration_ms` and
+ `switchyard.llm_call_duration_ms`. Attributes are `algorithm`, `selected_model`, and
+ `outcome` (`ok`/`error`) — failure rates are the
`outcome="error"` share of runs/calls.
+libsy also records compatibility metrics for final routed calls:
+
+| OpenTelemetry instrument | Prometheus family | Type | Labels | Meaning |
+|---|---|---|---|---|
+| `switchyard.total_requests` | `switchyard_total_requests` | gauge | none | Successful and failed routed calls |
+| `switchyard.total_errors` | `switchyard_total_errors` | gauge | none | Failed routed calls |
+| `switchyard.requests` | `switchyard_requests_total` | counter | `model`, optional `tier` | Successful routed calls |
+| `switchyard.errors` | `switchyard_errors_total` | counter | `model`, optional `tier` | Failed routed calls |
+| `switchyard.model_call_latency_ms` | `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful routed-call latency |
+
+Classifier calls remain visible in the general LLM-call instruments but are excluded from these
+compatibility families. The `tier` label is present only when a decision supplies a stable tier.
+Hosts that need the two zero-valued gauges before the first call must invoke
+`libsy::initialize_metrics()` after installing the global meter provider.
+
libsy owns no exporter. The host installs an OpenTelemetry SDK meter provider
(`opentelemetry::global::set_meter_provider`) and a `tracing` subscriber (bridge spans
into OTel with `tracing-opentelemetry` if desired); with neither installed, all
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 433ea606f..a87e7edfd 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -28,6 +28,7 @@ use crate::{
pub struct FallThroughDecision {
model: String,
reason: String,
+ tier: Option<&'static str>,
}
impl Decision for FallThroughDecision {
@@ -35,6 +36,10 @@ impl Decision for FallThroughDecision {
&self.model
}
+ fn routing_tier(&self) -> Option<&str> {
+ self.tier
+ }
+
fn reasoning(&self) -> Option<&str> {
Some(&self.reason)
}
@@ -119,30 +124,33 @@ impl Algorithm<SharedState> for FallThrough {
// 2. Fall through the cascade: the first classifier to score decides (argmax). The
// per-request driver is offered to each — driver-backed classifiers use it.
- let mut winner: Option<Score> = None;
+ let mut maybe_score: Option<Score> = None;
+ let mut maybe_tier: Option<&'static str> = None;
for classifier in &self.classifiers {
let scores = classifier
.score(&mut state, &mut request, Some(&driver))
.await?;
- if let Some(score) = scores.argmax(false)? {
- winner = Some(score);
+ maybe_score = scores.argmax(false)?;
+ if let Some(s) = maybe_score.as_ref() {
+ maybe_tier = classifier.routing_tier(&s.target);
break;
}
}
- let Some(winner) = winner else {
+ let Some(score) = maybe_score else {
return Err(LibsyError::AlgorithmError {
message: "every classifier abstained".to_string(),
});
};
// 3. Resolve the target and publish the decision.
- let target = self.targets.get_target(&winner.target)?;
+ let target = self.targets.get_target(&score.target)?;
let decision: Arc<dyn Decision> = Arc::new(FallThroughDecision {
- model: winner.target.clone(),
+ model: score.target.clone(),
reason: format!(
"fall-through selected {} (confidence {:.3})",
- winner.target, winner.confidence
+ score.target, score.confidence
),
+ tier: maybe_tier,
});
driver.info(ctx.without_state(), decision.clone()).await?;
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 19e5dfc05..eb5061bf3 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -165,6 +165,8 @@ impl JudgePolicy for TaskClassifierPolicy {
/// A full-request capability classifier configured with the packaged prompt and schema.
pub struct LlmTaskClassifier {
classifier: JudgeClassifier<CapabilityJudge, TaskClassifierPolicy>,
+ efficient_target: String,
+ capable_target: String,
}
impl LlmTaskClassifier {
@@ -184,16 +186,20 @@ impl LlmTaskClassifier {
let judge = CapabilityJudge {
config: Self::load_judge_config()?,
};
+ let efficient_target = efficient_target.semantic_name;
+ let capable_target = capable_target.semantic_name;
Ok(Self {
classifier: JudgeClassifier::new(
judge,
judge_target,
TaskClassifierPolicy::new(
- efficient_target.semantic_name,
- capable_target.semantic_name,
+ efficient_target.clone(),
+ capable_target.clone(),
threshold,
),
),
+ efficient_target,
+ capable_target,
})
}
@@ -225,10 +231,22 @@ impl LlmTaskClassifier {
#[async_trait]
impl Classifier for LlmTaskClassifier {
+ fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
+ if self.efficient_target == self.capable_target {
+ None
+ } else if selected_model == self.efficient_target {
+ Some("weak")
+ } else if selected_model == self.capable_target {
+ Some("strong")
+ } else {
+ None
+ }
+ }
+
async fn score(
&self,
state: &mut State,
- request: &Request,
+ request: &mut Request,
driver: Option<&Driver>,
) -> Result<Classification> {
self.classifier.score(state, request, driver).await
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 8c2ff481b..04b29cdd8 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -126,7 +126,7 @@ where
async fn score(
&self,
state: &mut State,
- request: &Request,
+ request: &mut Request,
driver: Option<&Driver>,
) -> Result<Classification> {
// A missing driver is a broken composition, not an unavailable judge.
@@ -165,6 +165,10 @@ impl Decision for JudgeDecision {
&self.model
}
+ fn is_routed_call(&self) -> bool {
+ false
+ }
+
fn reasoning(&self) -> Option<&str> {
Some("llm judge consultation")
}
@@ -293,15 +297,17 @@ mod tests {
let mut steps = Box::pin(driver.stream());
let classifier = classifier();
let mut state = State::default();
- let request = request();
+ let mut request = request();
let serve = async {
if let Some(Ok(Step::CallLlm(call))) = steps.next().await {
let _ = call.respond(reply);
}
};
- let (classification, ()) =
- tokio::join!(classifier.score(&mut state, &request, Some(&driver)), serve);
+ let (classification, ()) = tokio::join!(
+ classifier.score(&mut state, &mut request, Some(&driver)),
+ serve
+ );
selected(classification?)
}
@@ -376,7 +382,7 @@ mod tests {
#[tokio::test]
async fn a_missing_driver_is_an_error_not_a_fallback() -> Result<()> {
let error = classifier()
- .score(&mut State::default(), &request(), None)
+ .score(&mut State::default(), &mut request(), None)
.await
.err()
.ok_or_else(|| LibsyError::AlgorithmError {
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 041c78f64..c12678b53 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -148,6 +148,8 @@ impl Driver {
pub async fn call_llm(&self, routed: RoutedRequest) -> Result<Response> {
let algorithm = observability::algorithm_label(&routed.ctx).to_string();
let selected_model = routed.decision.selected_model().to_string();
+ let tier = routed.decision.routing_tier().map(str::to_string);
+ let is_routed = routed.decision.is_routed_call();
let started = Instant::now();
let result = self
.driver
@@ -156,6 +158,8 @@ impl Driver {
observability::record_llm_call(
&algorithm,
&selected_model,
+ tier.as_deref(),
+ is_routed,
started.elapsed(),
&result,
&tracing::Span::current(),
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index cbcc61aed..0d949ddb1 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -70,6 +70,11 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {
/// Scores each of the classifier's targets given State the current Request
#[async_trait]
pub trait Classifier: Send + Sync {
+ /// Stable tier represented by `selected_model`, when this classifier defines one.
+ fn routing_tier(&self, _selected_model: &str) -> Option<&'static str> {
+ None
+ }
+
/// Score the classifier's targets given the current state and request.
/// driver is optional. It is used to offload model calls
///
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 68bddd10a..e4a6bea86 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -88,6 +88,13 @@ pub mod algorithms;
mod observability;
pub use algorithms::util::tool_signals::{ToolSignals, DEFAULT_RECENT_WINDOW};
+/// Registers process-wide compatibility gauges with the global meter provider.
+///
+/// Hosts should call this after installing their OpenTelemetry meter provider.
+pub fn initialize_metrics() {
+ observability::initialize_metrics();
+}
+
/// Stage-router scoring and tier selection — the shared signal-driven routing
/// core (scorer, picker, and the `StageClassifier`).
// TODO cleanup once switchyard-components is removed
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 0b21ee6f6..a2eeae382 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -7,7 +7,7 @@
//! The crate's provided run methods call these helpers around the [`Decision`]
//! hook and the offload boundary, so every algorithm is instrumented from the
//! outside and carries no telemetry code of its own. Metrics record through the
-//! OpenTelemetry **global** meter provider under the `libsy` scope — the host
+//! OpenTelemetry **global** meter provider under the `switchyard` scope — the host
//! installs an SDK provider and exporters; with none installed, recording is a
//! no-op. Spans and logs use the `tracing` facade (the async-native surface the
//! OpenTelemetry ecosystem bridges with `tracing-opentelemetry` /
@@ -19,8 +19,8 @@
//! tasks create there (see the `tracing` docs on spans in asynchronous code).
//!
//! Instrument names use the OTel dotted form with the unit baked into the name
-//! (`libsy.run_duration_ms`), matching the switchyard metric surface; a
-//! Prometheus exporter sanitizes them to `libsy_run_duration_ms`. Attribute
+//! (`switchyard.run_duration_ms`), matching the switchyard metric surface; a
+//! Prometheus exporter sanitizes them to `switchyard_run_duration_ms`. Attribute
//! cardinality is bounded: `algorithm` and `selected_model` are small
//! configured sets and `outcome` is `ok`/`error`. Nothing per-request becomes a
//! metric attribute — correlation ids ride on the `libsy.run` span instead.
@@ -31,16 +31,22 @@
//! negligible next to a model call.
use std::future::Future;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::OnceLock;
use std::time::{Duration, Instant};
-use opentelemetry::metrics::Meter;
+use opentelemetry::metrics::{Meter, ObservableGauge};
use opentelemetry::{global, KeyValue};
use tracing::Span;
use crate::{Context, Decision, Metadata, Response, Result};
-/// Instrumentation scope for every libsy meter, span, and log line.
-const SCOPE: &str = "libsy";
+const METRICS_SCOPE: &str = "switchyard";
+const TRACING_TARGET: &str = "libsy";
+
+static TOTAL_REQUESTS: AtomicU64 = AtomicU64::new(0);
+static TOTAL_ERRORS: AtomicU64 = AtomicU64::new(0);
+static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = OnceLock::new();
/// [`Context::values`] key under which `run_stream` stamps the algorithm's
/// telemetry label ([`Algorithm::name`](crate::Algorithm::name)).
@@ -56,7 +62,27 @@ pub(crate) fn algorithm_label<S>(ctx: &Context<S>) -> &str {
/// The `libsy`-scoped meter from the globally installed provider.
fn meter() -> Meter {
- global::meter(SCOPE)
+ global::meter(METRICS_SCOPE)
+}
+
+/// Registers process-wide compatibility gauges with the installed global meter provider.
+pub(crate) fn initialize_metrics() {
+ TOTAL_GAUGES.get_or_init(|| {
+ let meter = meter();
+ let requests = meter
+ .u64_observable_gauge("switchyard.total_requests")
+ .with_callback(|observer| {
+ observer.observe(TOTAL_REQUESTS.load(Ordering::Relaxed), &[]);
+ })
+ .build();
+ let errors = meter
+ .u64_observable_gauge("switchyard.total_errors")
+ .with_callback(|observer| {
+ observer.observe(TOTAL_ERRORS.load(Ordering::Relaxed), &[]);
+ })
+ .build();
+ (requests, errors)
+ });
}
/// `outcome` attribute value for a result: `ok` or `error`.
@@ -77,7 +103,7 @@ fn outcome_value<T>(result: &Result<T>) -> &'static str {
/// by [`record_run`] when the run ends.
pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {
let span = tracing::info_span!(
- target: SCOPE,
+ target: TRACING_TARGET,
"libsy.run",
algorithm,
session_id = tracing::field::Empty,
@@ -143,7 +169,12 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
span.record("outcome", outcome);
if let Err(error) = result {
span.record("error", tracing::field::display(error));
- tracing::warn!(target: SCOPE, algorithm, error = %error, "algorithm run failed");
+ tracing::warn!(
+ target: TRACING_TARGET,
+ algorithm,
+ error = %error,
+ "algorithm run failed"
+ );
}
let attributes = [
@@ -151,9 +182,12 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
KeyValue::new("outcome", outcome),
];
let meter = meter();
- meter.u64_counter("libsy.runs").build().add(1, &attributes);
meter
- .f64_histogram("libsy.run_duration_ms")
+ .u64_counter("switchyard.runs")
+ .build()
+ .add(1, &attributes);
+ meter
+ .f64_histogram("switchyard.run_duration_ms")
.build()
.record(duration.as_secs_f64() * 1000.0, &attributes);
}
@@ -165,6 +199,8 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
pub(crate) fn record_llm_call(
algorithm: &str,
selected_model: &str,
+ tier: Option<&str>,
+ is_routed: bool,
duration: Duration,
result: &Result<Response>,
span: &Span,
@@ -179,14 +215,38 @@ pub(crate) fn record_llm_call(
KeyValue::new("outcome", outcome),
];
meter
- .u64_counter("libsy.llm_calls")
+ .u64_counter("switchyard.llm_calls")
.build()
.add(1, &call_attributes);
meter
- .f64_histogram("libsy.llm_call_duration_ms")
+ .f64_histogram("switchyard.llm_call_duration_ms")
.build()
.record(duration.as_secs_f64() * 1000.0, &call_attributes);
+ if is_routed {
+ TOTAL_REQUESTS.fetch_add(1, Ordering::Relaxed);
+ let mut routed_attributes = vec![KeyValue::new("model", selected_model.to_string())];
+ if let Some(tier) = tier {
+ routed_attributes.push(KeyValue::new("tier", tier.to_string()));
+ }
+ if result.is_ok() {
+ meter
+ .u64_counter("switchyard.requests")
+ .build()
+ .add(1, &routed_attributes);
+ meter
+ .f64_histogram("switchyard.model_call_latency_ms")
+ .build()
+ .record(duration.as_secs_f64() * 1000.0, &routed_attributes);
+ } else {
+ TOTAL_ERRORS.fetch_add(1, Ordering::Relaxed);
+ meter
+ .u64_counter("switchyard.errors")
+ .build()
+ .add(1, &routed_attributes);
+ }
+ }
+
match result {
Ok(response) => {
// Token usage exists only once a response is buffered; a streamed
@@ -199,11 +259,23 @@ pub(crate) fn record_llm_call(
KeyValue::new("selected_model", selected_model.to_string()),
];
for (counter, field, value) in [
- ("libsy.input_tokens", "input_tokens", usage.input_tokens),
- ("libsy.output_tokens", "output_tokens", usage.output_tokens),
- ("libsy.total_tokens", "total_tokens", usage.total_tokens),
(
- "libsy.reasoning_tokens",
+ "switchyard.input_tokens",
+ "input_tokens",
+ usage.input_tokens,
+ ),
+ (
+ "switchyard.output_tokens",
+ "output_tokens",
+ usage.output_tokens,
+ ),
+ (
+ "switchyard.total_tokens",
+ "total_tokens",
+ usage.total_tokens,
+ ),
+ (
+ "switchyard.reasoning_tokens",
"reasoning_tokens",
usage.reasoning_tokens,
),
@@ -220,7 +292,7 @@ pub(crate) fn record_llm_call(
Err(error) => {
span.record("error", tracing::field::display(error));
tracing::warn!(
- target: SCOPE,
+ target: TRACING_TARGET,
algorithm,
selected_model,
error = %error,
@@ -236,13 +308,13 @@ pub(crate) fn record_decision(ctx: &Context, decision: &dyn Decision) {
let algorithm = algorithm_label(ctx);
let selected_model = decision.selected_model();
tracing::debug!(
- target: SCOPE,
+ target: TRACING_TARGET,
algorithm,
selected_model,
reasoning = decision.reasoning().unwrap_or(""),
"routing decision"
);
- meter().u64_counter("libsy.decisions").build().add(
+ meter().u64_counter("switchyard.decisions").build().add(
1,
&[
KeyValue::new("algorithm", algorithm.to_string()),
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 521af72e4..63f2cd6a5 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -26,11 +26,12 @@ use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt};
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::Layer;
+use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
AggLlmResponse, Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmTarget,
- LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, Step, Usage,
+ LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, SharedState, Step, Usage,
};
-use switchyard_protocol::text_request;
+use switchyard_protocol::{text_request, text_response, LlmClientError};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
@@ -152,6 +153,7 @@ fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvid
let reader = PeriodicReader::builder(exporter.clone()).build();
let provider = SdkMeterProvider::builder().with_reader(reader).build();
opentelemetry::global::set_meter_provider(provider.clone());
+ switchyard_libsy::initialize_metrics();
let store = CaptureStore::default();
let subscriber = tracing_subscriber::registry().with(CaptureLayer {
@@ -164,6 +166,16 @@ fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvid
})
}
+/// The three tests in this file must not overlap because metrics are global.
+/// There is no Rust/cargo-native way of saying this (people use `serial_test` crate) so use a
+/// lock.
+/// Each file in `tests/` (integration tests) runs as a separate test process, so we are not
+/// concerned with interactions with tests in other files.
+fn serialize_test() -> &'static tokio::sync::Mutex<()> {
+ static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
+ LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
+}
+
/// Flushes the metric pipeline and returns every exported snapshot.
fn flushed_metrics(
exporter: &InMemoryMetricExporter,
@@ -192,7 +204,7 @@ fn attributes_match<'a>(
.all(|(key, value)| present.iter().any(|(k, v)| k == key && v == value))
}
-/// Latest (max) value for a `libsy`-scoped metric across snapshots, with
+/// Latest (max) value for a `switchyard`-scoped metric across snapshots, with
/// `extract` pulling the matching data points' values out of one metric's
/// aggregated data. Counters and histogram counts are cumulative, so the max
/// across snapshots is the most recent value.
@@ -204,7 +216,7 @@ fn latest_metric_value(
snapshots
.iter()
.flat_map(|snapshot| snapshot.scope_metrics())
- .filter(|scope| scope.scope().name() == "libsy")
+ .filter(|scope| scope.scope().name() == "switchyard")
.flat_map(|scope| scope.metrics())
.filter(|metric| metric.name() == name)
.flat_map(|metric| extract(metric.data()))
@@ -243,6 +255,16 @@ fn f64_histogram_count(
})
}
+/// Latest value of a `u64` observable gauge.
+fn u64_gauge_value(snapshots: &[ResourceMetrics], name: &str) -> Option<u64> {
+ latest_metric_value(snapshots, name, |data| match data {
+ AggregatedMetrics::U64(MetricData::Gauge(gauge)) => {
+ gauge.data_points().map(|point| point.value()).collect()
+ }
+ _ => Vec::new(),
+ })
+}
+
/// Decision with a fixed model and reasoning string.
struct StaticDecision {
model: String,
@@ -266,6 +288,30 @@ struct UsageClient {
usage: Usage,
}
+/// Client that returns a weak classifier verdict.
+struct ClassifierClient;
+
+#[async_trait]
+impl RoutedLlmClient for ClassifierClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, LlmClientError> {
+ let model = decision.selected_model().to_string();
+ let completion = if decision.is_routed_call() {
+ "routed response"
+ } else {
+ r#"{"recommended_route":"weak","p_solve":0.9,"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}"#
+ };
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(Some(model), completion)),
+ metadata: None,
+ })
+ }
+}
+
#[async_trait]
impl RoutedLlmClient for UsageClient {
async fn call(
@@ -359,9 +405,15 @@ fn find_span(spans: &[SpanRecord], name: &str, field: &str, value: &str) -> Span
#[tokio::test]
async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
let (store, exporter, provider) = telemetry();
const ALGO: &str = "obs-success-algo";
const MODEL: &str = "obs-success-model";
+ let before = flushed_metrics(exporter, provider);
+ let total_requests_before =
+ u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
+ let total_errors_before =
+ u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
let client = Arc::new(UsageClient {
usage: Usage {
@@ -391,41 +443,62 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
];
let token_attrs = [("algorithm", ALGO), ("selected_model", MODEL)];
assert_eq!(
- u64_counter_value(&snapshots, "libsy.runs", &run_attrs),
+ u64_counter_value(&snapshots, "switchyard.runs", &run_attrs),
Some(1)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.llm_calls", &call_attrs),
+ u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),
Some(1)
);
assert_eq!(
- f64_histogram_count(&snapshots, "libsy.run_duration_ms", &run_attrs),
+ f64_histogram_count(&snapshots, "switchyard.run_duration_ms", &run_attrs),
Some(1)
);
assert_eq!(
- f64_histogram_count(&snapshots, "libsy.llm_call_duration_ms", &call_attrs),
+ f64_histogram_count(&snapshots, "switchyard.llm_call_duration_ms", &call_attrs),
Some(1)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.input_tokens", &token_attrs),
+ u64_counter_value(&snapshots, "switchyard.input_tokens", &token_attrs),
Some(11)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.output_tokens", &token_attrs),
+ u64_counter_value(&snapshots, "switchyard.output_tokens", &token_attrs),
Some(7)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.total_tokens", &token_attrs),
+ u64_counter_value(&snapshots, "switchyard.total_tokens", &token_attrs),
Some(18)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.reasoning_tokens", &token_attrs),
+ u64_counter_value(&snapshots, "switchyard.reasoning_tokens", &token_attrs),
Some(2)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.decisions", &token_attrs),
+ u64_counter_value(&snapshots, "switchyard.decisions", &token_attrs),
+ Some(1)
+ );
+ let routed_attrs = [("model", MODEL)];
+ assert_eq!(
+ u64_counter_value(&snapshots, "switchyard.requests", &routed_attrs),
+ Some(1)
+ );
+ assert_eq!(
+ f64_histogram_count(
+ &snapshots,
+ "switchyard.model_call_latency_ms",
+ &routed_attrs
+ ),
Some(1)
);
+ assert_eq!(
+ u64_gauge_value(&snapshots, "switchyard.total_requests"),
+ Some(total_requests_before + 1)
+ );
+ assert_eq!(
+ u64_gauge_value(&snapshots, "switchyard.total_errors"),
+ Some(total_errors_before)
+ );
// Spans: one run span carrying the correlation ids and outcome, one child
// llm_call span carrying the selection, outcome, and token counts.
@@ -517,9 +590,15 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
#[tokio::test]
async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
let (store, exporter, provider) = telemetry();
const ALGO: &str = "obs-failure-algo";
const MODEL: &str = "obs-failure-model";
+ let before = flushed_metrics(exporter, provider);
+ let total_requests_before =
+ u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
+ let total_errors_before =
+ u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
// Client-less target: the call is offloaded and we fail it by hand.
let stream = algo(ALGO, MODEL, None).run_stream(
@@ -556,21 +635,33 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
("outcome", "error"),
];
assert_eq!(
- u64_counter_value(&snapshots, "libsy.runs", &run_attrs),
+ u64_counter_value(&snapshots, "switchyard.runs", &run_attrs),
Some(1)
);
assert_eq!(
- u64_counter_value(&snapshots, "libsy.llm_calls", &call_attrs),
+ u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),
Some(1)
);
assert_eq!(
u64_counter_value(
&snapshots,
- "libsy.input_tokens",
+ "switchyard.input_tokens",
&[("algorithm", ALGO), ("selected_model", MODEL)],
),
None
);
+ assert_eq!(
+ u64_counter_value(&snapshots, "switchyard.errors", &[("model", MODEL)]),
+ Some(1)
+ );
+ assert_eq!(
+ u64_gauge_value(&snapshots, "switchyard.total_requests"),
+ Some(total_requests_before + 1)
+ );
+ assert_eq!(
+ u64_gauge_value(&snapshots, "switchyard.total_errors"),
+ Some(total_errors_before + 1)
+ );
// Spans: both spans carry outcome=error and the propagated error text.
let spans = store.spans();
@@ -617,3 +708,83 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
);
Ok(())
}
+
+#[tokio::test]
+async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let (_store, exporter, provider) = telemetry();
+ let before = flushed_metrics(exporter, provider);
+ let total_requests_before =
+ u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
+
+ let client = Arc::new(ClassifierClient);
+ let target = |name: &str| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: Some(client.clone()),
+ };
+ let targets = LlmTargetSet::new(vec![target("weak"), target("strong")]);
+ let weak = targets.get_target("weak")?;
+ let strong = targets.get_target("strong")?;
+ let router = Arc::new(FallThrough::new(targets).with_classifier(Arc::new(
+ LlmTaskClassifier::new(target("classifier"), weak, strong, 0.5)?,
+ )));
+
+ let (trace, _response) = router
+ .run(
+ Context::<SharedState>::default(),
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "classify this"),
+ raw_request: None,
+ metadata: None,
+ },
+ )
+ .await?;
+
+ assert_eq!(
+ trace.last().and_then(|decision| decision.routing_tier()),
+ Some("weak")
+ );
+
+ let snapshots = flushed_metrics(exporter, provider);
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ "switchyard.llm_calls",
+ &[
+ ("algorithm", ""),
+ ("selected_model", "classifier"),
+ ("outcome", "ok"),
+ ],
+ ),
+ Some(1)
+ );
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ "switchyard.requests",
+ &[("model", "weak"), ("tier", "weak")],
+ ),
+ Some(1)
+ );
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ "switchyard.requests",
+ &[("model", "classifier")],
+ ),
+ None
+ );
+ assert_eq!(
+ f64_histogram_count(
+ &snapshots,
+ "switchyard.model_call_latency_ms",
+ &[("model", "classifier")],
+ ),
+ None
+ );
+ assert_eq!(
+ u64_gauge_value(&snapshots, "switchyard.total_requests"),
+ Some(total_requests_before + 1)
+ );
+ Ok(())
+}
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index c04cb03e2..6e546054f 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -113,6 +113,14 @@ pub enum LlmClientError {
pub trait Decision: Send + Sync {
/// The model this decision selected (e.g. the routed target's name).
fn selected_model(&self) -> &str;
+ /// Stable routing tier for the selected model, when the algorithm provides one.
+ fn routing_tier(&self) -> Option<&str> {
+ None
+ }
+ /// Whether this is the final call selected to serve the request.
+ fn is_routed_call(&self) -> bool {
+ true
+ }
/// A human-readable explanation of the decision, for logs and traces.
fn reasoning(&self) -> Option<&str>;
/// Downcast handle: a consumer that knows the algorithm can recover the
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 4aa8fb47d..f37767a53 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -64,4 +64,21 @@ never contains the secret itself. If omitted, the client sends no authentication
Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them
for equal weighting. The optional `seed` reproduces the selection sequence for the same call order.
+## Metrics
+
+`GET /metrics` exposes Prometheus text from the server's process-wide OpenTelemetry provider.
+Routed-call compatibility metrics are:
+
+| Metric | Type | Labels | Meaning |
+|---|---|---|---|
+| `switchyard_build_info` | gauge | `version` | Constant `1` for this server version |
+| `switchyard_total_requests` | gauge | none | Successful and failed final routed calls |
+| `switchyard_total_errors` | gauge | none | Failed final routed calls |
+| `switchyard_requests_total` | counter | `model`, optional `tier` | Successful final routed calls |
+| `switchyard_errors_total` | counter | `model`, optional `tier` | Failed final routed calls |
+| `switchyard_model_call_latency_ms` | histogram | `model`, optional `tier` | Successful final routed-call latency |
+
+The `tier` label is `strong` or `weak` for a distinguishable built-in LLM-classifier decision and
+is omitted for untiered algorithms. Classifier calls are excluded from these families.
+
See [CONFIGURATION.md](CONFIGURATION.md) to add an LLM client, target, or algorithm.
diff --git a/crates/switchyard-server/src/main.rs b/crates/switchyard-server/src/main.rs
index e111f6fcf..51c44a4be 100644
--- a/crates/switchyard-server/src/main.rs
+++ b/crates/switchyard-server/src/main.rs
@@ -10,6 +10,7 @@ use tracing_subscriber::EnvFilter;
mod cli;
const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";
+const OPENTELEMETRY_LOG_FILTER: &str = "opentelemetry=warn";
#[tokio::main(flavor = "multi_thread")]
async fn main() -> ExitCode {
@@ -27,8 +28,9 @@ async fn main() -> ExitCode {
}
fn init_logging() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
- let filter =
- EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER));
+ let filter = EnvFilter::try_from_default_env()
+ .unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER))
+ .add_directive(OPENTELEMETRY_LOG_FILTER.parse()?);
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_ansi(false)
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
index de76d6221..7b75233c6 100644
--- a/crates/switchyard-server/src/metrics.rs
+++ b/crates/switchyard-server/src/metrics.rs
@@ -1,11 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Process-wide Prometheus export for libsy's OpenTelemetry metrics.
+//! Process-wide Prometheus export for Switchyard's OpenTelemetry metrics.
use std::sync::OnceLock;
-use opentelemetry::global;
+use opentelemetry::{global, KeyValue};
use opentelemetry_sdk::metrics::SdkMeterProvider;
use prometheus::{Encoder, Registry, TextEncoder};
@@ -34,6 +34,11 @@ fn initialize() -> Result<Metrics, String> {
.map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;
let provider = SdkMeterProvider::builder().with_reader(exporter).build();
global::set_meter_provider(provider.clone());
+ libsy::initialize_metrics();
+ global::meter("switchyard")
+ .u64_gauge("switchyard.build_info")
+ .build()
+ .record(1, &[KeyValue::new("version", env!("CARGO_PKG_VERSION"))]);
Ok(Metrics {
registry,
_provider: provider,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 54aad67f6..24dac9ef9 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -32,6 +32,7 @@ type TestError = Box<dyn Error + Send + Sync>;
type TestResult<T = ()> = Result<T, TestError>;
const ROUTE_MODEL: &str = "switchyard/random";
+const VERSION: &str = env!("CARGO_PKG_VERSION");
struct MockUpstream {
base_url: String,
@@ -161,7 +162,7 @@ async fn test_app(routes: &[(&str, &[&str])]) -> TestResult<(MockUpstream, Route
}
#[tokio::test]
-async fn metrics_exposes_libsy_otel_instruments() -> TestResult {
+async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
let before = send(&app, "GET", "/metrics", None).await?;
@@ -189,10 +190,16 @@ async fn metrics_exposes_libsy_otel_instruments() -> TestResult {
let after = send(&app, "GET", "/metrics", None).await?;
let metrics = after.text()?;
for expected in [
- "# TYPE libsy_runs_total counter",
- "# TYPE libsy_llm_calls_total counter",
- "# TYPE libsy_run_duration_ms histogram",
- "# TYPE libsy_llm_call_duration_ms histogram",
+ "# TYPE switchyard_build_info gauge",
+ &format!("switchyard_build_info{{version=\"{VERSION}\""),
+ "# TYPE switchyard_total_requests gauge",
+ "# TYPE switchyard_total_errors gauge",
+ "# TYPE switchyard_requests_total counter",
+ "# TYPE switchyard_model_call_latency_ms histogram",
+ "# TYPE switchyard_runs_total counter",
+ "# TYPE switchyard_llm_calls_total counter",
+ "# TYPE switchyard_run_duration_ms histogram",
+ "# TYPE switchyard_llm_call_duration_ms histogram",
"algorithm=\"random\"",
"selected_model=\"model/a\"",
] {