Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): emit stage router metrics directly (#342)

nachiketb-nvidiabced06f3bee7
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 7c7865120..340c0a240 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -34,11 +34,12 @@ use tracing_subscriber::registry::LookupSpan;
use switchyard_libsy::{
Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,
- RoutedRequest, Step, TaskClassifierConfig,
+ PickerMode, RoutedRequest, StageRouter, StageRouterConfig, Step, TaskClassifierConfig,
};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
use switchyard_protocol::{
- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,
+ ContentBlock, Context, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response,
+ Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,
};
use switchyard_protocol::{
LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,
@@ -826,6 +827,82 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Ok(())
}
+#[tokio::test]
+async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let (_, exporter, provider, _, _) = telemetry();
+ const STRONG: &str = "obs-stage-strong";
+ const WEAK: &str = "obs-stage-weak";
+ let target = |name: &str| LlmTarget {
+ semantic_name: name.to_string(),
+ };
+ let algorithm = Arc::new(StageRouter::new(
+ target(STRONG),
+ target(WEAK),
+ StageRouterConfig::new(PickerMode::EfficientFirst, 0.5),
+ )?) as Arc<dyn Algorithm>;
+ let request = Request {
+ llm_request: LlmRequest {
+ model: Some("auto".to_string()),
+ messages: vec![
+ Message::text(Role::User, "fix the build"),
+ Message {
+ role: Role::Assistant,
+ content: vec![ContentBlock::ToolCall(ToolCall {
+ id: "call_1".to_string(),
+ name: "Bash".to_string(),
+ arguments: json!({"command": "cargo test"}),
+ })],
+ },
+ Message {
+ role: Role::Tool,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "call_1".to_string(),
+ content: vec![ContentBlock::Text {
+ text: "fatal runtime error: out of memory".to_string(),
+ }],
+ is_error: Some(true),
+ })],
+ },
+ ],
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: Some(Metadata {
+ wire_format: Some(WireFormat::OpenAiChat),
+ session_id: Some("obs-stage-session".to_string()),
+ ..Metadata::default()
+ }),
+ };
+ let client = Arc::new(UsageClient {
+ usage: Usage::default(),
+ }) as Arc<dyn RoutedLlmClient>;
+
+ let (trace, _) = run(algorithm, client, request).await?;
+ assert_eq!(trace[0].selected_model_id(), STRONG);
+
+ let snapshots = flushed_metrics(exporter, provider);
+ assert_eq!(
+ u64_counter_value(
+ &snapshots,
+ "switchyard.stage_router.routing_decisions",
+ &[("decision_source", "override"), ("target_name", STRONG)],
+ ),
+ Some(1)
+ );
+ for name in [
+ "switchyard.stage_router.score",
+ "switchyard.stage_router.confidence",
+ "switchyard.stage_router.severity",
+ "switchyard.stage_router.spinning",
+ "switchyard.stage_router.exploring",
+ "switchyard.stage_router.production_intensity",
+ ] {
+ assert_eq!(f64_histogram_count(&snapshots, name, &[]), Some(1));
+ }
+ Ok(())
+}
+
#[tokio::test]
async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 5091d1d3f..e25f06b37 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -22,7 +22,7 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierCon
use super::util::prompts::{SystemPromptProcessor, TargetPrompts};
use super::util::stage::{
DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets,
- record_decision_source,
+ record_decision_source, record_routing_decision,
};
use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};
use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};
@@ -57,8 +57,9 @@ impl Classifier<State> for SourceStamp {
) -> Result<(Classification, Option<Response>)> {
let (classification, served) = self.inner.score(state, request, driver).await?;
// An abstaining classifier passes the turn on, so it is not its to claim.
- if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) {
+ if let Some(winner) = classification.argmax(false)? {
record_decision_source(state, self.source);
+ record_routing_decision(self.source, &winner.target);
}
Ok((classification, served))
}
@@ -183,7 +184,6 @@ fn build_route(
let signals = ToolSignalProcessor {
recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW),
};
-
let target_set = LlmTargetSet::new(vec![capable.clone(), efficient.clone()]);
let mut router = FallThrough::<State>::new_with_state(target_set)
.with_name(STAGE_ROUTER)
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 3105c1102..32fcf8f52 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -18,6 +18,7 @@
//! how much corroboration a decisive escalation needs (see [`score_signal`]).
use async_trait::async_trait;
+use opentelemetry::KeyValue;
use serde::Deserialize;
use super::prompts;
@@ -26,6 +27,7 @@ use crate::Result;
use crate::core::algorithm::Driver;
use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::{State, StateValue};
+use crate::observability::meter;
use switchyard_protocol::Request;
/// Turn depth below which stall signals stay quiet — early no-write turns are
@@ -44,6 +46,22 @@ const SIGNAL_UNIT: f64 = 0.10;
/// Critical severity forces the capable tier regardless of the scorer.
const SEVERITY_CRITICAL: f32 = 1.0;
+/// Counts final stage-router choices by decision source and semantic target.
+const ROUTING_DECISIONS_METRIC: &str = "switchyard.stage_router.routing_decisions";
+
+/// Distribution of the stage scorer's signed routing score.
+const SCORE_METRIC: &str = "switchyard.stage_router.score";
+/// Distribution of the confidence used to resolve or defer a turn.
+const CONFIDENCE_METRIC: &str = "switchyard.stage_router.confidence";
+/// Distribution of detected tool-failure severity.
+const SEVERITY_METRIC: &str = "switchyard.stage_router.severity";
+/// Distribution of repeated unproductive tool activity.
+const SPINNING_METRIC: &str = "switchyard.stage_router.spinning";
+/// Distribution of exploratory tool activity.
+const EXPLORING_METRIC: &str = "switchyard.stage_router.exploring";
+/// Distribution of production-oriented tool activity.
+const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_intensity";
+
/// The two tiers a turn can route to.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Tier {
@@ -247,6 +265,41 @@ pub fn dimensions_from_signal(signal: &ToolSignals) -> CodingAgentDimensions {
}
}
+/// Records one stage-router decision with bounded labels.
+pub(crate) fn record_routing_decision(source: DecisionSource, target_name: &str) {
+ meter().u64_counter(ROUTING_DECISIONS_METRIC).build().add(
+ 1,
+ &[
+ KeyValue::new("decision_source", source.as_str()),
+ KeyValue::new("target_name", target_name.to_string()),
+ ],
+ );
+}
+
+/// Records the scorer's bounded inputs and output as six explicit histograms.
+fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {
+ let (score, confidence) = match outcome {
+ PickOutcome::Resolved {
+ score, confidence, ..
+ } => (*score, confidence.unwrap_or(score.abs())),
+ PickOutcome::ConsultClassifier {
+ score, confidence, ..
+ } => (*score, *confidence),
+ };
+ let dimensions = dimensions_from_signal(signal);
+ let meter = meter();
+ for (name, value) in [
+ (SCORE_METRIC, score),
+ (CONFIDENCE_METRIC, confidence),
+ (SEVERITY_METRIC, dimensions.severity),
+ (SPINNING_METRIC, dimensions.spinning),
+ (EXPLORING_METRIC, dimensions.exploring),
+ (PRODUCTION_INTENSITY_METRIC, dimensions.production_intensity),
+ ] {
+ meter.f64_histogram(name).build().record(value, &[]);
+ }
+}
+
/// Score a signal: weighted sum of the dimensions, `tanh`-squashed.
///
/// The raw sum is small — one maxed signal is `±0.10`, two corroborating
@@ -494,6 +547,7 @@ impl Classifier<State> for StageClassifier {
};
let outcome = pick_tier(signal, self.mode, self.confidence_threshold);
+ record_score_metrics(signal, &outcome);
match outcome {
PickOutcome::Resolved {
tier,
@@ -503,6 +557,7 @@ impl Classifier<State> for StageClassifier {
} => {
let target = self.targets.name(tier);
record_decision_source(state, source);
+ record_routing_decision(source, target);
// Only a resolved turn routes on this classifier's target, so it
// is the only branch whose tier the signals actually chose — an
// ambiguous turn is decided further down the cascade.
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 301208114..c86e5c43c 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -68,7 +68,7 @@ pub fn algorithm_label(ctx: &Context) -> &str {
}
/// The `libsy`-scoped meter from the globally installed provider.
-fn meter() -> Meter {
+pub(crate) fn meter() -> Meter {
global::meter(METRICS_SCOPE)
}