Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): emit stage router metrics directly (#342)
nachiketb-nvidiabced06f3bee7
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 7c7865120..340c0a240 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -34,11 +34,12 @@ use tracing_subscriber::registry::LookupSpan;use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,- RoutedRequest, Step, TaskClassifierConfig,+ PickerMode, RoutedRequest, StageRouter, StageRouterConfig, Step, TaskClassifierConfig,};use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};use switchyard_protocol::{- Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,+ ContentBlock, Context, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response,+ Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,};use switchyard_protocol::{LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,@@ -826,6 +827,82 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lOk(())}+#[tokio::test]+async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Result<()> {+ let _guard = serialize_test().lock().await;+ let (_, exporter, provider, _, _) = telemetry();+ const STRONG: &str = "obs-stage-strong";+ const WEAK: &str = "obs-stage-weak";+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ };+ let algorithm = Arc::new(StageRouter::new(+ target(STRONG),+ target(WEAK),+ StageRouterConfig::new(PickerMode::EfficientFirst, 0.5),+ )?) as Arc<dyn Algorithm>;+ let request = Request {+ llm_request: LlmRequest {+ model: Some("auto".to_string()),+ messages: vec![+ Message::text(Role::User, "fix the build"),+ Message {+ role: Role::Assistant,+ content: vec![ContentBlock::ToolCall(ToolCall {+ id: "call_1".to_string(),+ name: "Bash".to_string(),+ arguments: json!({"command": "cargo test"}),+ })],+ },+ Message {+ role: Role::Tool,+ content: vec![ContentBlock::ToolResult(ToolResult {+ tool_call_id: "call_1".to_string(),+ content: vec![ContentBlock::Text {+ text: "fatal runtime error: out of memory".to_string(),+ }],+ is_error: Some(true),+ })],+ },+ ],+ ..LlmRequest::default()+ },+ raw_request: None,+ metadata: Some(Metadata {+ wire_format: Some(WireFormat::OpenAiChat),+ session_id: Some("obs-stage-session".to_string()),+ ..Metadata::default()+ }),+ };+ let client = Arc::new(UsageClient {+ usage: Usage::default(),+ }) as Arc<dyn RoutedLlmClient>;++ let (trace, _) = run(algorithm, client, request).await?;+ assert_eq!(trace[0].selected_model_id(), STRONG);++ let snapshots = flushed_metrics(exporter, provider);+ assert_eq!(+ u64_counter_value(+ &snapshots,+ "switchyard.stage_router.routing_decisions",+ &[("decision_source", "override"), ("target_name", STRONG)],+ ),+ Some(1)+ );+ for name in [+ "switchyard.stage_router.score",+ "switchyard.stage_router.confidence",+ "switchyard.stage_router.severity",+ "switchyard.stage_router.spinning",+ "switchyard.stage_router.exploring",+ "switchyard.stage_router.production_intensity",+ ] {+ assert_eq!(f64_histogram_count(&snapshots, name, &[]), Some(1));+ }+ Ok(())+}+#[tokio::test]async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex 5091d1d3f..e25f06b37 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -22,7 +22,7 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConuse super::util::prompts::{SystemPromptProcessor, TargetPrompts};use super::util::stage::{DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets,- record_decision_source,+ record_decision_source, record_routing_decision,};use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet};@@ -57,8 +57,9 @@ impl Classifier<State> for SourceStamp {) -> Result<(Classification, Option<Response>)> {let (classification, served) = self.inner.score(state, request, driver).await?;// An abstaining classifier passes the turn on, so it is not its to claim.- if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) {+ if let Some(winner) = classification.argmax(false)? {record_decision_source(state, self.source);+ record_routing_decision(self.source, &winner.target);}Ok((classification, served))}@@ -183,7 +184,6 @@ fn build_route(let signals = ToolSignalProcessor {recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW),};-let target_set = LlmTargetSet::new(vec![capable.clone(), efficient.clone()]);let mut router = FallThrough::<State>::new_with_state(target_set).with_name(STAGE_ROUTER)diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex 3105c1102..32fcf8f52 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -18,6 +18,7 @@//! how much corroboration a decisive escalation needs (see [`score_signal`]).use async_trait::async_trait;+use opentelemetry::KeyValue;use serde::Deserialize;use super::prompts;@@ -26,6 +27,7 @@ use crate::Result;use crate::core::algorithm::Driver;use crate::core::classifier::{Classification, Classifier, Score};use crate::core::state::{State, StateValue};+use crate::observability::meter;use switchyard_protocol::Request;/// Turn depth below which stall signals stay quiet — early no-write turns are@@ -44,6 +46,22 @@ const SIGNAL_UNIT: f64 = 0.10;/// Critical severity forces the capable tier regardless of the scorer.const SEVERITY_CRITICAL: f32 = 1.0;+/// Counts final stage-router choices by decision source and semantic target.+const ROUTING_DECISIONS_METRIC: &str = "switchyard.stage_router.routing_decisions";++/// Distribution of the stage scorer's signed routing score.+const SCORE_METRIC: &str = "switchyard.stage_router.score";+/// Distribution of the confidence used to resolve or defer a turn.+const CONFIDENCE_METRIC: &str = "switchyard.stage_router.confidence";+/// Distribution of detected tool-failure severity.+const SEVERITY_METRIC: &str = "switchyard.stage_router.severity";+/// Distribution of repeated unproductive tool activity.+const SPINNING_METRIC: &str = "switchyard.stage_router.spinning";+/// Distribution of exploratory tool activity.+const EXPLORING_METRIC: &str = "switchyard.stage_router.exploring";+/// Distribution of production-oriented tool activity.+const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_intensity";+/// The two tiers a turn can route to.#[derive(Clone, Copy, Debug, Eq, PartialEq)]pub enum Tier {@@ -247,6 +265,41 @@ pub fn dimensions_from_signal(signal: &ToolSignals) -> CodingAgentDimensions {}}+/// Records one stage-router decision with bounded labels.+pub(crate) fn record_routing_decision(source: DecisionSource, target_name: &str) {+ meter().u64_counter(ROUTING_DECISIONS_METRIC).build().add(+ 1,+ &[+ KeyValue::new("decision_source", source.as_str()),+ KeyValue::new("target_name", target_name.to_string()),+ ],+ );+}++/// Records the scorer's bounded inputs and output as six explicit histograms.+fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {+ let (score, confidence) = match outcome {+ PickOutcome::Resolved {+ score, confidence, ..+ } => (*score, confidence.unwrap_or(score.abs())),+ PickOutcome::ConsultClassifier {+ score, confidence, ..+ } => (*score, *confidence),+ };+ let dimensions = dimensions_from_signal(signal);+ let meter = meter();+ for (name, value) in [+ (SCORE_METRIC, score),+ (CONFIDENCE_METRIC, confidence),+ (SEVERITY_METRIC, dimensions.severity),+ (SPINNING_METRIC, dimensions.spinning),+ (EXPLORING_METRIC, dimensions.exploring),+ (PRODUCTION_INTENSITY_METRIC, dimensions.production_intensity),+ ] {+ meter.f64_histogram(name).build().record(value, &[]);+ }+}+/// Score a signal: weighted sum of the dimensions, `tanh`-squashed.////// The raw sum is small — one maxed signal is `±0.10`, two corroborating@@ -494,6 +547,7 @@ impl Classifier<State> for StageClassifier {};let outcome = pick_tier(signal, self.mode, self.confidence_threshold);+ record_score_metrics(signal, &outcome);match outcome {PickOutcome::Resolved {tier,@@ -503,6 +557,7 @@ impl Classifier<State> for StageClassifier {} => {let target = self.targets.name(tier);record_decision_source(state, source);+ record_routing_decision(source, target);// Only a resolved turn routes on this classifier's target, so it// is the only branch whose tier the signals actually chose — an// ambiguous turn is decided further down the cascade.diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 301208114..c86e5c43c 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -68,7 +68,7 @@ pub fn algorithm_label(ctx: &Context) -> &str {}/// The `libsy`-scoped meter from the globally installed provider.-fn meter() -> Meter {+pub(crate) fn meter() -> Meter {global::meter(METRICS_SCOPE)}