Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(libsy): decide stage-router picker on score-as-probability (#546)
sabhatinas479cbcbb3931
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 61048d162..7ba82134b 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -994,7 +994,7 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::ResSome(1));for name in [- "switchyard.stage_router.score",+ "switchyard.stage_router.probability","switchyard.stage_router.confidence","switchyard.stage_router.severity","switchyard.stage_router.spinning",diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex f1aeefe65..f87c745d3 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -50,8 +50,8 @@ const SEVERITY_CRITICAL: f32 = 1.0;/// Counts final stage-router choices by decision source and semantic target.const ROUTING_DECISIONS_METRIC: &str = "switchyard.stage_router.routing_decisions";-/// Distribution of the stage scorer's signed routing score.-const SCORE_METRIC: &str = "switchyard.stage_router.score";+/// Distribution of the picker's capable-tier probability.+const PROBABILITY_METRIC: &str = "switchyard.stage_router.probability";/// Distribution of the confidence used to resolve or defer a turn.const CONFIDENCE_METRIC: &str = "switchyard.stage_router.confidence";/// Distribution of detected tool-failure severity.@@ -65,7 +65,7 @@ const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_in// Histogram boundaries live with the instruments so every host exports the// same stage-router distributions without duplicating algorithm knowledge.-const SCORE_BUCKETS: &[f64] = &[-1.0, -0.75, -0.5, -0.25, 0.0, 0.25, 0.5, 0.75, 1.0];+const PROBABILITY_BUCKETS: &[f64] = &[0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0];const UNIT_BUCKETS: &[f64] = &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0];/// The two tiers a turn can route to.@@ -246,6 +246,13 @@ pub struct ScoreResult {pub confidence: f64,}+impl ScoreResult {+ /// `score` remapped from `(-1, +1)` onto a `(0, 1)` capable-tier probability.+ fn probability(self) -> f64 {+ (self.score + 1.0) / 2.0+ }+}+/// The two-axis feature view of a single [`ToolSignals`].#[derive(Clone, Copy, Debug, PartialEq)]pub struct CodingAgentDimensions {@@ -269,16 +276,17 @@ pub enum PickOutcome {tier: Tier,/// What produced it.source: DecisionSource,- /// Signed scorer value (`0.0` for override / tests-passed).- score: f64,+ /// Capable-tier probability in `(0, 1)`; `0.5` (neutral) for the+ /// hard-rule paths, which bypass the scorer.+ probability: f64,/// Scorer confidence (`None` where the scorer did not run).confidence: Option<f64>,},/// The scorer was below threshold. The caller runs its classifier; if it/// has none (or it fails), fall open to `default_tier`.ConsultClassifier {- /// Signed scorer value, for logging.- score: f64,+ /// Capable-tier probability, for logging.+ probability: f64,/// Scorer confidence, for logging.confidence: f64,/// Tier to fall open to when no classifier resolves it.@@ -324,18 +332,26 @@ pub(crate) fn record_routing_decision(source: DecisionSource, target_name: &str)/// Records the scorer's bounded inputs and output as six explicit histograms.fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {- let (score, confidence) = match outcome {+ let (probability, confidence) = match outcome {PickOutcome::Resolved {- score, confidence, ..- } => (*score, confidence.unwrap_or(score.abs())),+ probability,+ confidence,+ ..+ } => (+ *probability,+ // Distance from 0.5, doubled onto [0, 1] — probability's `score.abs()`.+ confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs()),+ ),PickOutcome::ConsultClassifier {- score, confidence, ..- } => (*score, *confidence),+ probability,+ confidence,+ ..+ } => (*probability, *confidence),};let dimensions = dimensions_from_signal(signal);let meter = meter();for (name, value, boundaries) in [- (SCORE_METRIC, score, SCORE_BUCKETS),+ (PROBABILITY_METRIC, probability, PROBABILITY_BUCKETS),(CONFIDENCE_METRIC, confidence, UNIT_BUCKETS),(SEVERITY_METRIC, dimensions.severity, UNIT_BUCKETS),(SPINNING_METRIC, dimensions.spinning, UNIT_BUCKETS),@@ -413,19 +429,21 @@ fn should_deescalate(signal: &ToolSignals) -> bool {pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f64) -> PickOutcome {// 1. Escalate — a hard reason to go capable, ahead of everything else.if should_escalate(signal) {- return resolved(Tier::Capable, DecisionSource::Override, 0.0, Some(1.0));+ return resolved(Tier::Capable, DecisionSource::Override, 0.5, Some(1.0));}// 2. De-escalate — a hard reason to go cheap (the turn is winding down).if should_deescalate(signal) {- return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.0, None);+ return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.5, None);}- // 3. Scorer — no hard reason either way, so weigh error vs production. If- // confident enough, follow the sign: positive → capable, negative → efficient.+ // 3. Scorer — no hard reason either way, so weigh error vs production.+ // Resolve outside the closed ambiguous band [0.5 - t/2, 0.5 + t/2].let scored = score_signal(signal);- if scored.confidence >= confidence_threshold {- let tier = if scored.score > 0.0 {+ let probability = scored.probability();+ let half_threshold = confidence_threshold / 2.0;+ if probability > 0.5 + half_threshold || probability < 0.5 - half_threshold {+ let tier = if probability > 0.5 {Tier::Capable} else {Tier::Efficient@@ -433,7 +451,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: freturn resolved(tier,DecisionSource::Dimensions,- scored.score,+ probability,Some(scored.confidence),);}@@ -441,7 +459,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f// 4. Fall open — the signals didn't corroborate enough to be sure. Hand off// to the caller's classifier; with none, land on the picker's default.PickOutcome::ConsultClassifier {- score: scored.score,+ probability,confidence: scored.confidence,default_tier: mode.default_tier(),}@@ -451,13 +469,13 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: ffn resolved(tier: Tier,source: DecisionSource,- score: f64,+ probability: f64,confidence: Option<f64>,) -> PickOutcome {PickOutcome::Resolved {tier,source,- score,+ probability,confidence,}}@@ -606,8 +624,8 @@ impl Classifier<State> for StageClassifier {PickOutcome::Resolved {tier,source,- score,- ..+ probability,+ confidence,} => {let target = self.targets.name(tier);record_decision_source(state, source);@@ -616,7 +634,9 @@ impl Classifier<State> for StageClassifier {// is the only branch whose tier the signals actually chose — an// ambiguous turn is decided further down the cascade.self.apply_handoff_note(request, tier, source);- let conf = score.abs();+ // Prefer pick_tier's own confidence (e.g. 1.0 for an Override)+ // over re-deriving it from the neutral 0.5 placeholder.+ let conf = confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs());Ok((Classification::Scores(vec![Score {target: target.clone(),@@ -689,6 +709,20 @@ mod tests {);}+ #[test]+ fn pick_tier_treats_the_exact_threshold_score_as_ambiguous() {+ // score == threshold maps to p == 0.5 + t/2 exactly, the closed+ // ambiguous-band endpoint — a deliberate boundary reclassification+ // from the old `confidence >= threshold` framing.+ let mut signal = signal_from(json!([{"role": "user", "content": "hi"}]));+ signal.severity = HARD_SEVERITY as f32;+ let threshold = score_signal(&signal).confidence;+ assert!(matches!(+ pick_tier(&signal, PickerMode::EfficientFirst, threshold),+ PickOutcome::ConsultClassifier { .. }+ ));+ }+#[test]fn the_picker_mode_names_the_tier_an_undecided_turn_falls_back_to() {assert_eq!(PickerMode::CapableFirst.default_tier(), Tier::Capable);@@ -752,6 +786,8 @@ mod tests {Classification::Scores(scores) => {assert_eq!(scores.len(), 1);assert_eq!(scores[0].target, "strong");+ // An override is maximally certain, not `score.abs() == 0.0`.+ assert_eq!(scores[0].confidence, 1.0);}_ => panic!("expected a definite classification"),}diff --git a/crates/switchyard-server/src/stats/algorithms/stage_router.rs b/crates/switchyard-server/src/stats/algorithms/stage_router.rsindex 9b3fc113f..4f5b9e70c 100644--- a/crates/switchyard-server/src/stats/algorithms/stage_router.rs+++ b/crates/switchyard-server/src/stats/algorithms/stage_router.rs@@ -9,7 +9,7 @@ use prometheus::proto::{Metric, MetricFamily};use serde::Serialize;const ROUTING_DECISIONS_METRIC: &str = "switchyard_stage_router_routing_decisions_total";-const SCORE_METRIC: &str = "switchyard_stage_router_score";+const PROBABILITY_METRIC: &str = "switchyard_stage_router_probability";const CONFIDENCE_METRIC: &str = "switchyard_stage_router_confidence";const SEVERITY_METRIC: &str = "switchyard_stage_router_severity";const SPINNING_METRIC: &str = "switchyard_stage_router_spinning";@@ -19,7 +19,7 @@ const PRODUCTION_INTENSITY_METRIC: &str = "switchyard_stage_router_production_in#[derive(Clone, Debug, Default)]pub(super) struct StageRouterCumulative {decisions: BTreeMap<DecisionKey, u64>,- score: HistogramTotal,+ probability: HistogramTotal,confidence: HistogramTotal,severity: HistogramTotal,spinning: HistogramTotal,@@ -56,7 +56,7 @@ pub(crate) struct DecisionStatsSnapshot {/// Stage scorer output and input-dimension summaries.#[derive(Clone, Debug, Default, PartialEq, Serialize)]pub(crate) struct ScoringStatsSnapshot {- pub score: MetricSummary,+ pub probability: MetricSummary,pub confidence: MetricSummary,pub dimensions: DimensionStatsSnapshot,}@@ -81,7 +81,7 @@ impl StageRouterCumulative {pub(super) fn collect(families: &[MetricFamily]) -> Self {Self {decisions: collect_decisions(families),- score: collect_histogram(families, SCORE_METRIC),+ probability: collect_histogram(families, PROBABILITY_METRIC),confidence: collect_histogram(families, CONFIDENCE_METRIC),severity: collect_histogram(families, SEVERITY_METRIC),spinning: collect_histogram(families, SPINNING_METRIC),@@ -104,7 +104,7 @@ impl StageRouterCumulative {StageRouterStatsSnapshot {routing_decisions,scoring: ScoringStatsSnapshot {- score: self.score.delta(baseline.score),+ probability: self.probability.delta(baseline.probability),confidence: self.confidence.delta(baseline.confidence),dimensions: DimensionStatsSnapshot {severity: self.severity.delta(baseline.severity),@@ -231,9 +231,9 @@ mod tests {KeyValue::new("target_name", "model/capable"),],);- for value in [0.5, -0.25] {+ for value in [0.5, 0.25] {meter- .f64_histogram("switchyard.stage_router.score")+ .f64_histogram("switchyard.stage_router.probability").build().record(value, &[]);}@@ -251,8 +251,8 @@ mod tests {assert_eq!(dimensions.total, 3);assert_eq!(dimensions.targets["model/efficient"], 2);assert_eq!(dimensions.targets["model/capable"], 1);- assert_eq!(stage.scoring.score.count, 2);- assert_eq!(stage.scoring.score.mean, 0.125);+ assert_eq!(stage.scoring.probability.count, 2);+ assert_eq!(stage.scoring.probability.mean, 0.375);assert_eq!(stage.scoring.confidence.mean, 0.75);stats.reset();