Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): decide stage-router picker on score-as-probability (#546)

sabhatinas479cbcbb3931
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 61048d162..7ba82134b 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -994,7 +994,7 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res
Some(1)
);
for name in [
- "switchyard.stage_router.score",
+ "switchyard.stage_router.probability",
"switchyard.stage_router.confidence",
"switchyard.stage_router.severity",
"switchyard.stage_router.spinning",
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index f1aeefe65..f87c745d3 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -50,8 +50,8 @@ const SEVERITY_CRITICAL: f32 = 1.0;
/// Counts final stage-router choices by decision source and semantic target.
const ROUTING_DECISIONS_METRIC: &str = "switchyard.stage_router.routing_decisions";
-/// Distribution of the stage scorer's signed routing score.
-const SCORE_METRIC: &str = "switchyard.stage_router.score";
+/// Distribution of the picker's capable-tier probability.
+const PROBABILITY_METRIC: &str = "switchyard.stage_router.probability";
/// Distribution of the confidence used to resolve or defer a turn.
const CONFIDENCE_METRIC: &str = "switchyard.stage_router.confidence";
/// Distribution of detected tool-failure severity.
@@ -65,7 +65,7 @@ const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_in
// Histogram boundaries live with the instruments so every host exports the
// same stage-router distributions without duplicating algorithm knowledge.
-const SCORE_BUCKETS: &[f64] = &[-1.0, -0.75, -0.5, -0.25, 0.0, 0.25, 0.5, 0.75, 1.0];
+const PROBABILITY_BUCKETS: &[f64] = &[0.0, 0.125, 0.25, 0.375, 0.5, 0.625, 0.75, 0.875, 1.0];
const UNIT_BUCKETS: &[f64] = &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0];
/// The two tiers a turn can route to.
@@ -246,6 +246,13 @@ pub struct ScoreResult {
pub confidence: f64,
}
+impl ScoreResult {
+ /// `score` remapped from `(-1, +1)` onto a `(0, 1)` capable-tier probability.
+ fn probability(self) -> f64 {
+ (self.score + 1.0) / 2.0
+ }
+}
+
/// The two-axis feature view of a single [`ToolSignals`].
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct CodingAgentDimensions {
@@ -269,16 +276,17 @@ pub enum PickOutcome {
tier: Tier,
/// What produced it.
source: DecisionSource,
- /// Signed scorer value (`0.0` for override / tests-passed).
- score: f64,
+ /// Capable-tier probability in `(0, 1)`; `0.5` (neutral) for the
+ /// hard-rule paths, which bypass the scorer.
+ probability: f64,
/// Scorer confidence (`None` where the scorer did not run).
confidence: Option<f64>,
},
/// The scorer was below threshold. The caller runs its classifier; if it
/// has none (or it fails), fall open to `default_tier`.
ConsultClassifier {
- /// Signed scorer value, for logging.
- score: f64,
+ /// Capable-tier probability, for logging.
+ probability: f64,
/// Scorer confidence, for logging.
confidence: f64,
/// Tier to fall open to when no classifier resolves it.
@@ -324,18 +332,26 @@ pub(crate) fn record_routing_decision(source: DecisionSource, target_name: &str)
/// Records the scorer's bounded inputs and output as six explicit histograms.
fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {
- let (score, confidence) = match outcome {
+ let (probability, confidence) = match outcome {
PickOutcome::Resolved {
- score, confidence, ..
- } => (*score, confidence.unwrap_or(score.abs())),
+ probability,
+ confidence,
+ ..
+ } => (
+ *probability,
+ // Distance from 0.5, doubled onto [0, 1] — probability's `score.abs()`.
+ confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs()),
+ ),
PickOutcome::ConsultClassifier {
- score, confidence, ..
- } => (*score, *confidence),
+ probability,
+ confidence,
+ ..
+ } => (*probability, *confidence),
};
let dimensions = dimensions_from_signal(signal);
let meter = meter();
for (name, value, boundaries) in [
- (SCORE_METRIC, score, SCORE_BUCKETS),
+ (PROBABILITY_METRIC, probability, PROBABILITY_BUCKETS),
(CONFIDENCE_METRIC, confidence, UNIT_BUCKETS),
(SEVERITY_METRIC, dimensions.severity, UNIT_BUCKETS),
(SPINNING_METRIC, dimensions.spinning, UNIT_BUCKETS),
@@ -413,19 +429,21 @@ fn should_deescalate(signal: &ToolSignals) -> bool {
pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f64) -> PickOutcome {
// 1. Escalate — a hard reason to go capable, ahead of everything else.
if should_escalate(signal) {
- return resolved(Tier::Capable, DecisionSource::Override, 0.0, Some(1.0));
+ return resolved(Tier::Capable, DecisionSource::Override, 0.5, Some(1.0));
}
// 2. De-escalate — a hard reason to go cheap (the turn is winding down).
if should_deescalate(signal) {
- return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.0, None);
+ return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.5, None);
}
- // 3. Scorer — no hard reason either way, so weigh error vs production. If
- // confident enough, follow the sign: positive → capable, negative → efficient.
+ // 3. Scorer — no hard reason either way, so weigh error vs production.
+ // Resolve outside the closed ambiguous band [0.5 - t/2, 0.5 + t/2].
let scored = score_signal(signal);
- if scored.confidence >= confidence_threshold {
- let tier = if scored.score > 0.0 {
+ let probability = scored.probability();
+ let half_threshold = confidence_threshold / 2.0;
+ if probability > 0.5 + half_threshold || probability < 0.5 - half_threshold {
+ let tier = if probability > 0.5 {
Tier::Capable
} else {
Tier::Efficient
@@ -433,7 +451,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f
return resolved(
tier,
DecisionSource::Dimensions,
- scored.score,
+ probability,
Some(scored.confidence),
);
}
@@ -441,7 +459,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f
// 4. Fall open — the signals didn't corroborate enough to be sure. Hand off
// to the caller's classifier; with none, land on the picker's default.
PickOutcome::ConsultClassifier {
- score: scored.score,
+ probability,
confidence: scored.confidence,
default_tier: mode.default_tier(),
}
@@ -451,13 +469,13 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f
fn resolved(
tier: Tier,
source: DecisionSource,
- score: f64,
+ probability: f64,
confidence: Option<f64>,
) -> PickOutcome {
PickOutcome::Resolved {
tier,
source,
- score,
+ probability,
confidence,
}
}
@@ -606,8 +624,8 @@ impl Classifier<State> for StageClassifier {
PickOutcome::Resolved {
tier,
source,
- score,
- ..
+ probability,
+ confidence,
} => {
let target = self.targets.name(tier);
record_decision_source(state, source);
@@ -616,7 +634,9 @@ impl Classifier<State> for StageClassifier {
// is the only branch whose tier the signals actually chose — an
// ambiguous turn is decided further down the cascade.
self.apply_handoff_note(request, tier, source);
- let conf = score.abs();
+ // Prefer pick_tier's own confidence (e.g. 1.0 for an Override)
+ // over re-deriving it from the neutral 0.5 placeholder.
+ let conf = confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs());
Ok((
Classification::Scores(vec![Score {
target: target.clone(),
@@ -689,6 +709,20 @@ mod tests {
);
}
+ #[test]
+ fn pick_tier_treats_the_exact_threshold_score_as_ambiguous() {
+ // score == threshold maps to p == 0.5 + t/2 exactly, the closed
+ // ambiguous-band endpoint — a deliberate boundary reclassification
+ // from the old `confidence >= threshold` framing.
+ let mut signal = signal_from(json!([{"role": "user", "content": "hi"}]));
+ signal.severity = HARD_SEVERITY as f32;
+ let threshold = score_signal(&signal).confidence;
+ assert!(matches!(
+ pick_tier(&signal, PickerMode::EfficientFirst, threshold),
+ PickOutcome::ConsultClassifier { .. }
+ ));
+ }
+
#[test]
fn the_picker_mode_names_the_tier_an_undecided_turn_falls_back_to() {
assert_eq!(PickerMode::CapableFirst.default_tier(), Tier::Capable);
@@ -752,6 +786,8 @@ mod tests {
Classification::Scores(scores) => {
assert_eq!(scores.len(), 1);
assert_eq!(scores[0].target, "strong");
+ // An override is maximally certain, not `score.abs() == 0.0`.
+ assert_eq!(scores[0].confidence, 1.0);
}
_ => panic!("expected a definite classification"),
}
diff --git a/crates/switchyard-server/src/stats/algorithms/stage_router.rs b/crates/switchyard-server/src/stats/algorithms/stage_router.rs
index 9b3fc113f..4f5b9e70c 100644
--- a/crates/switchyard-server/src/stats/algorithms/stage_router.rs
+++ b/crates/switchyard-server/src/stats/algorithms/stage_router.rs
@@ -9,7 +9,7 @@ use prometheus::proto::{Metric, MetricFamily};
use serde::Serialize;
const ROUTING_DECISIONS_METRIC: &str = "switchyard_stage_router_routing_decisions_total";
-const SCORE_METRIC: &str = "switchyard_stage_router_score";
+const PROBABILITY_METRIC: &str = "switchyard_stage_router_probability";
const CONFIDENCE_METRIC: &str = "switchyard_stage_router_confidence";
const SEVERITY_METRIC: &str = "switchyard_stage_router_severity";
const SPINNING_METRIC: &str = "switchyard_stage_router_spinning";
@@ -19,7 +19,7 @@ const PRODUCTION_INTENSITY_METRIC: &str = "switchyard_stage_router_production_in
#[derive(Clone, Debug, Default)]
pub(super) struct StageRouterCumulative {
decisions: BTreeMap<DecisionKey, u64>,
- score: HistogramTotal,
+ probability: HistogramTotal,
confidence: HistogramTotal,
severity: HistogramTotal,
spinning: HistogramTotal,
@@ -56,7 +56,7 @@ pub(crate) struct DecisionStatsSnapshot {
/// Stage scorer output and input-dimension summaries.
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub(crate) struct ScoringStatsSnapshot {
- pub score: MetricSummary,
+ pub probability: MetricSummary,
pub confidence: MetricSummary,
pub dimensions: DimensionStatsSnapshot,
}
@@ -81,7 +81,7 @@ impl StageRouterCumulative {
pub(super) fn collect(families: &[MetricFamily]) -> Self {
Self {
decisions: collect_decisions(families),
- score: collect_histogram(families, SCORE_METRIC),
+ probability: collect_histogram(families, PROBABILITY_METRIC),
confidence: collect_histogram(families, CONFIDENCE_METRIC),
severity: collect_histogram(families, SEVERITY_METRIC),
spinning: collect_histogram(families, SPINNING_METRIC),
@@ -104,7 +104,7 @@ impl StageRouterCumulative {
StageRouterStatsSnapshot {
routing_decisions,
scoring: ScoringStatsSnapshot {
- score: self.score.delta(baseline.score),
+ probability: self.probability.delta(baseline.probability),
confidence: self.confidence.delta(baseline.confidence),
dimensions: DimensionStatsSnapshot {
severity: self.severity.delta(baseline.severity),
@@ -231,9 +231,9 @@ mod tests {
KeyValue::new("target_name", "model/capable"),
],
);
- for value in [0.5, -0.25] {
+ for value in [0.5, 0.25] {
meter
- .f64_histogram("switchyard.stage_router.score")
+ .f64_histogram("switchyard.stage_router.probability")
.build()
.record(value, &[]);
}
@@ -251,8 +251,8 @@ mod tests {
assert_eq!(dimensions.total, 3);
assert_eq!(dimensions.targets["model/efficient"], 2);
assert_eq!(dimensions.targets["model/capable"], 1);
- assert_eq!(stage.scoring.score.count, 2);
- assert_eq!(stage.scoring.score.mean, 0.125);
+ assert_eq!(stage.scoring.probability.count, 2);
+ assert_eq!(stage.scoring.probability.mean, 0.375);
assert_eq!(stage.scoring.confidence.mean, 0.75);
stats.reset();