Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): let stage's fall-open tier be set at runtime (#518)
ryan-lempka2b5273049a0f
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex 289aa08b5..fbec4b007 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -10,23 +10,23 @@//!//! Signals do not decide every turn. An under-threshold turn abstains and falls//! through to the optional [`LlmTaskClassifier`] — the capability route's judge,-//! joined in unchanged — and then to the picker's default tier. The judge is-//! asked per turn and its verdict is never pinned to the session.+//! joined in unchanged — and then to the picker's default tier, or to an+//! override a decider ahead of stage set in its place.//!use std::sync::Arc;use async_trait::async_trait;-use super::fall_through::{DefaultTarget, FallThrough};+use super::fall_through::FallThrough;use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig};use super::util::prompts::{SystemPromptProcessor, TargetPrompts};use super::util::stage::{- DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets,- record_decision_source, record_routing_decision,+ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, Tier,+ fall_open_tier, record_decision_source, record_routing_decision,};use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};use crate::core::algorithm::{Algorithm, Driver};-use crate::core::classifier::{Classification, Classifier};+use crate::core::classifier::{Classification, Classifier, Score};use crate::core::state::State;use crate::{LibsyError, Result};use switchyard_protocol::{ModelId, Request, Response};@@ -65,6 +65,32 @@ impl Classifier<State> for SourceStamp {}}+/// Closes the cascade at zero confidence: a fallback, not a judgement.+struct FallOpen {+ targets: StageTargets,+ default_tier: Tier,+}++#[async_trait]+impl Classifier<State> for FallOpen {+ async fn score(+ &self,+ state: &mut State,+ _request: &mut Request,+ _driver: Option<&Driver>,+ ) -> Result<(Classification, Option<Response>)> {+ let tier = fall_open_tier(state).unwrap_or(self.default_tier);+ let target = self.targets.name(tier).clone();+ Ok((+ Classification::Scores(vec![Score {+ target,+ confidence: 0.0,+ }]),+ None,+ ))+ }+}+/// The capability judge a stage router falls through to.pub struct LlmFallback {/// Target the judge model is called through. It is not a routing@@ -165,9 +191,11 @@ fn build_route(// The tiers are a fixed pair; their targets are whatever the deployment calls// them, and the classifier scores onto those names.let targets = StageTargets::new(capable.clone(), efficient.clone());- // The picker's mode fixes the fallback tier up front, so the terminal- // classifier is a constant rather than a per-turn lookup.- let fall_open = targets.name(config.mode.default_tier()).to_string();+ let default_tier = config.mode.default_tier();+ let fall_open = FallOpen {+ targets: targets.clone(),+ default_tier,+ };let mut classifier = StageClassifier::new(targets, config.mode, config.confidence_threshold);if let Some(notes) = config.handoff_notes {@@ -194,10 +222,9 @@ fn build_route(source: DecisionSource::LlmClassifier,}));}- // Nothing behind this, so the turn lands on the picker's default tier —- // including when the judge could not tell.+ // Nothing behind this, so no turn is left unrouted.router = router.with_classifier(Arc::new(SourceStamp {- inner: Arc::new(DefaultTarget::new(fall_open)),+ inner: Arc::new(fall_open),source: DecisionSource::FallOpen,}));// Runs on the post-decision hook, so it applies to the target the cascade@@ -219,8 +246,8 @@ mod tests {};use super::*;- use crate::algorithms::util::stage::DECISION_SOURCE_KEY;- use crate::core::classifier::Score;+ use crate::algorithms::util::stage::{DECISION_SOURCE_KEY, clear_fall_open, set_fall_open};+ use crate::core::processor::{Event, Processor};use crate::core::state::StateValue;use crate::core::testing::{Serve, reply, test_drive};use switchyard_protocol::{Metadata, Response};@@ -465,6 +492,64 @@ mod tests {}}+ /// Stands in for a decider ahead of stage: sets the override once, clears it+ /// once, and leaves the turns between alone.+ #[derive(Default)]+ struct TierDecider {+ requests: Mutex<u32>,+ }++ #[async_trait]+ impl Processor<State> for TierDecider {+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {+ if matches!(event, Event::Request(_)) {+ let mut requests = self.requests.lock();+ *requests += 1;+ match *requests {+ 1 => set_fall_open(state, Tier::Efficient),+ 4 => clear_fall_open(state),+ _ => {}+ }+ }+ Ok(())+ }+ }++ #[tokio::test]+ async fn an_override_replaces_the_picker_default_and_leaves_the_signals_alone() -> Result<()> {+ let recorder = Arc::new(Recorder::default());+ // The picker would fall open to "strong"; the override says "weak".+ let config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5);+ let route: Arc<dyn Algorithm> = Arc::new(+ build_route(ModelId::from("strong"), ModelId::from("weak"), config)?+ .with_processor(Arc::new(TierDecider::default())),+ );++ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;+ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;+ test_drive(route.clone(), turn_request(true), recorder.serve()).await?;+ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;++ let routed = recorder.routed();+ assert_eq!(+ routed[0].target, "weak",+ "an undecided turn takes the override"+ );+ assert_eq!(+ routed[1].target, "weak",+ "which outlives the turn that set it"+ );+ assert_eq!(+ routed[2].target, "strong",+ "a critical failure still reaches the signals"+ );+ assert_eq!(+ routed[3].target, "strong",+ "clearing restores the picker default"+ );+ Ok(())+ }+#[tokio::test]async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> {let recorder = Arc::new(Recorder::default());diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex 3f16334fc..a1c0560bb 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -81,13 +81,22 @@ impl Tier {/// Stable label for stats and the [`routing_tier`](Classifier::routing_tier)/// hook, independent of what the tiers' targets are called. These are the/// strings the capability route reports too, so a deployment running both- /// sees one tier vocabulary.+ /// sees one tier vocabulary. Also the encoding a default-tier override is+ /// stored under, so changing these strings invalidates one.fn label(self) -> &'static str {match self {Self::Capable => "strong",Self::Efficient => "weak",}}++ fn from_label(label: &str) -> Option<Self> {+ match label {+ "strong" => Some(Self::Capable),+ "weak" => Some(Self::Efficient),+ _ => None,+ }+ }}/// The targets a stage router's two tiers route to.@@ -153,6 +162,32 @@ impl PickerMode {/// `State.extra` key under which the turn's [`DecisionSource`] is recorded.pub const DECISION_SOURCE_KEY: &str = "decision_source";+/// `State.extra` key under which a default-tier override is recorded.+const FALL_OPEN_KEY: &str = "fall_open";++/// Overrides the default tier undecided turns fall open to, until+/// [`clear_fall_open`] restores the picker's. Held in session state, so it+/// survives later requests only when they carry a session id.+pub fn set_fall_open(state: &mut State, tier: Tier) {+ state.extra.insert(+ FALL_OPEN_KEY.to_string(),+ StateValue::String(tier.label().to_string()),+ );+}++/// Restores the picker's default tier.+pub fn clear_fall_open(state: &mut State) {+ state.extra.remove(FALL_OPEN_KEY);+}++/// The default-tier override for this session, if any.+pub(crate) fn fall_open_tier(state: &State) -> Option<Tier> {+ match state.extra.get(FALL_OPEN_KEY) {+ Some(StateValue::String(label)) => Tier::from_label(label),+ _ => None,+ }+}+/// Record which component decided the turn.pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) {state.extra.insert(diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 8ae823689..47a26c083 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -37,8 +37,8 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals};// core (scorer, picker, and the `StageClassifier`).pub use algorithms::util::stage::{CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome,- PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal,- pick_tier, score_signal,+ PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, clear_fall_open,+ dimensions_from_signal, pick_tier, score_signal, set_fall_open,};mod observability;