Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): let stage's fall-open tier be set at runtime (#518)

ryan-lempka2b5273049a0f
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 289aa08b5..fbec4b007 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -10,23 +10,23 @@
//!
//! Signals do not decide every turn. An under-threshold turn abstains and falls
//! through to the optional [`LlmTaskClassifier`] — the capability route's judge,
-//! joined in unchanged — and then to the picker's default tier. The judge is
-//! asked per turn and its verdict is never pinned to the session.
+//! joined in unchanged — and then to the picker's default tier, or to an
+//! override a decider ahead of stage set in its place.
//!
use std::sync::Arc;
use async_trait::async_trait;
-use super::fall_through::{DefaultTarget, FallThrough};
+use super::fall_through::FallThrough;
use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig};
use super::util::prompts::{SystemPromptProcessor, TargetPrompts};
use super::util::stage::{
- DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets,
- record_decision_source, record_routing_decision,
+ DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, Tier,
+ fall_open_tier, record_decision_source, record_routing_decision,
};
use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};
use crate::core::algorithm::{Algorithm, Driver};
-use crate::core::classifier::{Classification, Classifier};
+use crate::core::classifier::{Classification, Classifier, Score};
use crate::core::state::State;
use crate::{LibsyError, Result};
use switchyard_protocol::{ModelId, Request, Response};
@@ -65,6 +65,32 @@ impl Classifier<State> for SourceStamp {
}
}
+/// Closes the cascade at zero confidence: a fallback, not a judgement.
+struct FallOpen {
+ targets: StageTargets,
+ default_tier: Tier,
+}
+
+#[async_trait]
+impl Classifier<State> for FallOpen {
+ async fn score(
+ &self,
+ state: &mut State,
+ _request: &mut Request,
+ _driver: Option<&Driver>,
+ ) -> Result<(Classification, Option<Response>)> {
+ let tier = fall_open_tier(state).unwrap_or(self.default_tier);
+ let target = self.targets.name(tier).clone();
+ Ok((
+ Classification::Scores(vec![Score {
+ target,
+ confidence: 0.0,
+ }]),
+ None,
+ ))
+ }
+}
+
/// The capability judge a stage router falls through to.
pub struct LlmFallback {
/// Target the judge model is called through. It is not a routing
@@ -165,9 +191,11 @@ fn build_route(
// The tiers are a fixed pair; their targets are whatever the deployment calls
// them, and the classifier scores onto those names.
let targets = StageTargets::new(capable.clone(), efficient.clone());
- // The picker's mode fixes the fallback tier up front, so the terminal
- // classifier is a constant rather than a per-turn lookup.
- let fall_open = targets.name(config.mode.default_tier()).to_string();
+ let default_tier = config.mode.default_tier();
+ let fall_open = FallOpen {
+ targets: targets.clone(),
+ default_tier,
+ };
let mut classifier = StageClassifier::new(targets, config.mode, config.confidence_threshold);
if let Some(notes) = config.handoff_notes {
@@ -194,10 +222,9 @@ fn build_route(
source: DecisionSource::LlmClassifier,
}));
}
- // Nothing behind this, so the turn lands on the picker's default tier —
- // including when the judge could not tell.
+ // Nothing behind this, so no turn is left unrouted.
router = router.with_classifier(Arc::new(SourceStamp {
- inner: Arc::new(DefaultTarget::new(fall_open)),
+ inner: Arc::new(fall_open),
source: DecisionSource::FallOpen,
}));
// Runs on the post-decision hook, so it applies to the target the cascade
@@ -219,8 +246,8 @@ mod tests {
};
use super::*;
- use crate::algorithms::util::stage::DECISION_SOURCE_KEY;
- use crate::core::classifier::Score;
+ use crate::algorithms::util::stage::{DECISION_SOURCE_KEY, clear_fall_open, set_fall_open};
+ use crate::core::processor::{Event, Processor};
use crate::core::state::StateValue;
use crate::core::testing::{Serve, reply, test_drive};
use switchyard_protocol::{Metadata, Response};
@@ -465,6 +492,64 @@ mod tests {
}
}
+ /// Stands in for a decider ahead of stage: sets the override once, clears it
+ /// once, and leaves the turns between alone.
+ #[derive(Default)]
+ struct TierDecider {
+ requests: Mutex<u32>,
+ }
+
+ #[async_trait]
+ impl Processor<State> for TierDecider {
+ async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
+ if matches!(event, Event::Request(_)) {
+ let mut requests = self.requests.lock();
+ *requests += 1;
+ match *requests {
+ 1 => set_fall_open(state, Tier::Efficient),
+ 4 => clear_fall_open(state),
+ _ => {}
+ }
+ }
+ Ok(())
+ }
+ }
+
+ #[tokio::test]
+ async fn an_override_replaces_the_picker_default_and_leaves_the_signals_alone() -> Result<()> {
+ let recorder = Arc::new(Recorder::default());
+ // The picker would fall open to "strong"; the override says "weak".
+ let config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5);
+ let route: Arc<dyn Algorithm> = Arc::new(
+ build_route(ModelId::from("strong"), ModelId::from("weak"), config)?
+ .with_processor(Arc::new(TierDecider::default())),
+ );
+
+ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;
+ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;
+ test_drive(route.clone(), turn_request(true), recorder.serve()).await?;
+ test_drive(route.clone(), turn_request(false), recorder.serve()).await?;
+
+ let routed = recorder.routed();
+ assert_eq!(
+ routed[0].target, "weak",
+ "an undecided turn takes the override"
+ );
+ assert_eq!(
+ routed[1].target, "weak",
+ "which outlives the turn that set it"
+ );
+ assert_eq!(
+ routed[2].target, "strong",
+ "a critical failure still reaches the signals"
+ );
+ assert_eq!(
+ routed[3].target, "strong",
+ "clearing restores the picker default"
+ );
+ Ok(())
+ }
+
#[tokio::test]
async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> {
let recorder = Arc::new(Recorder::default());
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 3f16334fc..a1c0560bb 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -81,13 +81,22 @@ impl Tier {
/// Stable label for stats and the [`routing_tier`](Classifier::routing_tier)
/// hook, independent of what the tiers' targets are called. These are the
/// strings the capability route reports too, so a deployment running both
- /// sees one tier vocabulary.
+ /// sees one tier vocabulary. Also the encoding a default-tier override is
+ /// stored under, so changing these strings invalidates one.
fn label(self) -> &'static str {
match self {
Self::Capable => "strong",
Self::Efficient => "weak",
}
}
+
+ fn from_label(label: &str) -> Option<Self> {
+ match label {
+ "strong" => Some(Self::Capable),
+ "weak" => Some(Self::Efficient),
+ _ => None,
+ }
+ }
}
/// The targets a stage router's two tiers route to.
@@ -153,6 +162,32 @@ impl PickerMode {
/// `State.extra` key under which the turn's [`DecisionSource`] is recorded.
pub const DECISION_SOURCE_KEY: &str = "decision_source";
+/// `State.extra` key under which a default-tier override is recorded.
+const FALL_OPEN_KEY: &str = "fall_open";
+
+/// Overrides the default tier undecided turns fall open to, until
+/// [`clear_fall_open`] restores the picker's. Held in session state, so it
+/// survives later requests only when they carry a session id.
+pub fn set_fall_open(state: &mut State, tier: Tier) {
+ state.extra.insert(
+ FALL_OPEN_KEY.to_string(),
+ StateValue::String(tier.label().to_string()),
+ );
+}
+
+/// Restores the picker's default tier.
+pub fn clear_fall_open(state: &mut State) {
+ state.extra.remove(FALL_OPEN_KEY);
+}
+
+/// The default-tier override for this session, if any.
+pub(crate) fn fall_open_tier(state: &State) -> Option<Tier> {
+ match state.extra.get(FALL_OPEN_KEY) {
+ Some(StateValue::String(label)) => Tier::from_label(label),
+ _ => None,
+ }
+}
+
/// Record which component decided the turn.
pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) {
state.extra.insert(
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 8ae823689..47a26c083 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -37,8 +37,8 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals};
// core (scorer, picker, and the `StageClassifier`).
pub use algorithms::util::stage::{
CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome,
- PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal,
- pick_tier, score_signal,
+ PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, clear_fall_open,
+ dimensions_from_signal, pick_tier, score_signal, set_fall_open,
};
mod observability;