Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(libsy): internalize fall-through routing (#197)
nachiketb-nvidiaa46cda7617cc
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rsindex f2bf4f312..a4dc02aea 100644--- a/crates/libsy/src/algorithms.rs+++ b/crates/libsy/src/algorithms.rs@@ -13,7 +13,7 @@ pub mod passthrough;pub mod rand;pub mod stage;-pub use fall_through::{DefaultTarget, FallThrough, FallThroughDecision};+pub(crate) use fall_through::{DefaultTarget, FallThrough};pub use llm_class::{LlmTaskClassifier, TaskClassifierConfig};pub use noop::{Noop, NoopDecision};pub use passthrough::{Passthrough, PassthroughDecision};diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 0d35142bf..a2aba53f8 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -170,22 +170,6 @@ whereself.classifiers.push(classifier);self}- /// Registers one dual-role component in *both* the processor chain and the classifier- /// cascade.- ///- /// A component that writes state as a [`Processor`] and reads it back as a- /// [`Classifier`] — such as [`AffinityRouter`](crate::algorithms::AffinityRouter) —- /// shares that state through the instance, so both roles must be the same `Arc`.- /// Registering the two separately is easy to half-wire: omit the processor and the- /// classifier silently never sees an assignment. This registers both at once.- pub fn with_component<T>(self, component: Arc<T>) -> Self- where- T: Processor<S> + Classifier<S> + 'static,- {- self.with_processor(component.clone())- .with_classifier(component)- }-/// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint.pub(crate) async fn execute(&self,@@ -447,10 +431,11 @@ where#[cfg(test)]mod tests {use super::*;+ use crate::algorithms::{append_note, SystemPromptProcessor, TargetPrompts};use crate::core::Classification;use switchyard_protocol::{- completion_text, text_response, LlmRequest, Message, Metadata, Role,+ completion_text, text_request, text_response, LlmRequest, Message, Metadata, Role,};use crate::{LlmResponse, LlmTarget, RoutedLlmClient};@@ -509,6 +494,59 @@ mod tests {}}+ const CAPABLE_PROMPT: &str = "diagnose before you edit";+ const EFFICIENT_PROMPT: &str = "follow the settled plan";+ const NOTE: &str = "the previous model was stalling";++ /// One model call as the prompt and note tests observe it.+ #[derive(Clone, Debug, Default)]+ struct RecordedCall {+ target: String,+ messages: Vec<String>,+ instructions: Vec<String>,+ }++ /// Captures the prompt-bearing request that reached the selected target.+ #[derive(Default)]+ struct RecordingPromptClient(Mutex<Option<RecordedCall>>);++ #[async_trait]+ impl RoutedLlmClient for RecordingPromptClient {+ async fn call(+ &self,+ _ctx: Context,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ *self.0.lock() = Some(RecordedCall {+ target: decision.selected_model().to_string(),+ messages: request+ .llm_request+ .messages+ .iter()+ .filter_map(|message| message.text_content("|"))+ .collect(),+ instructions: request+ .llm_request+ .instructions+ .iter()+ .filter_map(|block| block.content.iter().find_map(text_of))+ .collect(),+ });+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),+ metadata: None,+ })+ }+ }++ fn text_of(block: &switchyard_protocol::ContentBlock) -> Option<String> {+ match block {+ switchyard_protocol::ContentBlock::Text { text } => Some(text.clone()),+ _ => None,+ }+ }+/// A target set whose targets all serve via [`EchoClient`].fn target_set(names: &[&str]) -> LlmTargetSet {LlmTargetSet::new(@@ -522,6 +560,57 @@ mod tests {)}+ fn prompt_targets(client: &Arc<RecordingPromptClient>, names: &[&str]) -> LlmTargetSet {+ LlmTargetSet::new(+ names+ .iter()+ .map(|name| LlmTarget {+ semantic_name: (*name).to_string(),+ llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),+ })+ .collect(),+ )+ }++ fn target_prompts() -> TargetPrompts {+ TargetPrompts::default()+ .with("capable", CAPABLE_PROMPT)+ .with("efficient", EFFICIENT_PROMPT)+ }++ /// Routes one turn on a prompt test cascade and returns the recorded model call.+ async fn routed_prompt_call(+ client: &Arc<RecordingPromptClient>,+ router: FallThrough,+ ) -> Result<RecordedCall> {+ Arc::new(router)+ .run(+ Context::default(),+ Request {+ llm_request: text_request(Some("auto".to_string()), "fix the build"),+ raw_request: None,+ metadata: None,+ },+ )+ .await?;+ let call = client.0.lock().take();+ match call {+ Some(call) => Ok(call),+ None => panic!("the model was never called"),+ }+ }++ /// A prompt cascade that always routes to `target`.+ fn prompt_router(+ client: &Arc<RecordingPromptClient>,+ target: &str,+ prompts: TargetPrompts,+ ) -> FallThrough {+ FallThrough::new(prompt_targets(client, &["capable", "efficient"]))+ .with_processor(Arc::new(SystemPromptProcessor::new(prompts)))+ .with_classifier(Arc::new(DefaultTarget::new(target)))+ }+/// A classifier that emits fixed scores (empty = abstain).struct FixedClassifier(Vec<Score>);@@ -607,6 +696,94 @@ mod tests {// --- tests -------------------------------------------------------------------------+ #[tokio::test]+ async fn each_target_gets_its_own_prompt() -> Result<()> {+ for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {+ let client = Arc::new(RecordingPromptClient::default());+ let call =+ routed_prompt_call(&client, prompt_router(&client, target, target_prompts()))+ .await?;+ assert_eq!(call.target, target);+ assert_eq!(call.instructions, vec![expected.to_string()]);+ }+ Ok(())+ }++ #[tokio::test]+ async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {+ let client = Arc::new(RecordingPromptClient::default());+ let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);++ let call =+ routed_prompt_call(&client, prompt_router(&client, "efficient", only_capable)).await?;++ assert!(+ call.instructions.is_empty(),+ "one target's prompt must not leak onto another: {:?}",+ call.instructions+ );+ Ok(())+ }++ #[tokio::test]+ async fn the_prompt_follows_the_target_whichever_classifier_picked_it() -> Result<()> {+ // The first classifier abstains, so the second decides; the prompt follows the+ // target the cascade settled on rather than the classifier that named it.+ struct Abstains;++ #[async_trait]+ impl Classifier for Abstains {+ async fn score(+ &self,+ _state: &mut (),+ _request: &mut Request,+ _driver: Option<&Driver>,+ ) -> Result<(Classification, Option<Response>)> {+ Ok((Classification::Ambiguous(Vec::new()), None))+ }+ }++ let client = Arc::new(RecordingPromptClient::default());+ let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))+ .with_processor(Arc::new(SystemPromptProcessor::new(target_prompts())))+ .with_classifier(Arc::new(Abstains))+ .with_classifier(Arc::new(DefaultTarget::new("capable")));++ let call = routed_prompt_call(&client, router).await?;++ assert_eq!(call.target, "capable");+ assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);+ Ok(())+ }++ #[tokio::test]+ async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> {+ // Appends a note to every outbound request, the way a router would on a turn it+ // wants to explain.+ struct Noting;++ #[async_trait]+ impl Processor for Noting {+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {+ if let Event::Decision { request, .. } = event {+ append_note(request, NOTE);+ }+ Ok(())+ }+ }++ let client = Arc::new(RecordingPromptClient::default());+ let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))+ .with_processor(Arc::new(Noting))+ .with_classifier(Arc::new(DefaultTarget::new("capable")));++ let call = routed_prompt_call(&client, router).await?;++ assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);+ assert!(call.instructions.is_empty(), "a note is not an instruction");+ Ok(())+ }+/// Overflows for the named targets and echoes for the rest, recording every call.struct OverflowClient {overflowing: Vec<&'static str>,diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rsindex e9a1d28e5..4d0395c6d 100644--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs@@ -87,9 +87,11 @@ fn request(headers: &[(&str, &str)]) -> Request {/// Affinity replays an existing pin, the override seeds one for delegated work, and the/// terminal classifier serves everything else.fn router() -> Arc<FallThrough> {+ let affinity = Arc::new(AffinityRouter::for_subagents());Arc::new(FallThrough::<()>::new(targets())- .with_component(Arc::new(AffinityRouter::for_subagents()))+ .with_processor(affinity.clone())+ .with_classifier(affinity).with_classifier(Arc::new(SubagentOverride::new("worker"))).with_classifier(Arc::new(AlwaysOrchestrator)),)@@ -168,7 +170,8 @@ async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()>fn router_overriding_to(affinity: Arc<AffinityRouter>, worker: &str) -> Arc<FallThrough> {Arc::new(FallThrough::<()>::new(targets())- .with_component(affinity)+ .with_processor(affinity.clone())+ .with_classifier(affinity).with_classifier(Arc::new(SubagentOverride::new(worker))).with_classifier(Arc::new(AlwaysOrchestrator)),)@@ -224,9 +227,11 @@ async fn distinct_children_are_pinned_independently() -> Result<()> {async fn a_cascade_without_the_override_still_routes_root_traffic() -> Result<()> {// Affinity and the override are independent: dropping the override leaves a valid// cascade, which is the point of composing them rather than nesting one in the other.+ let affinity = Arc::new(AffinityRouter::for_subagents());let router = Arc::new(FallThrough::<()>::new(targets())- .with_component(Arc::new(AffinityRouter::for_subagents()))+ .with_processor(affinity.clone())+ .with_classifier(affinity).with_classifier(Arc::new(AlwaysOrchestrator)),);assert_eq!(turn(&router, &child("child-1")).await?, "orchestrator");diff --git a/crates/libsy/tests/prompts.rs b/crates/libsy/tests/prompts.rsdeleted file mode 100644index 31301f072..000000000--- a/crates/libsy/tests/prompts.rs+++ /dev/null@@ -1,209 +0,0 @@-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.-// SPDX-License-Identifier: Apache-2.0--//! Integration tests for the request-text helpers on a running cascade.-//!-//! The unit tests cover placement in isolation. These drive whole turns through-//! a plain [`FallThrough`] — no stage routing — to pin the two things only a-//! running composition can show: that the text survives to the model call, and-//! that it follows the target the cascade settled on rather than the classifier-//! that named it.--use std::sync::Arc;--use async_trait::async_trait;-use parking_lot::Mutex;--use switchyard_libsy::algorithms::{- append_note, DefaultTarget, FallThrough, SystemPromptProcessor, TargetPrompts,-};-use switchyard_libsy::{- Algorithm, Classification, Classifier, Context, Decision, Driver, Event, LlmResponse,- LlmTarget, LlmTargetSet, Processor, Request, Response, Result, RoutedLlmClient,-};-use switchyard_protocol::{text_request, text_response};--const CAPABLE_PROMPT: &str = "diagnose before you edit";-const EFFICIENT_PROMPT: &str = "follow the settled plan";-const NOTE: &str = "the previous model was stalling";--/// One model call as the client saw it.-#[derive(Clone, Debug, Default)]-struct Call {- target: String,- messages: Vec<String>,- instructions: Vec<String>,-}--/// Records what the model was handed, so a test asserts on that rather than on-/// composition internals.-#[derive(Default)]-struct RecordingClient(Mutex<Option<Call>>);--#[async_trait]-impl RoutedLlmClient for RecordingClient {- async fn call(- &self,- _ctx: Context,- request: Request,- decision: Arc<dyn Decision>,- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {- *self.0.lock() = Some(Call {- target: decision.selected_model().to_string(),- messages: request- .llm_request- .messages- .iter()- .filter_map(|message| message.text_content("|"))- .collect(),- instructions: request- .llm_request- .instructions- .iter()- .filter_map(|block| block.content.iter().find_map(text_of))- .collect(),- });- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),- metadata: None,- })- }-}--fn text_of(block: &switchyard_protocol::ContentBlock) -> Option<String> {- match block {- switchyard_protocol::ContentBlock::Text { text } => Some(text.clone()),- _ => None,- }-}--/// A classifier that never decides, so the next one in the cascade gets the turn.-struct Abstains;--#[async_trait]-impl Classifier for Abstains {- async fn score(- &self,- _state: &mut (),- _request: &mut Request,- _driver: Option<&Driver>,- ) -> Result<(Classification, Option<Response>)> {- Ok((Classification::Ambiguous(Vec::new()), None))- }-}--/// Appends a note to every outbound request, the way a router would on a turn-/// it wants to explain.-struct Noting;--#[async_trait]-impl Processor for Noting {- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {- if let Event::Decision { request, .. } = event {- append_note(request, NOTE);- }- Ok(())- }-}--fn targets(client: &Arc<RecordingClient>, names: &[&str]) -> LlmTargetSet {- LlmTargetSet::new(- names- .iter()- .map(|name| LlmTarget {- semantic_name: (*name).to_string(),- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),- })- .collect(),- )-}--fn prompts() -> TargetPrompts {- TargetPrompts::default()- .with("capable", CAPABLE_PROMPT)- .with("efficient", EFFICIENT_PROMPT)-}--/// Runs one turn on a cascade that always routes to `target`.-async fn routed_to(client: &Arc<RecordingClient>, router: FallThrough) -> Result<Call> {- Arc::new(router)- .run(- Context::default(),- Request {- llm_request: text_request(Some("auto".to_string()), "fix the build"),- raw_request: None,- metadata: None,- },- )- .await?;- let call = client.0.lock().take();- match call {- Some(call) => Ok(call),- None => panic!("the model was never called"),- }-}--/// A cascade that routes to `target` and applies `prompts`.-fn router(client: &Arc<RecordingClient>, target: &str, prompts: TargetPrompts) -> FallThrough {- FallThrough::new(targets(client, &["capable", "efficient"]))- .with_processor(Arc::new(SystemPromptProcessor::new(prompts)))- .with_classifier(Arc::new(DefaultTarget::new(target)))-}--#[tokio::test]-async fn each_target_gets_its_own_prompt() -> Result<()> {- for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {- let client = Arc::new(RecordingClient::default());- let call = routed_to(&client, router(&client, target, prompts())).await?;- assert_eq!(call.target, target);- assert_eq!(call.instructions, vec![expected.to_string()]);- }- Ok(())-}--#[tokio::test]-async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);-- let call = routed_to(&client, router(&client, "efficient", only_capable)).await?;-- assert!(- call.instructions.is_empty(),- "one target's prompt must not leak onto another: {:?}",- call.instructions- );- Ok(())-}--#[tokio::test]-async fn the_prompt_follows_the_target_whichever_classifier_picked_it() -> Result<()> {- // The first classifier abstains, so the second decides — the prompt still- // has to follow the target the cascade settled on.- let client = Arc::new(RecordingClient::default());- let router = FallThrough::new(targets(&client, &["capable", "efficient"]))- .with_processor(Arc::new(SystemPromptProcessor::new(prompts())))- .with_classifier(Arc::new(Abstains))- .with_classifier(Arc::new(DefaultTarget::new("capable")));-- let call = routed_to(&client, router).await?;-- assert_eq!(call.target, "capable");- assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);- Ok(())-}--#[tokio::test]-async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> {- let client = Arc::new(RecordingClient::default());- let router = FallThrough::new(targets(&client, &["capable", "efficient"]))- .with_processor(Arc::new(Noting))- .with_classifier(Arc::new(DefaultTarget::new("capable")));-- let call = routed_to(&client, router).await?;-- // Joined onto the trailing user turn rather than opening one of its own.- assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);- assert!(call.instructions.is_empty(), "a note is not an instruction");- Ok(())-}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 69ed372ca..be9322568 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -17,9 +17,8 @@ use axum::response::{IntoResponse, Response as HttpResponse};use axum::routing::post;use axum::{Json, Router};use http_body_util::BodyExt;-use libsy::algorithms::{FallThrough, Random};-use libsy::stage_router::{PickerMode, StageClassifier, StageTargets};-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State as AlgorithmState};+use libsy::algorithms::Random;+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};use serde_json::{json, Value};use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};use switchyard_server::config::load_server_state;@@ -809,78 +808,6 @@ targets = ["weak"]Ok(())}-// Build a stage_router (FallThrough) with an Anthropic `strong` tier and an-// OpenAI `weak` tier, both pointed at the mock upstream, and register it.-fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<ServerState> {- let cfg = |url: &str| HttpBackendConfig {- base_url: url.to_string(),- api_key: Some("k".to_string()),- extra_headers: BTreeMap::new(),- extra_body: BTreeMap::new(),- max_retries: 0,- };- let strong: Arc<dyn RoutedLlmClient> =- Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(- "strong",- Backend::Anthropic(cfg(&upstream.base_url)),- None,- )])?);- let weak: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(- "weak",- Backend::OpenAiChat(cfg(&upstream.base_url)),- None,- )])?);- let targets = LlmTargetSet::new(vec![- LlmTarget {- semantic_name: "strong".to_string(),- llm_client: Some(strong),- },- LlmTarget {- semantic_name: "weak".to_string(),- llm_client: Some(weak),- },- ]);- let stage: Arc<dyn Algorithm> = Arc::new(- FallThrough::<AlgorithmState>::new_with_state(targets).with_classifier(Arc::new(- StageClassifier::new(StageTargets::new("strong", "weak"), mode, 0.5),- )),- );- Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)-}--/// `count_tokens` is a **direct passthrough**, not a routed call, so on a stage-/// router it goes straight to the Anthropic (`strong`) tier — bypassing the-/// classifier cascade. This is exactly what makes it work where a completion-/// can't: a signal-less request (as `count_tokens` always is) gives the-/// `StageClassifier` nothing to score, so a *completion* on this bare stage-/// router abstains — but `count_tokens` still succeeds.-#[tokio::test]-async fn count_tokens_on_a_stage_router_passes_through_to_the_anthropic_tier() -> TestResult {- let upstream = MockUpstream::start().await?;- let app = build_switchyard_router(stage_router_state(&upstream, PickerMode::EfficientFirst)?);- let body = json!({- "model": "switchyard/stage",- "messages": [{"role": "user", "content": "hi"}]- });-- // A completion routes → the bare StageClassifier abstains on a signal-less- // request → error.- let completion = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;- assert!(completion.text()?.contains("abstained"));-- // count_tokens does NOT route — it passes through to the strong (Anthropic)- // tier and succeeds.- let count = send(&app, "POST", "/v1/messages/count_tokens", Some(body)).await?;- assert_eq!(count.status, StatusCode::OK);- assert_eq!(count.json()?["input_tokens"], 7);-- // The forwarded call went to count_tokens with the strong tier's model id.- let calls = upstream.calls.lock().await;- assert_eq!(calls.len(), 1);- assert_eq!(calls[0]["model"], "strong");- Ok(())-}-#[tokio::test]async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {let (upstream, app) = test_app(&[