Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): internalize fall-through routing (#197)

nachiketb-nvidiaa46cda7617cc
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index f2bf4f312..a4dc02aea 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -13,7 +13,7 @@ pub mod passthrough;
pub mod rand;
pub mod stage;
-pub use fall_through::{DefaultTarget, FallThrough, FallThroughDecision};
+pub(crate) use fall_through::{DefaultTarget, FallThrough};
pub use llm_class::{LlmTaskClassifier, TaskClassifierConfig};
pub use noop::{Noop, NoopDecision};
pub use passthrough::{Passthrough, PassthroughDecision};
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 0d35142bf..a2aba53f8 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -170,22 +170,6 @@ where
self.classifiers.push(classifier);
self
}
- /// Registers one dual-role component in *both* the processor chain and the classifier
- /// cascade.
- ///
- /// A component that writes state as a [`Processor`] and reads it back as a
- /// [`Classifier`] — such as [`AffinityRouter`](crate::algorithms::AffinityRouter) —
- /// shares that state through the instance, so both roles must be the same `Arc`.
- /// Registering the two separately is easy to half-wire: omit the processor and the
- /// classifier silently never sees an assignment. This registers both at once.
- pub fn with_component<T>(self, component: Arc<T>) -> Self
- where
- T: Processor<S> + Classifier<S> + 'static,
- {
- self.with_processor(component.clone())
- .with_classifier(component)
- }
-
/// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint.
pub(crate) async fn execute(
&self,
@@ -447,10 +431,11 @@ where
#[cfg(test)]
mod tests {
use super::*;
+ use crate::algorithms::{append_note, SystemPromptProcessor, TargetPrompts};
use crate::core::Classification;
use switchyard_protocol::{
- completion_text, text_response, LlmRequest, Message, Metadata, Role,
+ completion_text, text_request, text_response, LlmRequest, Message, Metadata, Role,
};
use crate::{LlmResponse, LlmTarget, RoutedLlmClient};
@@ -509,6 +494,59 @@ mod tests {
}
}
+ const CAPABLE_PROMPT: &str = "diagnose before you edit";
+ const EFFICIENT_PROMPT: &str = "follow the settled plan";
+ const NOTE: &str = "the previous model was stalling";
+
+ /// One model call as the prompt and note tests observe it.
+ #[derive(Clone, Debug, Default)]
+ struct RecordedCall {
+ target: String,
+ messages: Vec<String>,
+ instructions: Vec<String>,
+ }
+
+ /// Captures the prompt-bearing request that reached the selected target.
+ #[derive(Default)]
+ struct RecordingPromptClient(Mutex<Option<RecordedCall>>);
+
+ #[async_trait]
+ impl RoutedLlmClient for RecordingPromptClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ *self.0.lock() = Some(RecordedCall {
+ target: decision.selected_model().to_string(),
+ messages: request
+ .llm_request
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content("|"))
+ .collect(),
+ instructions: request
+ .llm_request
+ .instructions
+ .iter()
+ .filter_map(|block| block.content.iter().find_map(text_of))
+ .collect(),
+ });
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
+ metadata: None,
+ })
+ }
+ }
+
+ fn text_of(block: &switchyard_protocol::ContentBlock) -> Option<String> {
+ match block {
+ switchyard_protocol::ContentBlock::Text { text } => Some(text.clone()),
+ _ => None,
+ }
+ }
+
/// A target set whose targets all serve via [`EchoClient`].
fn target_set(names: &[&str]) -> LlmTargetSet {
LlmTargetSet::new(
@@ -522,6 +560,57 @@ mod tests {
)
}
+ fn prompt_targets(client: &Arc<RecordingPromptClient>, names: &[&str]) -> LlmTargetSet {
+ LlmTargetSet::new(
+ names
+ .iter()
+ .map(|name| LlmTarget {
+ semantic_name: (*name).to_string(),
+ llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),
+ })
+ .collect(),
+ )
+ }
+
+ fn target_prompts() -> TargetPrompts {
+ TargetPrompts::default()
+ .with("capable", CAPABLE_PROMPT)
+ .with("efficient", EFFICIENT_PROMPT)
+ }
+
+ /// Routes one turn on a prompt test cascade and returns the recorded model call.
+ async fn routed_prompt_call(
+ client: &Arc<RecordingPromptClient>,
+ router: FallThrough,
+ ) -> Result<RecordedCall> {
+ Arc::new(router)
+ .run(
+ Context::default(),
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "fix the build"),
+ raw_request: None,
+ metadata: None,
+ },
+ )
+ .await?;
+ let call = client.0.lock().take();
+ match call {
+ Some(call) => Ok(call),
+ None => panic!("the model was never called"),
+ }
+ }
+
+ /// A prompt cascade that always routes to `target`.
+ fn prompt_router(
+ client: &Arc<RecordingPromptClient>,
+ target: &str,
+ prompts: TargetPrompts,
+ ) -> FallThrough {
+ FallThrough::new(prompt_targets(client, &["capable", "efficient"]))
+ .with_processor(Arc::new(SystemPromptProcessor::new(prompts)))
+ .with_classifier(Arc::new(DefaultTarget::new(target)))
+ }
+
/// A classifier that emits fixed scores (empty = abstain).
struct FixedClassifier(Vec<Score>);
@@ -607,6 +696,94 @@ mod tests {
// --- tests -------------------------------------------------------------------------
+ #[tokio::test]
+ async fn each_target_gets_its_own_prompt() -> Result<()> {
+ for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {
+ let client = Arc::new(RecordingPromptClient::default());
+ let call =
+ routed_prompt_call(&client, prompt_router(&client, target, target_prompts()))
+ .await?;
+ assert_eq!(call.target, target);
+ assert_eq!(call.instructions, vec![expected.to_string()]);
+ }
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {
+ let client = Arc::new(RecordingPromptClient::default());
+ let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);
+
+ let call =
+ routed_prompt_call(&client, prompt_router(&client, "efficient", only_capable)).await?;
+
+ assert!(
+ call.instructions.is_empty(),
+ "one target's prompt must not leak onto another: {:?}",
+ call.instructions
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn the_prompt_follows_the_target_whichever_classifier_picked_it() -> Result<()> {
+ // The first classifier abstains, so the second decides; the prompt follows the
+ // target the cascade settled on rather than the classifier that named it.
+ struct Abstains;
+
+ #[async_trait]
+ impl Classifier for Abstains {
+ async fn score(
+ &self,
+ _state: &mut (),
+ _request: &mut Request,
+ _driver: Option<&Driver>,
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((Classification::Ambiguous(Vec::new()), None))
+ }
+ }
+
+ let client = Arc::new(RecordingPromptClient::default());
+ let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))
+ .with_processor(Arc::new(SystemPromptProcessor::new(target_prompts())))
+ .with_classifier(Arc::new(Abstains))
+ .with_classifier(Arc::new(DefaultTarget::new("capable")));
+
+ let call = routed_prompt_call(&client, router).await?;
+
+ assert_eq!(call.target, "capable");
+ assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> {
+ // Appends a note to every outbound request, the way a router would on a turn it
+ // wants to explain.
+ struct Noting;
+
+ #[async_trait]
+ impl Processor for Noting {
+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
+ if let Event::Decision { request, .. } = event {
+ append_note(request, NOTE);
+ }
+ Ok(())
+ }
+ }
+
+ let client = Arc::new(RecordingPromptClient::default());
+ let router = FallThrough::new(prompt_targets(&client, &["capable", "efficient"]))
+ .with_processor(Arc::new(Noting))
+ .with_classifier(Arc::new(DefaultTarget::new("capable")));
+
+ let call = routed_prompt_call(&client, router).await?;
+
+ assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);
+ assert!(call.instructions.is_empty(), "a note is not an instruction");
+ Ok(())
+ }
+
/// Overflows for the named targets and echoes for the rest, recording every call.
struct OverflowClient {
overflowing: Vec<&'static str>,
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index e9a1d28e5..4d0395c6d 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -87,9 +87,11 @@ fn request(headers: &[(&str, &str)]) -> Request {
/// Affinity replays an existing pin, the override seeds one for delegated work, and the
/// terminal classifier serves everything else.
fn router() -> Arc<FallThrough> {
+ let affinity = Arc::new(AffinityRouter::for_subagents());
Arc::new(
FallThrough::<()>::new(targets())
- .with_component(Arc::new(AffinityRouter::for_subagents()))
+ .with_processor(affinity.clone())
+ .with_classifier(affinity)
.with_classifier(Arc::new(SubagentOverride::new("worker")))
.with_classifier(Arc::new(AlwaysOrchestrator)),
)
@@ -168,7 +170,8 @@ async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()>
fn router_overriding_to(affinity: Arc<AffinityRouter>, worker: &str) -> Arc<FallThrough> {
Arc::new(
FallThrough::<()>::new(targets())
- .with_component(affinity)
+ .with_processor(affinity.clone())
+ .with_classifier(affinity)
.with_classifier(Arc::new(SubagentOverride::new(worker)))
.with_classifier(Arc::new(AlwaysOrchestrator)),
)
@@ -224,9 +227,11 @@ async fn distinct_children_are_pinned_independently() -> Result<()> {
async fn a_cascade_without_the_override_still_routes_root_traffic() -> Result<()> {
// Affinity and the override are independent: dropping the override leaves a valid
// cascade, which is the point of composing them rather than nesting one in the other.
+ let affinity = Arc::new(AffinityRouter::for_subagents());
let router = Arc::new(
FallThrough::<()>::new(targets())
- .with_component(Arc::new(AffinityRouter::for_subagents()))
+ .with_processor(affinity.clone())
+ .with_classifier(affinity)
.with_classifier(Arc::new(AlwaysOrchestrator)),
);
assert_eq!(turn(&router, &child("child-1")).await?, "orchestrator");
diff --git a/crates/libsy/tests/prompts.rs b/crates/libsy/tests/prompts.rs
deleted file mode 100644
index 31301f072..000000000
--- a/crates/libsy/tests/prompts.rs
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-//! Integration tests for the request-text helpers on a running cascade.
-//!
-//! The unit tests cover placement in isolation. These drive whole turns through
-//! a plain [`FallThrough`] — no stage routing — to pin the two things only a
-//! running composition can show: that the text survives to the model call, and
-//! that it follows the target the cascade settled on rather than the classifier
-//! that named it.
-
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use parking_lot::Mutex;
-
-use switchyard_libsy::algorithms::{
- append_note, DefaultTarget, FallThrough, SystemPromptProcessor, TargetPrompts,
-};
-use switchyard_libsy::{
- Algorithm, Classification, Classifier, Context, Decision, Driver, Event, LlmResponse,
- LlmTarget, LlmTargetSet, Processor, Request, Response, Result, RoutedLlmClient,
-};
-use switchyard_protocol::{text_request, text_response};
-
-const CAPABLE_PROMPT: &str = "diagnose before you edit";
-const EFFICIENT_PROMPT: &str = "follow the settled plan";
-const NOTE: &str = "the previous model was stalling";
-
-/// One model call as the client saw it.
-#[derive(Clone, Debug, Default)]
-struct Call {
- target: String,
- messages: Vec<String>,
- instructions: Vec<String>,
-}
-
-/// Records what the model was handed, so a test asserts on that rather than on
-/// composition internals.
-#[derive(Default)]
-struct RecordingClient(Mutex<Option<Call>>);
-
-#[async_trait]
-impl RoutedLlmClient for RecordingClient {
- async fn call(
- &self,
- _ctx: Context,
- request: Request,
- decision: Arc<dyn Decision>,
- ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
- *self.0.lock() = Some(Call {
- target: decision.selected_model().to_string(),
- messages: request
- .llm_request
- .messages
- .iter()
- .filter_map(|message| message.text_content("|"))
- .collect(),
- instructions: request
- .llm_request
- .instructions
- .iter()
- .filter_map(|block| block.content.iter().find_map(text_of))
- .collect(),
- });
- Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())),
- metadata: None,
- })
- }
-}
-
-fn text_of(block: &switchyard_protocol::ContentBlock) -> Option<String> {
- match block {
- switchyard_protocol::ContentBlock::Text { text } => Some(text.clone()),
- _ => None,
- }
-}
-
-/// A classifier that never decides, so the next one in the cascade gets the turn.
-struct Abstains;
-
-#[async_trait]
-impl Classifier for Abstains {
- async fn score(
- &self,
- _state: &mut (),
- _request: &mut Request,
- _driver: Option<&Driver>,
- ) -> Result<(Classification, Option<Response>)> {
- Ok((Classification::Ambiguous(Vec::new()), None))
- }
-}
-
-/// Appends a note to every outbound request, the way a router would on a turn
-/// it wants to explain.
-struct Noting;
-
-#[async_trait]
-impl Processor for Noting {
- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
- if let Event::Decision { request, .. } = event {
- append_note(request, NOTE);
- }
- Ok(())
- }
-}
-
-fn targets(client: &Arc<RecordingClient>, names: &[&str]) -> LlmTargetSet {
- LlmTargetSet::new(
- names
- .iter()
- .map(|name| LlmTarget {
- semantic_name: (*name).to_string(),
- llm_client: Some(client.clone() as Arc<dyn RoutedLlmClient>),
- })
- .collect(),
- )
-}
-
-fn prompts() -> TargetPrompts {
- TargetPrompts::default()
- .with("capable", CAPABLE_PROMPT)
- .with("efficient", EFFICIENT_PROMPT)
-}
-
-/// Runs one turn on a cascade that always routes to `target`.
-async fn routed_to(client: &Arc<RecordingClient>, router: FallThrough) -> Result<Call> {
- Arc::new(router)
- .run(
- Context::default(),
- Request {
- llm_request: text_request(Some("auto".to_string()), "fix the build"),
- raw_request: None,
- metadata: None,
- },
- )
- .await?;
- let call = client.0.lock().take();
- match call {
- Some(call) => Ok(call),
- None => panic!("the model was never called"),
- }
-}
-
-/// A cascade that routes to `target` and applies `prompts`.
-fn router(client: &Arc<RecordingClient>, target: &str, prompts: TargetPrompts) -> FallThrough {
- FallThrough::new(targets(client, &["capable", "efficient"]))
- .with_processor(Arc::new(SystemPromptProcessor::new(prompts)))
- .with_classifier(Arc::new(DefaultTarget::new(target)))
-}
-
-#[tokio::test]
-async fn each_target_gets_its_own_prompt() -> Result<()> {
- for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] {
- let client = Arc::new(RecordingClient::default());
- let call = routed_to(&client, router(&client, target, prompts())).await?;
- assert_eq!(call.target, target);
- assert_eq!(call.instructions, vec![expected.to_string()]);
- }
- Ok(())
-}
-
-#[tokio::test]
-async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT);
-
- let call = routed_to(&client, router(&client, "efficient", only_capable)).await?;
-
- assert!(
- call.instructions.is_empty(),
- "one target's prompt must not leak onto another: {:?}",
- call.instructions
- );
- Ok(())
-}
-
-#[tokio::test]
-async fn the_prompt_follows_the_target_whichever_classifier_picked_it() -> Result<()> {
- // The first classifier abstains, so the second decides — the prompt still
- // has to follow the target the cascade settled on.
- let client = Arc::new(RecordingClient::default());
- let router = FallThrough::new(targets(&client, &["capable", "efficient"]))
- .with_processor(Arc::new(SystemPromptProcessor::new(prompts())))
- .with_classifier(Arc::new(Abstains))
- .with_classifier(Arc::new(DefaultTarget::new("capable")));
-
- let call = routed_to(&client, router).await?;
-
- assert_eq!(call.target, "capable");
- assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]);
- Ok(())
-}
-
-#[tokio::test]
-async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> {
- let client = Arc::new(RecordingClient::default());
- let router = FallThrough::new(targets(&client, &["capable", "efficient"]))
- .with_processor(Arc::new(Noting))
- .with_classifier(Arc::new(DefaultTarget::new("capable")));
-
- let call = routed_to(&client, router).await?;
-
- // Joined onto the trailing user turn rather than opening one of its own.
- assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]);
- assert!(call.instructions.is_empty(), "a note is not an instruction");
- Ok(())
-}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 69ed372ca..be9322568 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -17,9 +17,8 @@ use axum::response::{IntoResponse, Response as HttpResponse};
use axum::routing::post;
use axum::{Json, Router};
use http_body_util::BodyExt;
-use libsy::algorithms::{FallThrough, Random};
-use libsy::stage_router::{PickerMode, StageClassifier, StageTargets};
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State as AlgorithmState};
+use libsy::algorithms::Random;
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
use serde_json::{json, Value};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use switchyard_server::config::load_server_state;
@@ -809,78 +808,6 @@ targets = ["weak"]
Ok(())
}
-// Build a stage_router (FallThrough) with an Anthropic `strong` tier and an
-// OpenAI `weak` tier, both pointed at the mock upstream, and register it.
-fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<ServerState> {
- let cfg = |url: &str| HttpBackendConfig {
- base_url: url.to_string(),
- api_key: Some("k".to_string()),
- extra_headers: BTreeMap::new(),
- extra_body: BTreeMap::new(),
- max_retries: 0,
- };
- let strong: Arc<dyn RoutedLlmClient> =
- Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(
- "strong",
- Backend::Anthropic(cfg(&upstream.base_url)),
- None,
- )])?);
- let weak: Arc<dyn RoutedLlmClient> = Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(
- "weak",
- Backend::OpenAiChat(cfg(&upstream.base_url)),
- None,
- )])?);
- let targets = LlmTargetSet::new(vec![
- LlmTarget {
- semantic_name: "strong".to_string(),
- llm_client: Some(strong),
- },
- LlmTarget {
- semantic_name: "weak".to_string(),
- llm_client: Some(weak),
- },
- ]);
- let stage: Arc<dyn Algorithm> = Arc::new(
- FallThrough::<AlgorithmState>::new_with_state(targets).with_classifier(Arc::new(
- StageClassifier::new(StageTargets::new("strong", "weak"), mode, 0.5),
- )),
- );
- Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)
-}
-
-/// `count_tokens` is a **direct passthrough**, not a routed call, so on a stage
-/// router it goes straight to the Anthropic (`strong`) tier — bypassing the
-/// classifier cascade. This is exactly what makes it work where a completion
-/// can't: a signal-less request (as `count_tokens` always is) gives the
-/// `StageClassifier` nothing to score, so a *completion* on this bare stage
-/// router abstains — but `count_tokens` still succeeds.
-#[tokio::test]
-async fn count_tokens_on_a_stage_router_passes_through_to_the_anthropic_tier() -> TestResult {
- let upstream = MockUpstream::start().await?;
- let app = build_switchyard_router(stage_router_state(&upstream, PickerMode::EfficientFirst)?);
- let body = json!({
- "model": "switchyard/stage",
- "messages": [{"role": "user", "content": "hi"}]
- });
-
- // A completion routes → the bare StageClassifier abstains on a signal-less
- // request → error.
- let completion = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;
- assert!(completion.text()?.contains("abstained"));
-
- // count_tokens does NOT route — it passes through to the strong (Anthropic)
- // tier and succeeds.
- let count = send(&app, "POST", "/v1/messages/count_tokens", Some(body)).await?;
- assert_eq!(count.status, StatusCode::OK);
- assert_eq!(count.json()?["input_tokens"], 7);
-
- // The forwarded call went to count_tokens with the strong tier's model id.
- let calls = upstream.calls.lock().await;
- assert_eq!(calls.len(), 1);
- assert_eq!(calls[0]["model"], "strong");
- Ok(())
-}
-
#[tokio::test]
async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
let (upstream, app) = test_app(&[