Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(prefill-router): add libsy algorithm wrapper (#593)
nachiketb-nvidiabbaa05c6aab5
diff --git a/Cargo.lock b/Cargo.lockindex 0209f7f89..c72a3d297 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1541,10 +1541,13 @@ dependencies = [name = "prefill-router"version = "0.2.0"dependencies = [+ "async-trait","pyo3","pyo3-build-config",+ "switchyard-libsy","switchyard-protocol","thiserror 2.0.18",+ "tokio","tracing",]diff --git a/crates/prefill-router/Cargo.toml b/crates/prefill-router/Cargo.tomlindex df4c5a3b9..9fe229cb7 100644--- a/crates/prefill-router/Cargo.toml+++ b/crates/prefill-router/Cargo.toml@@ -12,9 +12,12 @@ repository.workspace = truerust-version.workspace = true[dependencies]+async-trait.workspace = true+libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }pyo3 = { version = "0.28.3", features = ["auto-initialize"] }switchyard-protocol.workspace = truethiserror.workspace = true+tokio.workspace = truetracing.workspace = true[build-dependencies]diff --git a/crates/prefill-router/src/algorithm.rs b/crates/prefill-router/src/algorithm.rsnew file mode 100644index 000000000..f0979c96a--- /dev/null+++ b/crates/prefill-router/src/algorithm.rs@@ -0,0 +1,214 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! libsy adapter for checkpoint-backed prefill routing.++use std::path::PathBuf;+use std::sync::Arc;++use async_trait::async_trait;+use libsy::{+ AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,+};+use switchyard_protocol::{ModelId, Request, Role};+use tokio::sync::Mutex;++use crate::{+ PrefillForward, PrefillRouter, PrefillRouterError, Result, TransformersForward,+ TransformersForwardConfig,+};++const ALGORITHM_NAME: &str = "prefill_router";++/// Configuration for a libsy prefill-router algorithm.+#[derive(Clone, Debug)]+pub struct PrefillRouterConfig {+ /// Ordered routing targets, matching the checkpoint head order.+ pub targets: Vec<ModelId>,+ /// Tensor-only router checkpoint path.+ pub checkpoint: PathBuf,+ /// Torch device override. Auto-detected when omitted.+ pub device: Option<String>,+ /// Optional Hugging Face cache directory.+ pub cache_dir: Option<PathBuf>,+ /// Maximum tokenized prompt length.+ pub max_length: usize,+ /// Maximum prompts per encoder forward pass.+ pub batch_size: usize,+}++impl PrefillRouterConfig {+ /// Creates a prefill-router config with sensible Transformers and routing defaults.+ pub fn new(targets: Vec<ModelId>, checkpoint: impl Into<PathBuf>) -> Self {+ let forward = TransformersForwardConfig::new(checkpoint);+ Self {+ targets,+ checkpoint: forward.checkpoint,+ device: forward.device,+ cache_dir: forward.cache_dir,+ max_length: forward.max_length,+ batch_size: forward.batch_size,+ }+ }++ /// Builds the libsy algorithm described by this config.+ pub fn build(self) -> Result<PrefillRouterAlgo> {+ let Self {+ targets,+ checkpoint,+ device,+ cache_dir,+ max_length,+ batch_size,+ } = self;+ let forward = TransformersForward::new(TransformersForwardConfig {+ checkpoint,+ device,+ cache_dir,+ max_length,+ batch_size,+ })?;+ PrefillRouterAlgo::from_forward(targets, forward)+ }+}++/// libsy [`Algorithm`] wrapper around a [`PrefillRouter`].+///+/// The adapter scores only the latest text user turn. With the default user-turn+/// affinity, tool continuations reuse the prior decision instead of re-running+/// prefill inference.+pub struct PrefillRouterAlgo {+ router: Arc<Mutex<PrefillRouter<AnyPrefillForward>>>,+ targets: Vec<ModelId>,+ default_target: ModelId,+ affinity: AffinityRouter,+}++impl PrefillRouterAlgo {+ pub(crate) fn from_forward(+ targets: Vec<ModelId>,+ forward: impl PrefillForward + 'static,+ ) -> Result<Self> {+ let Some(default_target) = targets.first().cloned() else {+ return Err(PrefillRouterError::InvalidConfiguration(+ "at least one target is required".to_string(),+ ));+ };+ let router = PrefillRouter::new(targets.clone(), AnyPrefillForward(Box::new(forward)))?;+ Ok(Self {+ router: Arc::new(Mutex::new(router)),+ targets,+ default_target,+ affinity: AffinityRouter::new()+ .with_message_hash_fallback()+ .with_release_on_user_turn(),+ })+ }+}++#[async_trait]+impl Algorithm for PrefillRouterAlgo {+ fn name(&self) -> &str {+ ALGORITHM_NAME+ }++ async fn route(+ self: Arc<Self>,+ _driver: Driver,+ mut request: Request,+ ) -> libsy::Result<RoutingOutcome> {+ let mut state = ();+ if let Some(target) = self+ .affinity+ .score(&mut state, &mut request, None)+ .await?+ .0+ .argmax(false)+ .map(|score| score.map(|score| score.target))?+ {+ tracing::info!(target = %target, "prefill affinity selected target");+ return Ok(RoutingOutcome::route_to(+ target.clone(),+ self.targets+ .iter()+ .filter(|candidate| *candidate != &target)+ .cloned()+ .collect(),+ request,+ ));+ }++ let target = match request+ .llm_request+ .messages+ .iter()+ .rev()+ .filter(|message| message.role == Role::User)+ .filter_map(|message| message.text_content("\n"))+ .find(|text| !text.trim().is_empty())+ {+ Some(prompt) => {+ // Wait asynchronously before occupying a blocking worker with model inference.+ let mut router = Arc::clone(&self.router).lock_owned().await;+ let predictions = tokio::task::spawn_blocking(move || {+ router.predict(&prompt).map_err(|error| error.to_string())+ })+ .await+ .map_err(|error| LibsyError::AlgorithmError {+ message: format!("prefill router prediction task failed: {error}"),+ })?+ .map_err(|message| LibsyError::AlgorithmError {+ message: format!("prefill router prediction failed: {message}"),+ })?;+ predictions+ .into_iter()+ .max_by(|(_, left), (_, right)| left.total_cmp(right))+ .map(|(target, _)| target)+ .ok_or(LibsyError::NoTargets)?+ }+ None => {+ tracing::debug!(+ target = %self.default_target,+ "prefill route found no text user message; selecting default target"+ );+ self.default_target.clone()+ }+ };+ let mut state = ();+ self.affinity+ .process(+ &mut state,+ Event::Decision {+ request: &mut request,+ selected_model_id: &target,+ },+ )+ .await?;+ tracing::info!(target = %target, "prefill router selected target");+ Ok(RoutingOutcome::route_to(+ target.clone(),+ self.targets+ .iter()+ .filter(|candidate| *candidate != &target)+ .cloned()+ .collect(),+ request,+ ))+ }+}++struct AnyPrefillForward(Box<dyn PrefillForward>);++impl PrefillForward for AnyPrefillForward {+ fn output_count(&self) -> usize {+ self.0.output_count()+ }++ fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {+ self.0.forward(prompts)+ }++ fn unload(&mut self) -> Result<()> {+ self.0.unload()+ }+}diff --git a/crates/prefill-router/src/lib.rs b/crates/prefill-router/src/lib.rsindex 04f06a421..2b1b6b34e 100644--- a/crates/prefill-router/src/lib.rs+++ b/crates/prefill-router/src/lib.rs@@ -3,10 +3,12 @@//! Learned prefill routing behind a backend-neutral forward contract.+mod algorithm;mod error;mod router;mod transformers;+pub use algorithm::{PrefillRouterAlgo, PrefillRouterConfig};pub use error::{PrefillRouterError, Result};pub use router::PrefillRouter;pub use transformers::{TransformersForward, TransformersForwardConfig};@@ -25,3 +27,7 @@ pub trait PrefillForward: Send {/// Releases resources held by the implementation.fn unload(&mut self) -> Result<()>;}++#[cfg(test)]+#[path = "../tests/unit/algorithm.rs"]+mod algorithm_tests;diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rsnew file mode 100644index 000000000..9e71759f2--- /dev/null+++ b/crates/prefill-router/tests/unit/algorithm.rs@@ -0,0 +1,154 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering};++use libsy::{Algorithm, LibsyError};+use switchyard_protocol::{+ ContentBlock, LlmRequest, Message, ModelId, Request, Role, ToolResult, text_request,+};++use crate::{PrefillForward, PrefillRouterAlgo, Result};++struct RecordingForward {+ calls: Arc<AtomicUsize>,+ prompts: Arc<std::sync::Mutex<Vec<String>>>,+}++impl PrefillForward for RecordingForward {+ fn output_count(&self) -> usize {+ 2+ }++ fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {+ self.calls.fetch_add(1, Ordering::Relaxed);+ self.prompts+ .lock()+ .expect("test prompts lock")+ .extend_from_slice(prompts);+ Ok(prompts+ .iter()+ .map(|prompt| {+ if prompt.contains("follow-up") {+ vec![0.1, 0.9]+ } else {+ vec![0.8, 0.2]+ }+ })+ .collect())+ }++ fn unload(&mut self) -> Result<()> {+ Ok(())+ }+}++fn target_set() -> Vec<ModelId> {+ vec![ModelId::from("small"), ModelId::from("large")]+}++fn forward() -> (+ RecordingForward,+ Arc<AtomicUsize>,+ Arc<std::sync::Mutex<Vec<String>>>,+) {+ let calls = Arc::new(AtomicUsize::new(0));+ let prompts = Arc::new(std::sync::Mutex::new(Vec::new()));+ (+ RecordingForward {+ calls: Arc::clone(&calls),+ prompts: Arc::clone(&prompts),+ },+ calls,+ prompts,+ )+}++async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<String> {+ let outcome = libsy::drive(route, request, |call| async move {+ call.respond(Ok(switchyard_protocol::Response {+ llm_response: switchyard_protocol::LlmResponse::Agg(+ switchyard_protocol::text_response(None, "unused"),+ ),+ metadata: None,+ }))+ })+ .await?;+ Ok(outcome.selected_model_id()?.to_string())+}++fn request(messages: Vec<Message>) -> Request {+ Request {+ llm_request: LlmRequest {+ model: Some("auto".to_string()),+ messages,+ ..LlmRequest::default()+ },+ raw_request: None,+ metadata: None,+ }+}++#[tokio::test]+async fn routes_on_latest_user_text_and_reuses_decision_for_tool_steps() -> libsy::Result<()> {+ let (forward, calls, prompts) = forward();+ let route: Arc<dyn Algorithm> = Arc::new(+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {+ LibsyError::AlgorithmError {+ message: error.to_string(),+ }+ })?,+ );++ let routed = selected(+ Arc::clone(&route),+ request(vec![+ Message::text(Role::User, "initial task"),+ Message::text(Role::Assistant, "working"),+ Message::text(Role::User, "follow-up task"),+ ]),+ )+ .await?;+ assert_eq!(routed, "large");+ assert_eq!(+ prompts.lock().expect("test prompts lock").as_slice(),+ ["follow-up task"]+ );++ let continuation = selected(+ Arc::clone(&route),+ request(vec![+ Message::text(Role::User, "initial task"),+ Message::text(Role::Assistant, "working"),+ Message::text(Role::User, "follow-up task"),+ Message::text(Role::Assistant, "calling a tool"),+ Message {+ role: Role::User,+ content: vec![ContentBlock::ToolResult(ToolResult {+ tool_call_id: "call-1".to_string(),+ content: vec![ContentBlock::Text {+ text: "tool output".to_string(),+ }],+ is_error: None,+ })],+ },+ ]),+ )+ .await?;+ assert_eq!(continuation, "large");+ assert_eq!(calls.load(Ordering::Relaxed), 1);++ let fallback = selected(+ route,+ Request {+ llm_request: text_request(Some("auto".to_string()), ""),+ raw_request: None,+ metadata: None,+ },+ )+ .await?;+ assert_eq!(fallback, "small");+ assert_eq!(calls.load(Ordering::Relaxed), 1);+ Ok(())+}