Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(prefill-router): add libsy algorithm wrapper (#593)

nachiketb-nvidiabbaa05c6aab5
diff --git a/Cargo.lock b/Cargo.lock
index 0209f7f89..c72a3d297 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1541,10 +1541,13 @@ dependencies = [
name = "prefill-router"
version = "0.2.0"
dependencies = [
+ "async-trait",
"pyo3",
"pyo3-build-config",
+ "switchyard-libsy",
"switchyard-protocol",
"thiserror 2.0.18",
+ "tokio",
"tracing",
]
diff --git a/crates/prefill-router/Cargo.toml b/crates/prefill-router/Cargo.toml
index df4c5a3b9..9fe229cb7 100644
--- a/crates/prefill-router/Cargo.toml
+++ b/crates/prefill-router/Cargo.toml
@@ -12,9 +12,12 @@ repository.workspace = true
rust-version.workspace = true
[dependencies]
+async-trait.workspace = true
+libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }
pyo3 = { version = "0.28.3", features = ["auto-initialize"] }
switchyard-protocol.workspace = true
thiserror.workspace = true
+tokio.workspace = true
tracing.workspace = true
[build-dependencies]
diff --git a/crates/prefill-router/src/algorithm.rs b/crates/prefill-router/src/algorithm.rs
new file mode 100644
index 000000000..f0979c96a
--- /dev/null
+++ b/crates/prefill-router/src/algorithm.rs
@@ -0,0 +1,214 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! libsy adapter for checkpoint-backed prefill routing.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use libsy::{
+ AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,
+};
+use switchyard_protocol::{ModelId, Request, Role};
+use tokio::sync::Mutex;
+
+use crate::{
+ PrefillForward, PrefillRouter, PrefillRouterError, Result, TransformersForward,
+ TransformersForwardConfig,
+};
+
+const ALGORITHM_NAME: &str = "prefill_router";
+
+/// Configuration for a libsy prefill-router algorithm.
+#[derive(Clone, Debug)]
+pub struct PrefillRouterConfig {
+ /// Ordered routing targets, matching the checkpoint head order.
+ pub targets: Vec<ModelId>,
+ /// Tensor-only router checkpoint path.
+ pub checkpoint: PathBuf,
+ /// Torch device override. Auto-detected when omitted.
+ pub device: Option<String>,
+ /// Optional Hugging Face cache directory.
+ pub cache_dir: Option<PathBuf>,
+ /// Maximum tokenized prompt length.
+ pub max_length: usize,
+ /// Maximum prompts per encoder forward pass.
+ pub batch_size: usize,
+}
+
+impl PrefillRouterConfig {
+ /// Creates a prefill-router config with sensible Transformers and routing defaults.
+ pub fn new(targets: Vec<ModelId>, checkpoint: impl Into<PathBuf>) -> Self {
+ let forward = TransformersForwardConfig::new(checkpoint);
+ Self {
+ targets,
+ checkpoint: forward.checkpoint,
+ device: forward.device,
+ cache_dir: forward.cache_dir,
+ max_length: forward.max_length,
+ batch_size: forward.batch_size,
+ }
+ }
+
+ /// Builds the libsy algorithm described by this config.
+ pub fn build(self) -> Result<PrefillRouterAlgo> {
+ let Self {
+ targets,
+ checkpoint,
+ device,
+ cache_dir,
+ max_length,
+ batch_size,
+ } = self;
+ let forward = TransformersForward::new(TransformersForwardConfig {
+ checkpoint,
+ device,
+ cache_dir,
+ max_length,
+ batch_size,
+ })?;
+ PrefillRouterAlgo::from_forward(targets, forward)
+ }
+}
+
+/// libsy [`Algorithm`] wrapper around a [`PrefillRouter`].
+///
+/// The adapter scores only the latest text user turn. With the default user-turn
+/// affinity, tool continuations reuse the prior decision instead of re-running
+/// prefill inference.
+pub struct PrefillRouterAlgo {
+ router: Arc<Mutex<PrefillRouter<AnyPrefillForward>>>,
+ targets: Vec<ModelId>,
+ default_target: ModelId,
+ affinity: AffinityRouter,
+}
+
+impl PrefillRouterAlgo {
+ pub(crate) fn from_forward(
+ targets: Vec<ModelId>,
+ forward: impl PrefillForward + 'static,
+ ) -> Result<Self> {
+ let Some(default_target) = targets.first().cloned() else {
+ return Err(PrefillRouterError::InvalidConfiguration(
+ "at least one target is required".to_string(),
+ ));
+ };
+ let router = PrefillRouter::new(targets.clone(), AnyPrefillForward(Box::new(forward)))?;
+ Ok(Self {
+ router: Arc::new(Mutex::new(router)),
+ targets,
+ default_target,
+ affinity: AffinityRouter::new()
+ .with_message_hash_fallback()
+ .with_release_on_user_turn(),
+ })
+ }
+}
+
+#[async_trait]
+impl Algorithm for PrefillRouterAlgo {
+ fn name(&self) -> &str {
+ ALGORITHM_NAME
+ }
+
+ async fn route(
+ self: Arc<Self>,
+ _driver: Driver,
+ mut request: Request,
+ ) -> libsy::Result<RoutingOutcome> {
+ let mut state = ();
+ if let Some(target) = self
+ .affinity
+ .score(&mut state, &mut request, None)
+ .await?
+ .0
+ .argmax(false)
+ .map(|score| score.map(|score| score.target))?
+ {
+ tracing::info!(target = %target, "prefill affinity selected target");
+ return Ok(RoutingOutcome::route_to(
+ target.clone(),
+ self.targets
+ .iter()
+ .filter(|candidate| *candidate != &target)
+ .cloned()
+ .collect(),
+ request,
+ ));
+ }
+
+ let target = match request
+ .llm_request
+ .messages
+ .iter()
+ .rev()
+ .filter(|message| message.role == Role::User)
+ .filter_map(|message| message.text_content("\n"))
+ .find(|text| !text.trim().is_empty())
+ {
+ Some(prompt) => {
+ // Wait asynchronously before occupying a blocking worker with model inference.
+ let mut router = Arc::clone(&self.router).lock_owned().await;
+ let predictions = tokio::task::spawn_blocking(move || {
+ router.predict(&prompt).map_err(|error| error.to_string())
+ })
+ .await
+ .map_err(|error| LibsyError::AlgorithmError {
+ message: format!("prefill router prediction task failed: {error}"),
+ })?
+ .map_err(|message| LibsyError::AlgorithmError {
+ message: format!("prefill router prediction failed: {message}"),
+ })?;
+ predictions
+ .into_iter()
+ .max_by(|(_, left), (_, right)| left.total_cmp(right))
+ .map(|(target, _)| target)
+ .ok_or(LibsyError::NoTargets)?
+ }
+ None => {
+ tracing::debug!(
+ target = %self.default_target,
+ "prefill route found no text user message; selecting default target"
+ );
+ self.default_target.clone()
+ }
+ };
+ let mut state = ();
+ self.affinity
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &mut request,
+ selected_model_id: &target,
+ },
+ )
+ .await?;
+ tracing::info!(target = %target, "prefill router selected target");
+ Ok(RoutingOutcome::route_to(
+ target.clone(),
+ self.targets
+ .iter()
+ .filter(|candidate| *candidate != &target)
+ .cloned()
+ .collect(),
+ request,
+ ))
+ }
+}
+
+struct AnyPrefillForward(Box<dyn PrefillForward>);
+
+impl PrefillForward for AnyPrefillForward {
+ fn output_count(&self) -> usize {
+ self.0.output_count()
+ }
+
+ fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {
+ self.0.forward(prompts)
+ }
+
+ fn unload(&mut self) -> Result<()> {
+ self.0.unload()
+ }
+}
diff --git a/crates/prefill-router/src/lib.rs b/crates/prefill-router/src/lib.rs
index 04f06a421..2b1b6b34e 100644
--- a/crates/prefill-router/src/lib.rs
+++ b/crates/prefill-router/src/lib.rs
@@ -3,10 +3,12 @@
//! Learned prefill routing behind a backend-neutral forward contract.
+mod algorithm;
mod error;
mod router;
mod transformers;
+pub use algorithm::{PrefillRouterAlgo, PrefillRouterConfig};
pub use error::{PrefillRouterError, Result};
pub use router::PrefillRouter;
pub use transformers::{TransformersForward, TransformersForwardConfig};
@@ -25,3 +27,7 @@ pub trait PrefillForward: Send {
/// Releases resources held by the implementation.
fn unload(&mut self) -> Result<()>;
}
+
+#[cfg(test)]
+#[path = "../tests/unit/algorithm.rs"]
+mod algorithm_tests;
diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rs
new file mode 100644
index 000000000..9e71759f2
--- /dev/null
+++ b/crates/prefill-router/tests/unit/algorithm.rs
@@ -0,0 +1,154 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use libsy::{Algorithm, LibsyError};
+use switchyard_protocol::{
+ ContentBlock, LlmRequest, Message, ModelId, Request, Role, ToolResult, text_request,
+};
+
+use crate::{PrefillForward, PrefillRouterAlgo, Result};
+
+struct RecordingForward {
+ calls: Arc<AtomicUsize>,
+ prompts: Arc<std::sync::Mutex<Vec<String>>>,
+}
+
+impl PrefillForward for RecordingForward {
+ fn output_count(&self) -> usize {
+ 2
+ }
+
+ fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {
+ self.calls.fetch_add(1, Ordering::Relaxed);
+ self.prompts
+ .lock()
+ .expect("test prompts lock")
+ .extend_from_slice(prompts);
+ Ok(prompts
+ .iter()
+ .map(|prompt| {
+ if prompt.contains("follow-up") {
+ vec![0.1, 0.9]
+ } else {
+ vec![0.8, 0.2]
+ }
+ })
+ .collect())
+ }
+
+ fn unload(&mut self) -> Result<()> {
+ Ok(())
+ }
+}
+
+fn target_set() -> Vec<ModelId> {
+ vec![ModelId::from("small"), ModelId::from("large")]
+}
+
+fn forward() -> (
+ RecordingForward,
+ Arc<AtomicUsize>,
+ Arc<std::sync::Mutex<Vec<String>>>,
+) {
+ let calls = Arc::new(AtomicUsize::new(0));
+ let prompts = Arc::new(std::sync::Mutex::new(Vec::new()));
+ (
+ RecordingForward {
+ calls: Arc::clone(&calls),
+ prompts: Arc::clone(&prompts),
+ },
+ calls,
+ prompts,
+ )
+}
+
+async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<String> {
+ let outcome = libsy::drive(route, request, |call| async move {
+ call.respond(Ok(switchyard_protocol::Response {
+ llm_response: switchyard_protocol::LlmResponse::Agg(
+ switchyard_protocol::text_response(None, "unused"),
+ ),
+ metadata: None,
+ }))
+ })
+ .await?;
+ Ok(outcome.selected_model_id()?.to_string())
+}
+
+fn request(messages: Vec<Message>) -> Request {
+ Request {
+ llm_request: LlmRequest {
+ model: Some("auto".to_string()),
+ messages,
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: None,
+ }
+}
+
+#[tokio::test]
+async fn routes_on_latest_user_text_and_reuses_decision_for_tool_steps() -> libsy::Result<()> {
+ let (forward, calls, prompts) = forward();
+ let route: Arc<dyn Algorithm> = Arc::new(
+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {
+ LibsyError::AlgorithmError {
+ message: error.to_string(),
+ }
+ })?,
+ );
+
+ let routed = selected(
+ Arc::clone(&route),
+ request(vec![
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::Assistant, "working"),
+ Message::text(Role::User, "follow-up task"),
+ ]),
+ )
+ .await?;
+ assert_eq!(routed, "large");
+ assert_eq!(
+ prompts.lock().expect("test prompts lock").as_slice(),
+ ["follow-up task"]
+ );
+
+ let continuation = selected(
+ Arc::clone(&route),
+ request(vec![
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::Assistant, "working"),
+ Message::text(Role::User, "follow-up task"),
+ Message::text(Role::Assistant, "calling a tool"),
+ Message {
+ role: Role::User,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "call-1".to_string(),
+ content: vec![ContentBlock::Text {
+ text: "tool output".to_string(),
+ }],
+ is_error: None,
+ })],
+ },
+ ]),
+ )
+ .await?;
+ assert_eq!(continuation, "large");
+ assert_eq!(calls.load(Ordering::Relaxed), 1);
+
+ let fallback = selected(
+ route,
+ Request {
+ llm_request: text_request(Some("auto".to_string()), ""),
+ raw_request: None,
+ metadata: None,
+ },
+ )
+ .await?;
+ assert_eq!(fallback, "small");
+ assert_eq!(calls.load(Ordering::Relaxed), 1);
+ Ok(())
+}