Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! libsy adapter for checkpoint-backed prefill routing.
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use libsy::{
AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,
State,
};
use switchyard_protocol::{ContentBlock, Message, ModelId, Request, Role};
use tokio::sync::Mutex;
use crate::{
PrefillForward, PrefillRouter, PrefillRouterError, Result, TransformersForward,
TransformersForwardConfig,
};
const ALGORITHM_NAME: &str = "prefill_router";
/// Configuration for a libsy prefill-router algorithm.
#[derive(Clone, Debug)]
pub struct PrefillRouterConfig {
/// Ordered routing targets, matching the checkpoint head order.
pub targets: Vec<ModelId>,
/// Tensor-only router checkpoint path.
pub checkpoint: PathBuf,
/// Torch device override. Auto-detected when omitted.
pub device: Option<String>,
/// Optional Hugging Face cache directory.
pub cache_dir: Option<PathBuf>,
/// Maximum tokenized prompt length.
pub max_length: usize,
/// Maximum prompts per encoder forward pass.
pub batch_size: usize,
}
impl PrefillRouterConfig {
/// Creates a prefill-router config with sensible Transformers and routing defaults.
pub fn new(targets: Vec<ModelId>, checkpoint: impl Into<PathBuf>) -> Self {
let forward = TransformersForwardConfig::new(checkpoint);
Self {
targets,
checkpoint: forward.checkpoint,
device: forward.device,
cache_dir: forward.cache_dir,
max_length: forward.max_length,
batch_size: forward.batch_size,
}
}
/// Builds the libsy algorithm described by this config.
pub fn build(self) -> Result<PrefillRouterAlgo> {
let Self {
targets,
checkpoint,
device,
cache_dir,
max_length,
batch_size,
} = self;
let forward = TransformersForward::new(TransformersForwardConfig {
checkpoint,
device,
cache_dir,
max_length,
batch_size,
})?;
PrefillRouterAlgo::from_forward(targets, forward)
}
}
/// libsy [`Algorithm`] wrapper around a [`PrefillRouter`].
///
/// The adapter scores only the latest text user turn. With the default user-turn
/// affinity, tool continuations reuse the prior decision instead of re-running
/// prefill inference.
pub struct PrefillRouterAlgo {
router: Arc<Mutex<PrefillRouter<AnyPrefillForward>>>,
targets: Vec<ModelId>,
default_target: ModelId,
affinity: AffinityRouter,
}
impl PrefillRouterAlgo {
pub(crate) fn from_forward(
targets: Vec<ModelId>,
forward: impl PrefillForward + 'static,
) -> Result<Self> {
let Some(default_target) = targets.first().cloned() else {
return Err(PrefillRouterError::InvalidConfiguration(
"at least one target is required".to_string(),
));
};
let router = PrefillRouter::new(targets.clone(), AnyPrefillForward(Box::new(forward)))?;
Ok(Self {
router: Arc::new(Mutex::new(router)),
targets,
default_target,
affinity: AffinityRouter::new()
.with_message_hash_fallback()
.with_release_on_user_turn(),
})
}
}
/// The prompt to score: the newest user message that has text of its own.
/// A user message with no text (whitespace only, or an attachment with no
/// caption) is skipped, but once one has been skipped the search stops at the
/// first assistant message, so an earlier exchange's prompt is never scored as
/// the current request. A user message carrying tool results is tool activity
/// inside the current exchange, not a skipped turn, and is walked past as
/// before. `None` leaves the choice to the default target.
fn latest_user_prompt(messages: &[Message]) -> Option<String> {
let mut skipped = false;
for message in messages.iter().rev() {
if message.role == Role::User {
if let Some(text) = message
.text_content("\n")
.filter(|text| !text.trim().is_empty())
{
return Some(text);
}
if !message
.content
.iter()
.any(|block| matches!(block, ContentBlock::ToolResult(_)))
{
skipped = true;
}
} else if message.role == Role::Assistant && skipped {
return None;
}
}
None
}
#[async_trait]
impl Algorithm for PrefillRouterAlgo {
fn name(&self) -> &str {
ALGORITHM_NAME
}
async fn route(
self: Arc<Self>,
driver: Driver,
mut request: Request,
) -> libsy::Result<RoutingOutcome> {
let mut state = State::default();
if let Some(target) = self
.affinity
.score(&mut state, &mut request, &driver)
.await?
.0
.argmax(false)
.map(|score| score.map(|score| score.target))?
{
tracing::info!(target = %target, "prefill affinity selected target");
return Ok(RoutingOutcome::route_to(
target.clone(),
self.targets
.iter()
.filter(|candidate| *candidate != &target)
.cloned()
.collect(),
request,
));
}
let target = match latest_user_prompt(&request.llm_request.messages) {
Some(prompt) => {
// Wait asynchronously before occupying a blocking worker with model inference.
let mut router = Arc::clone(&self.router).lock_owned().await;
let predictions = tokio::task::spawn_blocking(move || {
router.predict(&prompt).map_err(|error| error.to_string())
})
.await
.map_err(|error| LibsyError::AlgorithmError {
message: format!("prefill router prediction task failed: {error}"),
})?
.map_err(|message| LibsyError::AlgorithmError {
message: format!("prefill router prediction failed: {message}"),
})?;
predictions
.into_iter()
.max_by(|(_, left), (_, right)| left.total_cmp(right))
.map(|(target, _)| target)
.ok_or(LibsyError::NoTargets)?
}
None => {
tracing::debug!(
target = %self.default_target,
"prefill route found no text user message; selecting default target"
);
self.default_target.clone()
}
};
let mut state = State::default();
self.affinity
.process(
&mut state,
Event::Decision {
request: &mut request,
selected_model_id: &target,
category: None,
driver: &driver,
},
)
.await?;
tracing::info!(target = %target, "prefill router selected target");
Ok(RoutingOutcome::route_to(
target.clone(),
self.targets
.iter()
.filter(|candidate| *candidate != &target)
.cloned()
.collect(),
request,
))
}
}
struct AnyPrefillForward(Box<dyn PrefillForward>);
impl PrefillForward for AnyPrefillForward {
fn output_count(&self) -> usize {
self.0.output_count()
}
fn forward(&mut self, prompts: &[String]) -> Result<Vec<Vec<f32>>> {
self.0.forward(prompts)
}
fn unload(&mut self) -> Result<()> {
self.0.unload()
}
}