Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): configure prefill routing (#616)

nachiketb-nvidiacf958179dd1d
diff --git a/Cargo.lock b/Cargo.lock
index c72a3d297..4adb2d8dc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2423,6 +2423,7 @@ version = "0.2.0"
dependencies = [
"async-trait",
"futures-util",
+ "prefill-router",
"reqwest",
"serde",
"serde_json",
diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml
index 5da89fb60..6d55d2367 100644
--- a/crates/switchyard-runner/Cargo.toml
+++ b/crates/switchyard-runner/Cargo.toml
@@ -13,8 +13,12 @@ repository.workspace = true
rust-version.workspace = true
publish = ["crates-io"]
+[features]
+prefill-router = ["dep:prefill-router"]
+
[dependencies]
libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }
+prefill-router = { path = "../prefill-router", version = "0.2.0", optional = true }
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index a601ab94a..d3e5f78ac 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -6,6 +6,7 @@
use std::collections::BTreeMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
+use std::path::PathBuf;
use std::sync::Arc;
use libsy::{
@@ -311,6 +312,22 @@ pub enum AlgorithmSpec {
#[serde(default = "default_fail_open")]
fail_open: bool,
},
+ /// Routes using a checkpoint-backed prefill classifier.
+ PrefillRouter {
+ /// Target names in checkpoint output order.
+ targets: Vec<String>,
+ /// Tensor-only router checkpoint path.
+ checkpoint: PathBuf,
+ /// PyTorch device used for encoder inference, such as `cpu`, `cuda`, or
+ /// `cuda:0`. Auto-detected when omitted.
+ device: Option<String>,
+ /// Directory where Hugging Face caches the downloaded encoder and tokenizer.
+ cache_dir: Option<PathBuf>,
+ /// Maximum tokenized encoder input length; longer prompts are truncated.
+ max_length: Option<usize>,
+ /// Maximum prompts per encoder forward pass.
+ batch_size: Option<usize>,
+ },
}
/// What fires an advisor route's review.
@@ -471,6 +488,7 @@ impl AlgorithmSpec {
Self::Advisor {
executor_target, ..
} => vec![executor_target],
+ Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(),
}
}
@@ -1072,6 +1090,46 @@ fn build_algorithm(
})?;
Ok(Arc::new(algorithm))
}
+ AlgorithmSpec::PrefillRouter {
+ targets: names,
+ checkpoint,
+ device,
+ cache_dir,
+ max_length,
+ batch_size,
+ } => {
+ #[cfg(feature = "prefill-router")]
+ {
+ let targets =
+ resolve_targets(route_name, names.iter().map(String::as_str), targets)?;
+ let mut config = prefill_router::PrefillRouterConfig::new(targets, checkpoint);
+ config.device.clone_from(device);
+ config.cache_dir.clone_from(cache_dir);
+ if let Some(max_length) = max_length {
+ config.max_length = *max_length;
+ }
+ if let Some(batch_size) = batch_size {
+ config.batch_size = *batch_size;
+ }
+ let algorithm = config.build().map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("prefill_router route {route_name}: {error}"),
+ error,
+ )
+ })?;
+ Ok(Arc::new(algorithm))
+ }
+
+ #[cfg(not(feature = "prefill-router"))]
+ {
+ let _ = (
+ names, checkpoint, device, cache_dir, max_length, batch_size, targets,
+ );
+ Err(AlgorithmConfigError::new(format!(
+ "prefill_router route {route_name} requires the `prefill-router` Cargo feature"
+ )))
+ }
+ }
}
}
diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs
index b6b1d1827..df03c147d 100644
--- a/crates/switchyard-runner/tests/route.rs
+++ b/crates/switchyard-runner/tests/route.rs
@@ -163,3 +163,37 @@ fn algorithm_build_reports_unknown_configured_target() {
"route plugin references unknown target missing"
);
}
+
+// Preserve checkpoint target order and explicit TOML overrides.
+#[test]
+fn prefill_router_config_preserves_target_order_and_overrides() {
+ let spec: AlgorithmSpec = toml::from_str(
+ r#"
+type = "prefill_router"
+targets = ["fast", "strong"]
+checkpoint = "/models/router.pt"
+device = "cuda:1"
+max_length = 4096
+batch_size = 8
+"#,
+ )
+ .expect("prefill router config should parse");
+
+ let AlgorithmSpec::PrefillRouter {
+ targets,
+ checkpoint,
+ device,
+ cache_dir,
+ max_length,
+ batch_size,
+ } = spec
+ else {
+ panic!("expected prefill router config");
+ };
+ assert_eq!(targets, ["fast", "strong"]);
+ assert_eq!(checkpoint.to_string_lossy(), "/models/router.pt");
+ assert_eq!(device.as_deref(), Some("cuda:1"));
+ assert_eq!(cache_dir, None);
+ assert_eq!(max_length, Some(4096));
+ assert_eq!(batch_size, Some(8));
+}
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 29cea432f..5c25b5805 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -16,6 +16,9 @@ documentation = "https://docs.rs/switchyard-server"
keywords = ["llm", "proxy", "routing", "openai", "anthropic"]
publish = ["crates-io"]
+[features]
+prefill-router = ["switchyard-runner/prefill-router"]
+
[dependencies]
async-stream.workspace = true
async-trait.workspace = true
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 3f59fd0af..0641f8308 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -73,8 +73,9 @@ upstream, and a route's `id` is the model clients send to select that algorithm.
Each target references an entry under `llm_clients`. All configured clients use
`TranslatingLlmClient`; supported formats are `openai_chat`, `openai_responses`, and
`anthropic_messages`. Supported algorithms are `noop`, `random`, `passthrough`,
-`llm_classifier`, and `stage_router`. An `api_key_env` value names an environment variable; the TOML
-never contains the secret itself. If omitted, the client sends no authentication.
+`llm_classifier`, and `stage_router`. The optional `prefill-router` feature also enables
+`prefill_router`. An `api_key_env` value names an environment variable; the TOML never contains the
+secret itself. If omitted, the client sends no authentication.
A client can set `forward_auth = true` instead of `api_key_env` to send the
caller's credential to the configured upstream. OpenAI clients forward
`authorization`, `chatgpt-account-id`, and `x-openai-fedramp`. Anthropic clients
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 966edd978..92d2b2b75 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -141,6 +141,29 @@ Splits traffic across targets. See
| `weights` | No | equal | Finite, non-negative relative weights in `targets` order, with at least one positive value. Invalid weights are rejected at load time. |
| `seed` | No | unset | Reproduces the selection sequence. |
+### `prefill_router`
+
+Routes the latest non-empty user message with a checkpoint-backed prefill classifier. Build
+`switchyard-server` with `--features prefill-router` and make the prefill router's Python
+dependencies available in the active virtual environment.
+
+| Key | Required | Default | Meaning |
+|---|:---:|---|---|
+| `targets` | Yes | — | Target names in the exact order expected by the checkpoint outputs. |
+| `checkpoint` | Yes | — | Path to the tensor-only router checkpoint. Relative paths use the server's working directory. |
+| `device` | No | auto | PyTorch device used for encoder inference, such as `cpu`, `cuda`, or `cuda:0`. |
+| `cache_dir` | No | unset | Directory where Hugging Face caches the downloaded encoder and tokenizer. |
+| `max_length` | No | `2048` | Maximum tokenized encoder input length; longer prompts are truncated. |
+| `batch_size` | No | `32` | Maximum prompts per encoder forward pass. |
+
+```toml
+[routes.prefill]
+id = "switchyard/prefill"
+type = "prefill_router"
+targets = ["fast", "strong"]
+checkpoint = "/models/router.pt"
+```
+
### `llm_classifier`
Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`.