Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): configure prefill routing (#616)
nachiketb-nvidiacf958179dd1d
diff --git a/Cargo.lock b/Cargo.lockindex c72a3d297..4adb2d8dc 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2423,6 +2423,7 @@ version = "0.2.0"dependencies = ["async-trait","futures-util",+ "prefill-router","reqwest","serde","serde_json",diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.tomlindex 5da89fb60..6d55d2367 100644--- a/crates/switchyard-runner/Cargo.toml+++ b/crates/switchyard-runner/Cargo.toml@@ -13,8 +13,12 @@ repository.workspace = truerust-version.workspace = truepublish = ["crates-io"]+[features]+prefill-router = ["dep:prefill-router"]+[dependencies]libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }+prefill-router = { path = "../prefill-router", version = "0.2.0", optional = true }reqwest.workspace = trueserde.workspace = trueserde_json.workspace = truediff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex a601ab94a..d3e5f78ac 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -6,6 +6,7 @@use std::collections::BTreeMap;use std::error::Error;use std::fmt::{Display, Formatter};+use std::path::PathBuf;use std::sync::Arc;use libsy::{@@ -311,6 +312,22 @@ pub enum AlgorithmSpec {#[serde(default = "default_fail_open")]fail_open: bool,},+ /// Routes using a checkpoint-backed prefill classifier.+ PrefillRouter {+ /// Target names in checkpoint output order.+ targets: Vec<String>,+ /// Tensor-only router checkpoint path.+ checkpoint: PathBuf,+ /// PyTorch device used for encoder inference, such as `cpu`, `cuda`, or+ /// `cuda:0`. Auto-detected when omitted.+ device: Option<String>,+ /// Directory where Hugging Face caches the downloaded encoder and tokenizer.+ cache_dir: Option<PathBuf>,+ /// Maximum tokenized encoder input length; longer prompts are truncated.+ max_length: Option<usize>,+ /// Maximum prompts per encoder forward pass.+ batch_size: Option<usize>,+ },}/// What fires an advisor route's review.@@ -471,6 +488,7 @@ impl AlgorithmSpec {Self::Advisor {executor_target, ..} => vec![executor_target],+ Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(),}}@@ -1072,6 +1090,46 @@ fn build_algorithm(})?;Ok(Arc::new(algorithm))}+ AlgorithmSpec::PrefillRouter {+ targets: names,+ checkpoint,+ device,+ cache_dir,+ max_length,+ batch_size,+ } => {+ #[cfg(feature = "prefill-router")]+ {+ let targets =+ resolve_targets(route_name, names.iter().map(String::as_str), targets)?;+ let mut config = prefill_router::PrefillRouterConfig::new(targets, checkpoint);+ config.device.clone_from(device);+ config.cache_dir.clone_from(cache_dir);+ if let Some(max_length) = max_length {+ config.max_length = *max_length;+ }+ if let Some(batch_size) = batch_size {+ config.batch_size = *batch_size;+ }+ let algorithm = config.build().map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("prefill_router route {route_name}: {error}"),+ error,+ )+ })?;+ Ok(Arc::new(algorithm))+ }++ #[cfg(not(feature = "prefill-router"))]+ {+ let _ = (+ names, checkpoint, device, cache_dir, max_length, batch_size, targets,+ );+ Err(AlgorithmConfigError::new(format!(+ "prefill_router route {route_name} requires the `prefill-router` Cargo feature"+ )))+ }+ }}}diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rsindex b6b1d1827..df03c147d 100644--- a/crates/switchyard-runner/tests/route.rs+++ b/crates/switchyard-runner/tests/route.rs@@ -163,3 +163,37 @@ fn algorithm_build_reports_unknown_configured_target() {"route plugin references unknown target missing");}++// Preserve checkpoint target order and explicit TOML overrides.+#[test]+fn prefill_router_config_preserves_target_order_and_overrides() {+ let spec: AlgorithmSpec = toml::from_str(+ r#"+type = "prefill_router"+targets = ["fast", "strong"]+checkpoint = "/models/router.pt"+device = "cuda:1"+max_length = 4096+batch_size = 8+"#,+ )+ .expect("prefill router config should parse");++ let AlgorithmSpec::PrefillRouter {+ targets,+ checkpoint,+ device,+ cache_dir,+ max_length,+ batch_size,+ } = spec+ else {+ panic!("expected prefill router config");+ };+ assert_eq!(targets, ["fast", "strong"]);+ assert_eq!(checkpoint.to_string_lossy(), "/models/router.pt");+ assert_eq!(device.as_deref(), Some("cuda:1"));+ assert_eq!(cache_dir, None);+ assert_eq!(max_length, Some(4096));+ assert_eq!(batch_size, Some(8));+}diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 29cea432f..5c25b5805 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -16,6 +16,9 @@ documentation = "https://docs.rs/switchyard-server"keywords = ["llm", "proxy", "routing", "openai", "anthropic"]publish = ["crates-io"]+[features]+prefill-router = ["switchyard-runner/prefill-router"]+[dependencies]async-stream.workspace = trueasync-trait.workspace = truediff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex 3f59fd0af..0641f8308 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -73,8 +73,9 @@ upstream, and a route's `id` is the model clients send to select that algorithm.Each target references an entry under `llm_clients`. All configured clients use`TranslatingLlmClient`; supported formats are `openai_chat`, `openai_responses`, and`anthropic_messages`. Supported algorithms are `noop`, `random`, `passthrough`,-`llm_classifier`, and `stage_router`. An `api_key_env` value names an environment variable; the TOML-never contains the secret itself. If omitted, the client sends no authentication.+`llm_classifier`, and `stage_router`. The optional `prefill-router` feature also enables+`prefill_router`. An `api_key_env` value names an environment variable; the TOML never contains the+secret itself. If omitted, the client sends no authentication.A client can set `forward_auth = true` instead of `api_key_env` to send thecaller's credential to the configured upstream. OpenAI clients forward`authorization`, `chatgpt-account-id`, and `x-openai-fedramp`. Anthropic clientsdiff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 966edd978..92d2b2b75 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -141,6 +141,29 @@ Splits traffic across targets. See| `weights` | No | equal | Finite, non-negative relative weights in `targets` order, with at least one positive value. Invalid weights are rejected at load time. || `seed` | No | unset | Reproduces the selection sequence. |+### `prefill_router`++Routes the latest non-empty user message with a checkpoint-backed prefill classifier. Build+`switchyard-server` with `--features prefill-router` and make the prefill router's Python+dependencies available in the active virtual environment.++| Key | Required | Default | Meaning |+|---|:---:|---|---|+| `targets` | Yes | — | Target names in the exact order expected by the checkpoint outputs. |+| `checkpoint` | Yes | — | Path to the tensor-only router checkpoint. Relative paths use the server's working directory. |+| `device` | No | auto | PyTorch device used for encoder inference, such as `cpu`, `cuda`, or `cuda:0`. |+| `cache_dir` | No | unset | Directory where Hugging Face caches the downloaded encoder and tokenizer. |+| `max_length` | No | `2048` | Maximum tokenized encoder input length; longer prompts are truncated. |+| `batch_size` | No | `32` | Maximum prompts per encoder forward pass. |++```toml+[routes.prefill]+id = "switchyard/prefill"+type = "prefill_router"+targets = ["fast", "strong"]+checkpoint = "/models/router.pt"+```+### `llm_classifier`Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`.