Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: Extract from switchyard-server into switchyard-runner (#517)

Graham King8888492a0e5e
diff --git a/Cargo.lock b/Cargo.lock
index a5093f441..fde36135f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2343,6 +2343,24 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "switchyard-runner"
+version = "0.2.0"
+dependencies = [
+ "async-trait",
+ "futures-util",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "switchyard-libsy",
+ "switchyard-llm-client",
+ "switchyard-protocol",
+ "thiserror 2.0.18",
+ "tokio",
+ "toml",
+ "tracing",
+]
+
[[package]]
name = "switchyard-server"
version = "0.2.0"
@@ -2362,17 +2380,16 @@ dependencies = [
"opentelemetry_sdk",
"parking_lot",
"prometheus",
- "reqwest",
"rustls",
"serde",
"serde_json",
"switchyard-libsy",
"switchyard-llm-client",
"switchyard-protocol",
+ "switchyard-runner",
"switchyard-translation",
"tempfile",
"tokio",
- "toml",
"tower",
"tracing",
"tracing-opentelemetry",
diff --git a/Cargo.toml b/Cargo.toml
index 0fc5030a7..c1b8d1a94 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
"crates/libsy-llm-client",
"crates/switchyard-py",
"crates/protocol",
+ "crates/switchyard-runner",
"crates/switchyard-server",
"crates/switchyard-skill-distillation",
"crates/switchyard-soak",
@@ -40,6 +41,7 @@ serde_json = { version = "1", features = ["preserve_order"] }
switchyard-libsy = { path = "crates/libsy", version = "0.2.0" }
switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" }
switchyard-protocol = { path = "crates/protocol", version = "0.2.0" }
+switchyard-runner = { path = "crates/switchyard-runner", version = "0.2.0" }
switchyard-server = { path = "crates/switchyard-server", version = "0.2.0" }
switchyard-translation = { path = "crates/switchyard-translation", version = "0.2.0" }
thiserror = "2"
diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rs
index 12b4395d6..1005d49f6 100644
--- a/crates/switchyard-py/src/server_bindings.rs
+++ b/crates/switchyard-py/src/server_bindings.rs
@@ -44,13 +44,8 @@ impl PyServer {
.map_err(|error| ServerConfigError::new_err(error.to_string()))?;
let caller_auth_by_model = state
.models()
- .map(|model| {
- state
- .caller_auth_kind(model)
- .map(|kind| (model.to_string(), kind))
- })
- .collect::<ServerResult<HashMap<_, _>>>()
- .map_err(server_error)?;
+ .map(|model| (model.to_string(), state.caller_auth_kind(model)))
+ .collect::<HashMap<_, _>>();
let runtime = pyo3_async_runtimes::tokio::get_runtime();
let server = {
let _guard = runtime.enter();
diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml
new file mode 100644
index 000000000..5545f2c19
--- /dev/null
+++ b/crates/switchyard-runner/Cargo.toml
@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "switchyard-runner"
+version.workspace = true
+description = "Configured routing execution for Switchyard serving surfaces"
+authors.workspace = true
+edition.workspace = true
+homepage = "https://github.com/NVIDIA-NeMo/Switchyard"
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+publish = ["crates-io"]
+
+[dependencies]
+libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }
+reqwest.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+switchyard-llm-client.workspace = true
+switchyard-protocol.workspace = true
+thiserror.workspace = true
+toml = "1.1"
+tracing.workspace = true
+
+[dev-dependencies]
+async-trait.workspace = true
+futures-util.workspace = true
+tokio.workspace = true
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
new file mode 100644
index 000000000..b4d400d74
--- /dev/null
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -0,0 +1,1063 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Schema-neutral algorithm configuration and construction.
+
+use std::collections::BTreeMap;
+use std::error::Error;
+use std::fmt::{Display, Formatter};
+use std::sync::Arc;
+
+use libsy::{
+ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,
+ ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig,
+ GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop,
+ Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter,
+ SubagentRouterConfig, TargetPrompts, TaskClassifierConfig,
+};
+use serde::Deserialize;
+use switchyard_protocol::ModelId;
+
+/// Error returned when an algorithm description cannot be constructed.
+#[derive(Debug)]
+pub struct AlgorithmConfigError {
+ message: String,
+ source: Option<Box<dyn Error + Send + Sync>>,
+}
+
+impl AlgorithmConfigError {
+ fn new(message: impl Into<String>) -> Self {
+ Self {
+ message: message.into(),
+ source: None,
+ }
+ }
+
+ fn with_source(message: impl Into<String>, source: impl Error + Send + Sync + 'static) -> Self {
+ Self {
+ message: message.into(),
+ source: Some(Box::new(source)),
+ }
+ }
+}
+
+impl Display for AlgorithmConfigError {
+ fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str(&self.message)
+ }
+}
+
+impl Error for AlgorithmConfigError {
+ fn source(&self) -> Option<&(dyn Error + 'static)> {
+ self.source.as_deref().map(|source| source as _)
+ }
+}
+
+type AlgorithmResult<T> = Result<T, AlgorithmConfigError>;
+
+/// How a custom classifier turns the judge's JSON verdict into a target.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum ClassifierPolicyConfig {
+ /// Reads the target name straight out of the judge's verdict.
+ TargetSelector {
+ /// JSON Pointer to the name, such as `/decision/target`.
+ selector: String,
+ },
+}
+
+/// Which of the three `llm_classifier` behaviors a route uses.
+#[derive(Clone, Copy, Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ClassifierMode {
+ /// Judges the request first, then serves the strong or weak target.
+ Capability,
+ /// Serves the weak target first and judges the finished turn, moving to the
+ /// strong target once the session latches.
+ Escalation,
+ /// Judges against your own JSON schema and routes to any configured target.
+ Custom,
+}
+
+impl ClassifierPolicyConfig {
+ fn into_libsy(self) -> CustomClassifierPolicy {
+ match self {
+ Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector),
+ }
+ }
+}
+
+#[derive(Clone, Debug)]
+enum LlmClassifierModeConfig {
+ Capability(CapabilityClassifierRouteConfig),
+ Escalation(EscalationClassifierRouteConfig),
+ Custom(CustomClassifierRouteConfig),
+}
+
+#[derive(Clone, Debug)]
+struct CapabilityClassifierRouteConfig {
+ strong_target: String,
+ weak_target: String,
+ base_threshold: f64,
+ threshold_step: f64,
+ classify_trigger: ClassifyTrigger,
+ message_hash_fallback: bool,
+ recent_turn_window: Option<usize>,
+ prompt: Option<String>,
+ response_format_type: ClassifierResponseFormat,
+ max_output_tokens: u64,
+}
+
+#[derive(Clone, Debug)]
+struct EscalationClassifierRouteConfig {
+ strong_target: String,
+ weak_target: String,
+ prompt: Option<String>,
+ response_format_type: ClassifierResponseFormat,
+ max_output_tokens: u64,
+ judge: EscalationJudgeConfig,
+}
+
+#[derive(Clone, Debug)]
+struct CustomClassifierRouteConfig {
+ classifier_target: String,
+ targets: Vec<String>,
+ default_target: String,
+ prompt: String,
+ response_schema: String,
+ policy: ClassifierPolicyConfig,
+ classify_trigger: ClassifyTrigger,
+ message_hash_fallback: bool,
+ recent_turn_window: Option<usize>,
+ max_output_tokens: u64,
+}
+
+/// Settings for an `llm_classifier` route. Which fields are required depends on
+/// the [`ClassifierMode`]; using a field from the wrong mode is an error.
+#[derive(Clone, Debug, Default, Deserialize)]
+#[serde(default, deny_unknown_fields)]
+pub struct LlmClassifierRouteConfig {
+ /// Target the judge is called through. Never a routing destination itself.
+ pub classifier_target: String,
+ /// Mode to run. Defaults to escalation when `escalation` is set, otherwise capability.
+ pub mode: Option<ClassifierMode>,
+ /// Capability and escalation modes: the capable tier.
+ pub strong_target: Option<String>,
+ /// Capability and escalation modes: the efficient tier.
+ pub weak_target: Option<String>,
+ /// Capability mode: lowest solve probability that still routes to the weak
+ /// target, from 0 to 1.
+ pub base_threshold: Option<f64>,
+ /// Capability mode: how much to raise the threshold when the judge is
+ /// uncertain. Added once for an uncertain verdict and twice for unsupported.
+ pub threshold_step: Option<f64>,
+ /// How often the judge runs: every request, once per user turn, or once per session.
+ pub classify_trigger: ClassifyTrigger,
+ /// Reuses the session's target by hashing the first user message when no
+ /// session ID is available. Needs `classify_trigger = "new_session"`.
+ pub message_hash_fallback: bool,
+ /// How many trailing turns the judge sees. Unset shows it the opening task
+ /// and the latest user follow-up only.
+ pub recent_turn_window: Option<usize>,
+ /// Replaces the packaged judge prompt. Required in custom mode.
+ pub prompt: Option<String>,
+ /// How the judge is asked for structured output. Use `json_object` when the
+ /// provider cannot do JSON Schema.
+ pub response_format_type: ClassifierResponseFormat,
+ /// Most completion tokens the judge verdict may use.
+ #[serde(default = "default_classifier_max_output_tokens")]
+ pub max_output_tokens: u64,
+ /// Escalation mode: how many escalate verdicts latch the session, and how
+ /// much of the transcript the judge sees.
+ pub escalation: Option<EscalationJudgeConfig>,
+ /// Custom mode: the target names the policy may pick from.
+ pub targets: Option<Vec<String>>,
+ /// Custom mode: target used when the judge fails or its verdict cannot be routed.
+ pub default_target: Option<String>,
+ /// Custom mode: JSON Schema the verdict must match, written as a string.
+ pub response_schema: Option<String>,
+ /// Custom mode: how to read the chosen target out of the verdict.
+ pub policy: Option<ClassifierPolicyConfig>,
+}
+
+/// Routing policy applied only to delegated sub-agent work, nested inside a
+/// `passthrough` or `stage_router` route.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum SubagentRouteConfig {
+ /// Sends all sub-agent work to one target.
+ Passthrough {
+ /// Target that serves sub-agent requests.
+ target: String,
+ },
+ /// Judges each sub-agent request. Only [`ClassifierMode::Custom`] is supported here.
+ LlmClassifier(Box<LlmClassifierRouteConfig>),
+}
+
+impl SubagentRouteConfig {
+ fn routing_target_names(&self) -> Vec<&str> {
+ match self {
+ Self::Passthrough { target } => vec![target],
+ Self::LlmClassifier(classifier) => classifier
+ .targets
+ .iter()
+ .flatten()
+ .map(String::as_str)
+ .collect(),
+ }
+ }
+
+ fn classifier_target_name(&self) -> Option<&str> {
+ match self {
+ Self::LlmClassifier(classifier) => Some(&classifier.classifier_target),
+ Self::Passthrough { .. } => None,
+ }
+ }
+}
+
+/// A routing algorithm described by configured target names.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+pub enum AlgorithmSpec {
+ /// Replies `OK` without calling any model. Useful for smoke tests.
+ Noop {},
+ /// Splits traffic across several targets.
+ Random {
+ /// Target names to choose from.
+ targets: Vec<String>,
+ /// Relative weights in `targets` order. Equal weights when unset.
+ weights: Option<Vec<f64>>,
+ /// Makes the sequence of choices repeatable.
+ seed: Option<u64>,
+ },
+ /// Sends every request to one target.
+ Passthrough {
+ /// Target that serves the request.
+ target: String,
+ /// Separate policy for delegated sub-agent work.
+ #[serde(default)]
+ subagents: Option<SubagentRouteConfig>,
+ },
+ /// Asks a judge model which target should serve the request.
+ LlmClassifier {
+ /// Judge and tier settings, written directly in the route table.
+ #[serde(flatten)]
+ config: LlmClassifierRouteConfig,
+ },
+ /// Picks a tier per turn by scoring signals from recent tool results.
+ StageRouter {
+ /// The capable tier.
+ capable_target: String,
+ /// The efficient tier.
+ efficient_target: String,
+ /// Tier to use when the signals are not confident.
+ picker: PickerMode,
+ /// How much agreement a decisive pick needs, from 0 to 1.
+ confidence_threshold: f64,
+ /// How many trailing tool results the signals are scored over.
+ #[serde(default)]
+ recent_turn_window: Option<usize>,
+ /// Notes handed to a tier when the router switches to it.
+ #[serde(default)]
+ handoff_notes: Option<HandoffNoteConfig>,
+ /// System prompt handed to the capable tier.
+ #[serde(default)]
+ capable_system_prompt: Option<String>,
+ /// System prompt handed to the efficient tier.
+ #[serde(default)]
+ efficient_system_prompt: Option<String>,
+ /// Judge consulted for turns the tool signals cannot decide.
+ #[serde(default)]
+ classifier: Option<StageClassifierConfig>,
+ /// Separate policy for delegated sub-agent work.
+ #[serde(default)]
+ subagents: Option<SubagentRouteConfig>,
+ },
+ /// Serves every turn from one target, and has a second model review some of
+ /// those turns before the caller sees them.
+ Advisor {
+ /// Serves every client-visible turn.
+ executor_target: String,
+ /// Reviews gated turns. Never a routing destination.
+ advisor_target: String,
+ /// Replaces the built-in APPROVE/REDO reviewer prompt.
+ #[serde(default)]
+ reviewer_system_prompt: Option<String>,
+ /// Replaces the built-in text put in front of a REDO plan.
+ #[serde(default)]
+ redo_feedback_prefix: Option<String>,
+ /// What fires a review.
+ #[serde(default)]
+ gate_trigger: AdvisorTriggerConfig,
+ /// Regular expression for the `pattern` trigger. Required by it, and unused otherwise.
+ #[serde(default)]
+ gate_trigger_pattern: Option<String>,
+ /// How many reviews one session may spend.
+ #[serde(default = "default_max_reviews")]
+ max_reviews: u32,
+ /// Reviews a turn after this many assistant turns, as a mid-task
+ /// checkpoint. Zero turns the checkpoint off.
+ #[serde(default)]
+ gate_stall_turns: u32,
+ /// Tool results a turn needs before it can be reviewed. Skips early chatty turns.
+ #[serde(default)]
+ gate_min_tool_results: u32,
+ /// Most output tokens one review may use.
+ #[serde(default = "default_advisor_max_tokens")]
+ advisor_max_tokens: u64,
+ /// Sampling temperature for reviews. Left off the request when unset.
+ #[serde(default)]
+ advisor_temperature: Option<f64>,
+ /// Size cap on the transcript sent to the advisor. Longer transcripts
+ /// are trimmed from the middle.
+ #[serde(default = "default_transcript_max_chars")]
+ transcript_max_chars: usize,
+ /// Lets the turn through when the advisor fails, instead of erroring.
+ #[serde(default = "default_fail_open")]
+ fail_open: bool,
+ },
+}
+
+/// What fires an advisor route's review.
+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum AdvisorTriggerConfig {
+ /// The executor's first turn without tool calls.
+ #[default]
+ NoToolCall,
+ /// The first turn whose text matches `gate_trigger_pattern`.
+ Pattern,
+}
+
+/// The judge a `stage_router` route falls through to, and how it routes.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct StageClassifierConfig {
+ /// Target the judge is called through. Not a routing destination.
+ pub target: String,
+ /// Lowest solve probability that still routes to the efficient tier, from 0 to 1.
+ pub base_threshold: f64,
+ /// How much to raise the threshold when the judge is uncertain. Added once
+ /// for an uncertain verdict and twice for unsupported.
+ #[serde(default)]
+ pub threshold_step: f64,
+ /// How often the judge runs. `new_session` has no effect here.
+ #[serde(default)]
+ pub classify_trigger: ClassifyTrigger,
+ /// Reuses the session's target by hashing the first user message when no
+ /// session ID is available.
+ #[serde(default)]
+ pub message_hash_fallback: bool,
+ /// How many trailing turns the judge sees. Unset shows it the opening task
+ /// and the latest user follow-up only.
+ #[serde(default)]
+ pub recent_turn_window: Option<usize>,
+ /// Replaces the packaged judge prompt.
+ #[serde(default)]
+ pub prompt: Option<String>,
+ /// How the judge is asked for structured output. Use `json_object` when the
+ /// provider cannot do JSON Schema.
+ #[serde(default)]
+ pub response_format_type: ClassifierResponseFormat,
+ /// Most completion tokens the judge verdict may use.
+ #[serde(default = "default_classifier_max_output_tokens")]
+ pub max_output_tokens: u64,
+}
+
+impl StageClassifierConfig {
+ fn task_classifier_config(&self) -> TaskClassifierConfig {
+ TaskClassifierConfig {
+ base_threshold: self.base_threshold,
+ threshold_step: self.threshold_step,
+ classify_trigger: self.classify_trigger,
+ message_hash_fallback: self.message_hash_fallback,
+ recent_turn_window: self.recent_turn_window,
+ contract: classifier_contract(self.prompt.as_deref())
+ .with_response_format_type(self.response_format_type),
+ max_output_tokens: self.max_output_tokens,
+ }
+ }
+}
+
+impl AlgorithmSpec {
+ /// Completion targets in algorithm order; judge-only targets are excluded.
+ pub fn routing_target_names(&self) -> Vec<&str> {
+ match self {
+ Self::Noop { .. } => Vec::new(),
+ Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(),
+ Self::Passthrough {
+ target, subagents, ..
+ } => {
+ let mut names = vec![target.as_str()];
+ if let Some(subagents) = subagents {
+ names.extend(subagents.routing_target_names());
+ }
+ names
+ }
+ Self::LlmClassifier { config, .. } => {
+ match config.mode.unwrap_or(if config.escalation.is_some() {
+ ClassifierMode::Escalation
+ } else {
+ ClassifierMode::Capability
+ }) {
+ ClassifierMode::Capability => config
+ .weak_target
+ .iter()
+ .chain(&config.strong_target)
+ .map(String::as_str)
+ .collect(),
+ ClassifierMode::Escalation => config
+ .strong_target
+ .iter()
+ .chain(&config.weak_target)
+ .map(String::as_str)
+ .collect(),
+ ClassifierMode::Custom => config
+ .targets
+ .iter()
+ .flatten()
+ .map(String::as_str)
+ .collect(),
+ }
+ }
+ Self::StageRouter {
+ capable_target,
+ efficient_target,
+ subagents,
+ ..
+ } => {
+ let mut names = vec![capable_target.as_str(), efficient_target.as_str()];
+ if let Some(subagents) = subagents {
+ names.extend(subagents.routing_target_names());
+ }
+ names
+ }
+ // The advisor is judge-only: reviews go through its own client,
+ // so it is not a completion (or count_tokens) destination.
+ Self::Advisor {
+ executor_target, ..
+ } => vec![executor_target],
+ }
+ }
+
+ /// Every target the algorithm may call, including judge-only targets.
+ ///
+ /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;
+ /// a classifier also calls its judge, and that call needs a client too.
+ pub fn callable_target_names(&self) -> Vec<&str> {
+ let mut names = self.routing_target_names();
+ match self {
+ Self::LlmClassifier { config, .. } => names.push(&config.classifier_target),
+ Self::Passthrough {
+ subagents: Some(subagents),
+ ..
+ } => names.extend(subagents.classifier_target_name()),
+ Self::StageRouter {
+ classifier,
+ subagents,
+ ..
+ } => {
+ if let Some(classifier) = classifier {
+ names.push(&classifier.target);
+ }
+ if let Some(subagents) = subagents {
+ names.extend(subagents.classifier_target_name());
+ }
+ }
+ Self::Advisor { advisor_target, .. } => names.push(advisor_target),
+ _ => {}
+ }
+ names
+ }
+ /// Builds this algorithm after resolving configured target names.
+ pub fn build(
+ &self,
+ context: &str,
+ targets: &BTreeMap<String, ModelId>,
+ ) -> AlgorithmResult<Arc<dyn Algorithm>> {
+ build_algorithm(context, self, targets)
+ }
+}
+impl LlmClassifierRouteConfig {
+ fn classifier_mode(&self, route_name: &str) -> AlgorithmResult<LlmClassifierModeConfig> {
+ let Self {
+ classifier_target,
+ mode,
+ strong_target,
+ weak_target,
+ base_threshold,
+ threshold_step,
+ classify_trigger,
+ message_hash_fallback,
+ recent_turn_window,
+ prompt,
+ response_format_type,
+ max_output_tokens,
+ escalation,
+ targets,
+ default_target,
+ response_schema,
+ policy,
+ } = self;
+
+ let selected_mode = match (mode, escalation.is_some()) {
+ (Some(mode), _) => *mode,
+ (None, true) => ClassifierMode::Escalation,
+ (None, false) => ClassifierMode::Capability,
+ };
+
+ match selected_mode {
+ ClassifierMode::Capability => {
+ if escalation.is_some() {
+ return Err(classifier_field_error(
+ route_name,
+ "escalation",
+ "capability",
+ ));
+ }
+ reject_custom_fields(
+ route_name,
+ "capability",
+ targets,
+ default_target,
+ response_schema,
+ policy,
+ )?;
+ Ok(LlmClassifierModeConfig::Capability(
+ CapabilityClassifierRouteConfig {
+ strong_target: required_classifier_field(
+ route_name,
+ "strong_target",
+ strong_target,
+ )?,
+ weak_target: required_classifier_field(
+ route_name,
+ "weak_target",
+ weak_target,
+ )?,
+ base_threshold: required_classifier_field(
+ route_name,
+ "base_threshold",
+ base_threshold,
+ )?,
+ threshold_step: threshold_step.unwrap_or_default(),
+ classify_trigger: *classify_trigger,
+ message_hash_fallback: *message_hash_fallback,
+ recent_turn_window: *recent_turn_window,
+ prompt: prompt.clone(),
+ response_format_type: *response_format_type,
+ max_output_tokens: *max_output_tokens,
+ },
+ ))
+ }
+ ClassifierMode::Escalation => {
+ reject_custom_fields(
+ route_name,
+ "escalation",
+ targets,
+ default_target,
+ response_schema,
+ policy,
+ )?;
+ if *classify_trigger != ClassifyTrigger::EveryRequest {
+ return Err(AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} mode escalation cannot use classify_trigger"
+ )));
+ }
+ if mode.is_some()
+ && (base_threshold.is_some()
+ || threshold_step.is_some()
+ || *message_hash_fallback
+ || recent_turn_window.is_some())
+ {
+ return Err(AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} mode escalation cannot use capability routing settings"
+ )));
+ }
+ Ok(LlmClassifierModeConfig::Escalation(
+ EscalationClassifierRouteConfig {
+ strong_target: required_classifier_field(
+ route_name,
+ "strong_target",
+ strong_target,
+ )?,
+ weak_target: required_classifier_field(
+ route_name,
+ "weak_target",
+ weak_target,
+ )?,
+ prompt: prompt.clone(),
+ response_format_type: *response_format_type,
+ max_output_tokens: *max_output_tokens,
+ judge: required_classifier_field(route_name, "escalation", escalation)?,
+ },
+ ))
+ }
+ ClassifierMode::Custom => {
+ if strong_target.is_some()
+ || weak_target.is_some()
+ || base_threshold.is_some()
+ || threshold_step.is_some()
+ || escalation.is_some()
+ || *response_format_type != ClassifierResponseFormat::JsonSchema
+ {
+ return Err(AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} mode custom cannot use capability or escalation fields and response_format_type must be 'json_schema'"
+ )));
+ }
+ Ok(LlmClassifierModeConfig::Custom(
+ CustomClassifierRouteConfig {
+ classifier_target: classifier_target.clone(),
+ targets: required_classifier_field(route_name, "targets", targets)?,
+ default_target: required_classifier_field(
+ route_name,
+ "default_target",
+ default_target,
+ )?,
+ prompt: required_classifier_field(route_name, "prompt", prompt)?,
+ response_schema: required_classifier_field(
+ route_name,
+ "response_schema",
+ response_schema,
+ )?,
+ policy: required_classifier_field(route_name, "policy", policy)?,
+ classify_trigger: *classify_trigger,
+ message_hash_fallback: *message_hash_fallback,
+ recent_turn_window: *recent_turn_window,
+ max_output_tokens: *max_output_tokens,
+ },
+ ))
+ }
+ }
+ }
+}
+
+fn reject_custom_fields(
+ route_name: &str,
+ mode: &str,
+ targets: &Option<Vec<String>>,
+ default_target: &Option<String>,
+ response_schema: &Option<String>,
+ policy: &Option<ClassifierPolicyConfig>,
+) -> AlgorithmResult<()> {
+ if targets.is_some()
+ || default_target.is_some()
+ || response_schema.is_some()
+ || policy.is_some()
+ {
+ return Err(AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields"
+ )));
+ }
+ Ok(())
+}
+
+fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> AlgorithmConfigError {
+ AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} mode {mode} cannot use {field}"
+ ))
+}
+
+fn required_classifier_field<T: Clone>(
+ route_name: &str,
+ field: &str,
+ value: &Option<T>,
+) -> AlgorithmResult<T> {
+ value.clone().ok_or_else(|| {
+ AlgorithmConfigError::new(format!(
+ "llm_classifier route {route_name} requires {field}"
+ ))
+ })
+}
+
+fn build_subagent_router_config(
+ route_name: &str,
+ config: &SubagentRouteConfig,
+ targets: &BTreeMap<String, ModelId>,
+) -> AlgorithmResult<SubagentRouterConfig> {
+ match config {
+ SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target(
+ resolve_target_model_id(route_name, target, targets)?,
+ )),
+ SubagentRouteConfig::LlmClassifier(config) => {
+ let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)?
+ else {
+ return Err(AlgorithmConfigError::new(format!(
+ "route {route_name}: subagents llm_classifier only supports mode custom"
+ )));
+ };
+ let judge_target =
+ resolve_target_model_id(route_name, &config.classifier_target, targets)?;
+ let resolved_targets = config
+ .targets
+ .iter()
+ .map(|name| {
+ resolve_target_model_id(route_name, name, targets)
+ .map(|target| (name.clone(), target))
+ })
+ .collect::<AlgorithmResult<Vec<_>>>()?;
+ let default_target = resolved_targets
+ .iter()
+ .find(|(name, _)| *name == config.default_target)
+ .map(|(_, target)| target.clone())
+ .ok_or_else(|| {
+ AlgorithmConfigError::new(format!(
+ "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets",
+ config.default_target
+ ))
+ })?;
+ let response_schema =
+ serde_json::from_str(&config.response_schema).map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!(
+ "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}"
+ ),
+ error,
+ )
+ })?;
+ let mut classifier_config = CustomClassifierConfig::new(
+ config.prompt,
+ response_schema,
+ config.policy.into_libsy(),
+ );
+ classifier_config.recent_turn_window = config.recent_turn_window;
+ classifier_config.max_output_tokens = config.max_output_tokens;
+ let subagent_targets = resolved_targets
+ .iter()
+ .map(|(_, target)| target.clone())
+ .collect();
+ let classifier = Arc::new(
+ LlmTaskClassifier::new(LlmClassifierConfig::Custom {
+ judge_target,
+ targets: resolved_targets,
+ default_target: config.default_target,
+ config: classifier_config,
+ })
+ .map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("route {route_name}: subagents llm_classifier: {error}"),
+ error,
+ )
+ })?,
+ );
+ Ok(SubagentRouterConfig {
+ targets: subagent_targets,
+ classifier,
+ default_target,
+ classify_trigger: config.classify_trigger,
+ message_hash_fallback: config.message_hash_fallback,
+ })
+ }
+ }
+}
+
+fn attach_subagent_router(
+ route_name: &str,
+ parent: Arc<dyn Algorithm>,
+ config: Option<&SubagentRouteConfig>,
+ targets: &BTreeMap<String, ModelId>,
+) -> AlgorithmResult<Arc<dyn Algorithm>> {
+ let Some(config) = config else {
+ return Ok(parent);
+ };
+ let config = build_subagent_router_config(route_name, config, targets)?;
+ let algorithm = SubagentRouter::new(parent, config).map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("route {route_name}: subagent routing: {error}"),
+ error,
+ )
+ })?;
+ Ok(Arc::new(algorithm))
+}
+
+fn build_algorithm(
+ route_name: &str,
+ config: &AlgorithmSpec,
+ targets: &BTreeMap<String, ModelId>,
+) -> AlgorithmResult<Arc<dyn Algorithm>> {
+ match config {
+ AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})),
+ AlgorithmSpec::Random {
+ targets: names,
+ weights,
+ seed,
+ ..
+ } => {
+ let target_set =
+ resolve_targets(route_name, names.iter().map(String::as_str), targets)?;
+ let algorithm = Random::new(target_set, weights.clone(), *seed).map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("random route {route_name}: {error}"),
+ error,
+ )
+ })?;
+ Ok(Arc::new(algorithm))
+ }
+ AlgorithmSpec::Passthrough {
+ target, subagents, ..
+ } => {
+ let parent_target = resolve_target_model_id(route_name, target, targets)?;
+ let algorithm = Passthrough::new(parent_target);
+ let parent: Arc<dyn Algorithm> = Arc::new(algorithm);
+ attach_subagent_router(route_name, parent, subagents.as_ref(), targets)
+ }
+ AlgorithmSpec::LlmClassifier {
+ config: classifier_config,
+ ..
+ } => {
+ let classifier =
+ resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?;
+ let mode = classifier_config.classifier_mode(route_name)?;
+ let algorithm = match mode {
+ LlmClassifierModeConfig::Capability(config) => {
+ let strong =
+ resolve_target_model_id(route_name, &config.strong_target, targets)?;
+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
+ let classifier_config = TaskClassifierConfig {
+ base_threshold: config.base_threshold,
+ threshold_step: config.threshold_step,
+ classify_trigger: config.classify_trigger,
+ message_hash_fallback: config.message_hash_fallback,
+ recent_turn_window: config.recent_turn_window,
+ contract: classifier_contract(config.prompt.as_deref())
+ .with_response_format_type(config.response_format_type),
+ max_output_tokens: config.max_output_tokens,
+ };
+ LlmTaskClassifier::new(LlmClassifierConfig::Capability {
+ judge_target: classifier,
+ efficient_target: weak,
+ capable_target: strong,
+ config: classifier_config,
+ })
+ }
+ LlmClassifierModeConfig::Escalation(config) => {
+ let strong =
+ resolve_target_model_id(route_name, &config.strong_target, targets)?;
+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
+ LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
+ judge_target: classifier,
+ efficient_target: weak,
+ capable_target: strong,
+ contract: classifier_contract(config.prompt.as_deref())
+ .with_response_format_type(config.response_format_type),
+ config: config.judge,
+ max_output_tokens: config.max_output_tokens,
+ })
+ }
+ LlmClassifierModeConfig::Custom(config) => {
+ let resolved_targets = config
+ .targets
+ .iter()
+ .map(|name| {
+ resolve_target_model_id(route_name, name, targets)
+ .map(|target| (name.clone(), target))
+ })
+ .collect::<AlgorithmResult<Vec<_>>>()?;
+ let response_schema = serde_json::from_str(&config.response_schema).map_err(
+ |error| {
+ AlgorithmConfigError::with_source(
+ format!(
+ "llm_classifier route {route_name}: response_schema is invalid JSON: {error}"
+ ),
+ error,
+ )
+ },
+ )?;
+ let mut classifier_config = CustomClassifierConfig::new(
+ config.prompt,
+ response_schema,
+ config.policy.into_libsy(),
+ );
+ classifier_config.classify_trigger = config.classify_trigger;
+ classifier_config.message_hash_fallback = config.message_hash_fallback;
+ classifier_config.recent_turn_window = config.recent_turn_window;
+ classifier_config.max_output_tokens = config.max_output_tokens;
+ LlmTaskClassifier::new(LlmClassifierConfig::Custom {
+ judge_target: classifier,
+ targets: resolved_targets,
+ default_target: config.default_target,
+ config: classifier_config,
+ })
+ }
+ }
+ .map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("llm_classifier route {route_name}: {error}"),
+ error,
+ )
+ })?;
+ Ok(Arc::new(algorithm))
+ }
+ AlgorithmSpec::StageRouter {
+ capable_target,
+ efficient_target,
+ picker,
+ confidence_threshold,
+ recent_turn_window,
+ handoff_notes,
+ capable_system_prompt,
+ efficient_system_prompt,
+ classifier,
+ subagents,
+ ..
+ } => {
+ if matches!(picker, PickerMode::CapableFirst) {
+ tracing::warn!(
+ "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."
+ );
+ }
+ let capable = resolve_target_model_id(route_name, capable_target, targets)?;
+ let efficient = resolve_target_model_id(route_name, efficient_target, targets)?;
+ let mut config = StageRouterConfig::new(*picker, *confidence_threshold);
+ config.recent_window = *recent_turn_window;
+ config.handoff_notes = handoff_notes.clone();
+ config.tier_prompts = tier_prompts(
+ &capable,
+ capable_system_prompt.as_deref(),
+ &efficient,
+ efficient_system_prompt.as_deref(),
+ );
+ // The judge is called through its own target, so it is not a routing
+ // destination and stays out of the tier pair.
+ config.llm_fallback = classifier
+ .as_ref()
+ .map(|classifier| {
+ resolve_target_model_id(route_name, &classifier.target, targets).map(
+ |judge_target| LlmFallback {
+ judge_target,
+ config: classifier.task_classifier_config(),
+ },
+ )
+ })
+ .transpose()?;
+ let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("stage_router route {route_name}: {error}"),
+ error,
+ )
+ })?;
+ let parent: Arc<dyn Algorithm> = Arc::new(algorithm);
+ attach_subagent_router(route_name, parent, subagents.as_ref(), targets)
+ }
+ AlgorithmSpec::Advisor {
+ executor_target,
+ advisor_target,
+ reviewer_system_prompt,
+ redo_feedback_prefix,
+ gate_trigger,
+ gate_trigger_pattern,
+ max_reviews,
+ gate_stall_turns,
+ gate_min_tool_results,
+ advisor_max_tokens,
+ advisor_temperature,
+ transcript_max_chars,
+ fail_open,
+ ..
+ } => {
+ let executor = resolve_target_model_id(route_name, executor_target, targets)?;
+ let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;
+ // A pattern set under the default trigger would be silently
+ // ignored; reject the misconfiguration instead.
+ if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {
+ return Err(AlgorithmConfigError::new(format!(
+ "advisor route {route_name}: gate_trigger_pattern requires \
+ gate_trigger = \"pattern\""
+ )));
+ }
+ let mut config = AdvisorGateConfig::default();
+ if let Some(prompt) = reviewer_system_prompt {
+ config.reviewer_system_prompt = prompt.clone();
+ }
+ if let Some(prefix) = redo_feedback_prefix {
+ config.redo_feedback_prefix = prefix.clone();
+ }
+ config.gate_trigger = match gate_trigger {
+ AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,
+ AdvisorTriggerConfig::Pattern => {
+ GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())
+ }
+ };
+ config.max_reviews = *max_reviews;
+ config.gate_stall_turns = *gate_stall_turns;
+ config.gate_min_tool_results = *gate_min_tool_results;
+ config.advisor_max_tokens = *advisor_max_tokens;
+ config.advisor_temperature = *advisor_temperature;
+ config.transcript_max_chars = *transcript_max_chars;
+ config.fail_open = *fail_open;
+ let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {
+ AlgorithmConfigError::with_source(
+ format!("advisor route {route_name}: {error}"),
+ error,
+ )
+ })?;
+ Ok(Arc::new(algorithm))
+ }
+ }
+}
+
+const fn default_max_reviews() -> u32 {
+ 1
+}
+
+const fn default_advisor_max_tokens() -> u64 {
+ 2048
+}
+
+const fn default_transcript_max_chars() -> usize {
+ 200_000
+}
+
+const fn default_fail_open() -> bool {
+ true
+}
+
+fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {
+ prompt.map_or_else(ClassifierContractConfig::default, |prompt| {
+ ClassifierContractConfig::default().with_prompt(prompt)
+ })
+}
+
+fn default_classifier_max_output_tokens() -> u64 {
+ TaskClassifierConfig::default().max_output_tokens
+}
+
+/// Keys each configured system prompt by the target it belongs to.
+fn tier_prompts(
+ capable: &str,
+ capable_prompt: Option<&str>,
+ efficient: &str,
+ efficient_prompt: Option<&str>,
+) -> TargetPrompts {
+ let mut prompts = TargetPrompts::default();
+ if let Some(prompt) = capable_prompt {
+ prompts = prompts.with(capable, prompt);
+ }
+ if let Some(prompt) = efficient_prompt {
+ prompts = prompts.with(efficient, prompt);
+ }
+ prompts
+}
+
+fn resolve_targets<'a>(
+ route_name: &str,
+ names: impl IntoIterator<Item = &'a str>,
+ targets: &BTreeMap<String, ModelId>,
+) -> AlgorithmResult<Vec<ModelId>> {
+ names
+ .into_iter()
+ .map(|name| resolve_target_model_id(route_name, name, targets))
+ .collect()
+}
+
+fn resolve_target_model_id(
+ route_name: &str,
+ name: &str,
+ targets: &BTreeMap<String, ModelId>,
+) -> AlgorithmResult<ModelId> {
+ targets.get(name).cloned().ok_or_else(|| {
+ AlgorithmConfigError::new(format!(
+ "route {route_name} references unknown target {name}"
+ ))
+ })
+}
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
new file mode 100644
index 000000000..7a90c7bba
--- /dev/null
+++ b/crates/switchyard-runner/src/config.rs
@@ -0,0 +1,1401 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Version-1 TOML deployment loading for the shared runner.
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+
+use serde::de::DeserializeOwned;
+use serde::{Deserialize, Deserializer};
+use serde_json::Value;
+use switchyard_llm_client::{
+ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
+ TranslatingLlmClient,
+};
+use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};
+
+use crate::{
+ AlgorithmSpec, CallerAuthKind, CountTokensTarget, DecisionTarget, ModelCapabilities, Route,
+ Runner, RunnerError,
+};
+
+const SUPPORTED_SCHEMA_VERSION: u32 = 1;
+const MAX_CONFIGURED_RETRIES: u32 = 10;
+
+type RunnerResult<T> = Result<T, RunnerError>;
+
+pub(crate) fn load_runner(path: impl AsRef<Path>) -> RunnerResult<Runner> {
+ let path = path.as_ref();
+ let source = fs::read_to_string(path).map_err(|error| {
+ RunnerError::configuration_source(
+ format!("failed to read server config {}: {error}", path.display()),
+ error,
+ )
+ })?;
+ runner_from_toml(&source).map_err(|error| {
+ RunnerError::configuration_source(
+ format!("invalid server config {}: {error}", path.display()),
+ error,
+ )
+ })
+}
+
+fn runner_from_toml(source: &str) -> RunnerResult<Runner> {
+ let config: DeploymentConfig = toml::from_str(source).map_err(|error| {
+ RunnerError::configuration_source(format!("failed to parse TOML: {error}"), error)
+ })?;
+ config.build()
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub(crate) struct DeploymentConfig {
+ schema_version: u32,
+ #[serde(default)]
+ llm_clients: BTreeMap<String, LlmClientConfig>,
+ targets: BTreeMap<String, TargetConfig>,
+ routes: BTreeMap<String, RouteConfig>,
+}
+
+#[derive(Debug)]
+struct RouteConfig {
+ id: ModelId,
+ context_window: Option<u32>,
+ tool_calling: Option<bool>,
+ reasoning: Option<bool>,
+ algorithm: AlgorithmSpec,
+}
+
+impl<'de> Deserialize<'de> for RouteConfig {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ let mut table = toml::Table::deserialize(deserializer)?;
+ let id = take_required(&mut table, "id")?;
+ let context_window = take_optional(&mut table, "context_window")?;
+ let tool_calling = take_optional(&mut table, "tool_calling")?;
+ let reasoning = take_optional(&mut table, "reasoning")?;
+ let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))
+ .map_err(serde::de::Error::custom)?;
+ Ok(Self {
+ id,
+ context_window,
+ tool_calling,
+ reasoning,
+ algorithm,
+ })
+ }
+}
+
+fn take_required<T, E>(table: &mut toml::Table, name: &'static str) -> Result<T, E>
+where
+ T: DeserializeOwned,
+ E: serde::de::Error,
+{
+ let value = table.remove(name).ok_or_else(|| E::missing_field(name))?;
+ T::deserialize(value).map_err(E::custom)
+}
+
+fn take_optional<T, E>(table: &mut toml::Table, name: &'static str) -> Result<Option<T>, E>
+where
+ T: DeserializeOwned,
+ E: serde::de::Error,
+{
+ table
+ .remove(name)
+ .map(|value| T::deserialize(value).map_err(E::custom))
+ .transpose()
+}
+
+impl RouteConfig {
+ fn capabilities(&self) -> ModelCapabilities {
+ ModelCapabilities {
+ context_window: self.context_window,
+ tool_calling: self.tool_calling,
+ reasoning: self.reasoning,
+ }
+ }
+
+ fn routing_target_names(&self) -> Vec<&str> {
+ self.algorithm.routing_target_names()
+ }
+
+ fn callable_target_names(&self) -> Vec<&str> {
+ self.algorithm.callable_target_names()
+ }
+}
+
+impl DeploymentConfig {
+ fn decision_target(&self, name: &str) -> Option<DecisionTarget> {
+ let target = self.targets.get(name)?;
+ let client = self.llm_clients.get(&target.llm_client)?;
+ Some(DecisionTarget {
+ target: name.to_string(),
+ model: target.id.clone(),
+ format: client.format.wire_format(),
+ base_url: client.base_url.as_str().to_string(),
+ extra_body: target.extra_body.clone(),
+ })
+ }
+
+ fn build(self) -> RunnerResult<Runner> {
+ if self.schema_version != SUPPORTED_SCHEMA_VERSION {
+ return Err(RunnerError::configuration(format!(
+ "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",
+ self.schema_version
+ )));
+ }
+
+ let mut seen_client_model_ids = HashSet::new();
+ for (target_name, target) in &self.targets {
+ validate_value("target name", target_name)?;
+ validate_value(&format!("target {target_name} id"), &target.id)?;
+ if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) {
+ tracing::warn!(
+ "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",
+ target.id,
+ target.llm_client
+ );
+ }
+ }
+
+ let clients = self.build_clients()?;
+ let targets = self.build_targets();
+ let mut routes = Vec::with_capacity(self.routes.len());
+ for (route_name, config) in &self.routes {
+ validate_value("route name", route_name)?;
+ validate_value(&format!("route {route_name} id"), &config.id)?;
+ for target_name in config.callable_target_names() {
+ self.targets.get(target_name).ok_or_else(|| {
+ RunnerError::configuration(format!(
+ "route references unknown target {target_name}"
+ ))
+ })?;
+ }
+ let capabilities = config.capabilities();
+ if capabilities.context_window == Some(0) {
+ return Err(RunnerError::configuration(format!(
+ "route {route_name} context_window must be greater than zero"
+ )));
+ }
+ let algorithm = config
+ .algorithm
+ .build(route_name, &targets)
+ .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;
+ let (route_clients, caller_auth) =
+ self.build_route_clients(route_name, config, &clients)?;
+ let count_tokens_target = self.build_count_tokens_target(config, &clients);
+ let decision_targets = config
+ .routing_target_names()
+ .into_iter()
+ .filter_map(|name| self.decision_target(name))
+ .collect();
+ let route = Route::new(
+ algorithm,
+ route_clients,
+ caller_auth,
+ capabilities,
+ count_tokens_target,
+ decision_targets,
+ );
+ routes.push((config.id.clone(), route));
+ }
+ Ok(Runner::new(routes))
+ }
+
+ fn build_clients(&self) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {
+ let mut models_by_client = self
+ .llm_clients
+ .keys()
+ .map(|name| (name.clone(), Vec::new()))
+ .collect::<BTreeMap<String, Vec<ModelConfig>>>();
+
+ for (name, client_config) in &self.llm_clients {
+ validate_value("llm client name", name)?;
+ build_backend(name, client_config, &BTreeMap::new())?;
+ }
+ for (target_name, target) in &self.targets {
+ let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
+ RunnerError::configuration(format!(
+ "target {target_name} references unknown llm client {}",
+ target.llm_client
+ ))
+ })?;
+ let model_configs = models_by_client
+ .get_mut(&target.llm_client)
+ .ok_or_else(|| {
+ RunnerError::configuration("validated llm client was not initialized")
+ })?;
+ model_configs.push(ModelConfig::new(
+ target.id.clone(),
+ build_backend(&target.llm_client, client_config, &target.extra_body)?,
+ None,
+ ));
+ }
+
+ let mut clients = BTreeMap::new();
+ for (name, model_configs) in models_by_client {
+ let client = Arc::new(
+ TranslatingLlmClient::new(&model_configs)
+ .map_err(|error| RunnerError::configuration(error.to_string()))?,
+ );
+ clients.insert(name, client);
+ }
+ Ok(clients)
+ }
+
+ fn build_targets(&self) -> BTreeMap<String, ModelId> {
+ self.targets
+ .iter()
+ .map(|(name, config)| (name.clone(), config.id.clone()))
+ .collect()
+ }
+
+ fn build_route_clients(
+ &self,
+ route_name: &str,
+ route: &RouteConfig,
+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
+ ) -> RunnerResult<(ClientRouter, Option<CallerAuthKind>)> {
+ let mut by_model = HashMap::new();
+ let mut caller_auth = None;
+ for name in route.callable_target_names() {
+ let target = self.targets.get(name).ok_or_else(|| {
+ RunnerError::configuration(format!("route references unknown target {name}"))
+ })?;
+ let client = clients.get(&target.llm_client).ok_or_else(|| {
+ RunnerError::configuration(format!("target {name} has no constructed llm client"))
+ })?;
+ let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
+ RunnerError::configuration(format!(
+ "target {name} references unknown llm client {}",
+ target.llm_client
+ ))
+ })?;
+ if client_config.forward_auth {
+ let target_auth = client_config.format.caller_auth_kind();
+ if caller_auth.is_some_and(|kind| kind != target_auth) {
+ return Err(RunnerError::configuration(format!(
+ "route {route_name} cannot forward both Anthropic and OpenAI caller credentials"
+ )));
+ }
+ caller_auth = Some(target_auth);
+ }
+ let client: Arc<dyn RoutedLlmClient> = client.clone();
+ by_model.insert(target.id.clone(), client);
+ }
+ Ok((ClientRouter::new(by_model), caller_auth))
+ }
+
+ fn build_count_tokens_target(
+ &self,
+ route: &RouteConfig,
+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
+ ) -> Option<CountTokensTarget> {
+ route
+ .routing_target_names()
+ .into_iter()
+ .enumerate()
+ .filter_map(|(index, name)| {
+ let target = self.targets.get(name)?;
+ let client = clients.get(&target.llm_client)?;
+ client.supports_count_tokens(&target.id).then_some((
+ count_tokens_priority(name, &target.id),
+ index,
+ target,
+ client,
+ ))
+ })
+ .min_by_key(|(priority, index, _, _)| (*priority, *index))
+ .map(|(_, _, target, client)| CountTokensTarget {
+ model: target.id.clone(),
+ client: client.clone(),
+ })
+ }
+}
+
+fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
+ let target_name = target_name.to_ascii_lowercase();
+ let model_id = model_id.to_ascii_lowercase();
+ ["opus", "sonnet", "haiku"]
+ .iter()
+ .position(|hint| target_name.contains(hint) || model_id.contains(hint))
+ .unwrap_or(3)
+}
+
+/// A client endpoint, parsed when the config loads rather than checked afterwards.
+///
+/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no
+/// later stage has to re-check it or can forget to.
+#[derive(Clone, Debug)]
+struct HttpBaseUrl(reqwest::Url);
+
+impl HttpBaseUrl {
+ fn as_str(&self) -> &str {
+ self.0.as_str()
+ }
+}
+
+impl<'de> Deserialize<'de> for HttpBaseUrl {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ let raw = String::deserialize(deserializer)?;
+ let url = reqwest::Url::parse(raw.trim()).map_err(|error| {
+ serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}"))
+ })?;
+ if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
+ return Err(serde::de::Error::custom(
+ "base_url must be an absolute HTTP(S) URL",
+ ));
+ }
+ Ok(Self(url))
+ }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct LlmClientConfig {
+ format: ClientFormat,
+ base_url: HttpBaseUrl,
+ api_key_env: Option<String>,
+ #[serde(default)]
+ forward_auth: bool,
+ #[serde(default)]
+ extra_headers: BTreeMap<String, String>,
+ #[serde(default = "default_max_retries")]
+ max_retries: u32,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct TargetConfig {
+ id: ModelId,
+ llm_client: String,
+ #[serde(default)]
+ extra_body: BTreeMap<String, Value>,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize)]
+enum ClientFormat {
+ #[serde(rename = "openai_chat")]
+ OpenAiChat,
+ #[serde(rename = "openai_responses")]
+ OpenAiResponses,
+ #[serde(rename = "anthropic_messages")]
+ AnthropicMessages,
+}
+
+impl ClientFormat {
+ const fn wire_format(self) -> WireFormat {
+ match self {
+ Self::OpenAiChat => WireFormat::OpenAiChat,
+ Self::OpenAiResponses => WireFormat::OpenAiResponses,
+ Self::AnthropicMessages => WireFormat::AnthropicMessages,
+ }
+ }
+
+ const fn caller_auth_kind(self) -> CallerAuthKind {
+ match self {
+ Self::AnthropicMessages => CallerAuthKind::Anthropic,
+ Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi,
+ }
+ }
+}
+fn build_backend(
+ client_name: &str,
+ config: &LlmClientConfig,
+ extra_body: &BTreeMap<String, Value>,
+) -> RunnerResult<Backend> {
+ if config.max_retries > MAX_CONFIGURED_RETRIES {
+ return Err(RunnerError::configuration(format!(
+ "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"
+ )));
+ }
+ if config.forward_auth && config.api_key_env.is_some() {
+ return Err(RunnerError::configuration(format!(
+ "llm client {client_name} cannot set both forward_auth and api_key_env"
+ )));
+ }
+ let api_key = config
+ .api_key_env
+ .as_deref()
+ .map(|variable| {
+ if variable.trim().is_empty() {
+ return Err(RunnerError::configuration(format!(
+ "llm client {client_name} api_key_env must not be empty"
+ )));
+ }
+ let api_key = std::env::var(variable).map_err(|error| {
+ RunnerError::configuration(format!(
+ "llm client {client_name} could not read api_key_env {variable}: {error}"
+ ))
+ })?;
+ if api_key.trim().is_empty() {
+ return Err(RunnerError::configuration(format!(
+ "llm client {client_name} api_key_env {variable} is empty"
+ )));
+ }
+ Ok(api_key)
+ })
+ .transpose()?;
+ let http = HttpBackendConfig {
+ base_url: config.base_url.as_str().to_string(),
+ api_key,
+ forward_auth: config.forward_auth,
+ extra_headers: config.extra_headers.clone(),
+ extra_body: extra_body.clone(),
+ max_retries: config.max_retries,
+ };
+ let backend = match config.format {
+ ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
+ ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),
+ ClientFormat::AnthropicMessages => Backend::Anthropic(http),
+ };
+ Ok(backend)
+}
+
+// A function so that serde default can use it.
+const fn default_max_retries() -> u32 {
+ DEFAULT_MAX_RETRIES
+}
+
+fn validate_value(label: &str, value: &str) -> RunnerResult<()> {
+ if value.trim().is_empty() || value.trim() != value {
+ return Err(RunnerError::configuration(format!(
+ "{label} must be non-empty and have no surrounding whitespace"
+ )));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn route_presentation_fields_are_split_from_the_algorithm() {
+ let route: RouteConfig = toml::from_str(
+ r#"
+type = "random"
+id = "switchyard/random"
+context_window = 128000
+tool_calling = true
+reasoning = false
+targets = ["fast", "strong"]
+weights = [1.0, 2.0]
+seed = 7
+"#,
+ )
+ .expect("route should deserialize");
+
+ assert_eq!(route.id, "switchyard/random");
+ assert_eq!(route.context_window, Some(128_000));
+ assert_eq!(route.tool_calling, Some(true));
+ assert_eq!(route.reasoning, Some(false));
+ assert_eq!(route.algorithm.routing_target_names(), ["fast", "strong"]);
+ }
+
+ #[test]
+ fn route_algorithm_unknown_fields_are_rejected() {
+ let error = toml::from_str::<RouteConfig>(
+ r#"
+type = "passthrough"
+id = "switchyard/fast"
+target = "fast"
+bogus = true
+"#,
+ )
+ .expect_err("unknown route keys must be rejected");
+
+ assert!(error.to_string().contains("bogus"), "{error}");
+ }
+
+ #[test]
+ fn deployment_errors_keep_one_path_context() {
+ let path = Path::new("/definitely/missing/switchyard-routes.toml");
+ let error = match load_runner(path) {
+ Ok(_) => panic!("missing config should fail"),
+ Err(error) => error,
+ };
+ let message = error.to_string();
+
+ assert_eq!(message.matches(&path.display().to_string()).count(), 1);
+ assert!(message.starts_with("failed to read server config"));
+ }
+}
+#[cfg(test)]
+mod deployment_tests {
+ use super::*;
+ use serde_json::json;
+
+ const VALID_CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[llm_clients.responses]
+format = "openai_responses"
+base_url = "https://example.test/v1"
+
+[llm_clients.anthropic]
+format = "anthropic_messages"
+base_url = "https://example.test"
+
+[targets.classifier]
+id = "classifier/model"
+llm_client = "primary"
+
+[targets.strong]
+id = "strong/model"
+llm_client = "responses"
+
+[targets.weak]
+id = "weak/model"
+llm_client = "anthropic"
+
+[routes.noop]
+id = "switchyard/noop"
+type = "noop"
+
+[routes.random]
+id = "switchyard/random"
+type = "random"
+targets = ["strong", "weak"]
+
+[routes.classifier]
+id = "switchyard/classifier"
+type = "llm_classifier"
+classifier_target = "classifier"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 0.5
+
+[routes.passthrough]
+id = "switchyard/passthrough"
+type = "passthrough"
+target = "weak"
+"#;
+
+ fn error_message(toml: &str) -> String {
+ match runner_from_toml(toml) {
+ Ok(_) => "configuration unexpectedly succeeded".to_string(),
+ Err(error) => error.to_string(),
+ }
+ }
+
+ fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {
+ let mut configured = config.to_string();
+ configured.push_str(&format!("\n[routes.{route}.subagents]\n"));
+ configured.push_str(
+ r#"type = "llm_classifier"
+mode = "custom"
+classifier_target = "classifier"
+targets = ["strong", "weak"]
+default_target = "weak"
+prompt = "Select a target for this delegated task."
+response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}'
+policy = { type = "target_selector", selector = "/target" }
+classify_trigger = "new_session""#,
+ );
+ configured.push_str(extra);
+ configured
+ }
+
+ fn with_subagent_passthrough(config: &str, route: &str) -> String {
+ format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")
+ }
+
+ fn stage_config() -> String {
+ format!(
+ r#"{VALID_CONFIG}
+[targets.stage_judge]
+id = "stage-judge/model"
+llm_client = "primary"
+
+[routes.stage]
+id = "switchyard/stage"
+type = "stage_router"
+capable_target = "strong"
+efficient_target = "weak"
+picker = "efficient_first"
+confidence_threshold = 1.0
+
+[routes.stage.classifier]
+target = "stage_judge"
+base_threshold = 0.5
+"#
+ )
+ }
+
+ #[test]
+ fn builds_all_supported_algorithm_types() -> RunnerResult<()> {
+ let state = runner_from_toml(VALID_CONFIG)?;
+ // The model id array is sorted alphabetically
+ assert_eq!(
+ state
+ .models()
+ .map(|model| model.id.as_str())
+ .collect::<Vec<_>>(),
+ [
+ "switchyard/classifier",
+ "switchyard/noop",
+ "switchyard/passthrough",
+ "switchyard/random",
+ ]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> {
+ let stage = stage_config();
+ let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", "");
+ let parsed: DeploymentConfig = toml::from_str(&stage_with_classifier).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse stage config: {error}"))
+ })?;
+ let Some(stage_route) = parsed.routes.get("stage") else {
+ return Err(RunnerError::configuration("stage route is missing"));
+ };
+ let callable_targets = stage_route.callable_target_names();
+ for expected in ["strong", "weak", "stage_judge", "classifier"] {
+ assert!(callable_targets.contains(&expected));
+ }
+
+ for configured in [
+ with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""),
+ with_subagent_passthrough(VALID_CONFIG, "passthrough"),
+ stage_with_classifier,
+ with_subagent_passthrough(&stage, "stage"),
+ ] {
+ runner_from_toml(&configured)?;
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_invalid_unreferenced_llm_client() {
+ let invalid = format!(
+ "{VALID_CONFIG}\n\
+ [llm_clients.unused]\n\
+ format = \"openai_chat\"\n\
+ base_url = \"not a url\"\n"
+ );
+ let message = error_message(&invalid);
+ assert!(
+ message.contains("base_url must be an absolute HTTP(S) URL"),
+ "unexpected error: {message}"
+ );
+ }
+
+ #[test]
+ fn an_escalation_table_switches_the_classifier_route_to_escalation() -> RunnerResult<()> {
+ // Present: the classifier target judges the weak tier's reply each turn instead of
+ // picking a tier ahead of it. The route builds either way, so the assertion is that
+ // the knob parses and its settings reach the algorithm's validation.
+ let escalating = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { confirmations = 2 }",
+ );
+ runner_from_toml(&escalating)?;
+
+ // A setting that would starve the judge is rejected here rather than on the first
+ // request, the same as any other unusable route configuration.
+ let starved = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { confirmations = 0 }",
+ );
+ assert!(error_message(&starved).contains("confirmations must be at least 1"));
+ Ok(())
+ }
+
+ #[test]
+ fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> {
+ let capability = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 512",
+ );
+ runner_from_toml(&capability)?;
+
+ let escalation = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",
+ );
+ runner_from_toml(&escalation)?;
+ Ok(())
+ }
+
+ #[test]
+ fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> {
+ let capability = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nprompt = \"custom capability rubric\"",
+ );
+ runner_from_toml(&capability)?;
+
+ let escalation = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }",
+ );
+ runner_from_toml(&escalation)?;
+
+ let empty = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nprompt = \" \"",
+ );
+ assert!(error_message(&empty).contains("classifier prompt must not be empty"));
+
+ let schema_placeholder = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",
+ );
+ assert!(
+ error_message(&schema_placeholder)
+ .contains("Switchyard supplies the schema automatically")
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn mode_custom_rejects_capability_fields() {
+ let mixed = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "mode = \"custom\"\nbase_threshold = 0.5",
+ );
+
+ assert!(
+ error_message(&mixed)
+ .contains("mode custom cannot use capability or escalation fields")
+ );
+ }
+
+ #[test]
+ fn rejects_unknown_fields_and_algorithm_types() {
+ let unknown_field =
+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");
+ assert!(error_message(&unknown_field).contains("unknown field"));
+
+ let nested_completion_cap = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",
+ );
+ assert!(error_message(&nested_completion_cap).contains("unknown field"));
+
+ let unknown_classifier_field = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nclassifier_magic = true",
+ );
+ assert!(error_message(&unknown_classifier_field).contains("unknown field"));
+
+ let target_capability = VALID_CONFIG.replace(
+ "llm_client = \"responses\"",
+ "llm_client = \"responses\"\ncontext_window = 1000000",
+ );
+ assert!(error_message(&target_capability).contains("unknown field `context_window`"));
+
+ let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
+ assert!(error_message(&unknown_algorithm).contains("unknown variant"));
+ }
+
+ #[test]
+ fn rejects_unknown_stage_classifier_fields() {
+ // Nested classifier typos must fail instead of silently using a default.
+ let config = format!(
+ r#"{VALID_CONFIG}
+
+[routes.stage]
+id = "switchyard/stage"
+type = "stage_router"
+capable_target = "strong"
+efficient_target = "weak"
+picker = "efficient_first"
+confidence_threshold = 1.0
+
+[routes.stage.classifier]
+target = "classifier"
+base_threshold = 0.5
+classifier_magic = true
+"#
+ );
+
+ let error = error_message(&config);
+ assert!(
+ error.contains("unknown field `classifier_magic`"),
+ "{error}"
+ );
+ }
+
+ #[test]
+ fn rejects_invalid_references_and_parameters() {
+ let cases = [
+ (
+ VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),
+ "unknown llm client missing",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"missing\"]",
+ ),
+ "unknown target missing",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"strong\"]",
+ ),
+ "random targets must be unique",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [1]",
+ ),
+ "expected 2 weights, got 1",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",
+ ),
+ "at least one weight must be positive",
+ ),
+ (
+ VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),
+ "base_threshold must be between 0 and 1",
+ ),
+ (
+ VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""),
+ "route references unknown target",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "classifier_target = \"classifier\"",
+ "classifier_target = \"\"",
+ ),
+ "route references unknown target",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "classifier_target = \"classifier\"",
+ "classifier_target = \" \"",
+ ),
+ "route references unknown target",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nthreshold_step = -0.1",
+ ),
+ "threshold_step must be finite and greater than or equal to 0",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.8\nthreshold_step = 0.11",
+ ),
+ "base_threshold + 2 * threshold_step must be at most 1",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",
+ ),
+ "max_output_tokens must be at least 1",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nmessage_hash_fallback = true",
+ ),
+ "message_hash_fallback requires classify_trigger = new_session",
+ ),
+ (
+ with_subagent_llm_classifier(
+ VALID_CONFIG,
+ "passthrough",
+ "\nmessage_hash_fallback = true",
+ ),
+ "cannot use message_hash_fallback",
+ ),
+ (
+ with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "")
+ .replace("mode = \"custom\"", "mode = \"capability\""),
+ "mode capability cannot use custom classifier fields",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"",
+ ),
+ "mode escalation cannot use classify_trigger",
+ ),
+ (
+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),
+ "unsupported schema_version 2",
+ ),
+ (
+ VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),
+ "target name must be non-empty and have no surrounding whitespace",
+ ),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\ncontext_window = 0",
+ ),
+ "route random context_window must be greater than zero",
+ ),
+ ];
+
+ for (toml, expected) in cases {
+ let error = error_message(&toml);
+ assert!(
+ error.contains(expected),
+ "expected error containing {expected}, got {error}"
+ );
+ }
+ }
+
+ #[test]
+ fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> {
+ // Two targets share one model id on one client. The client keeps one and drops the
+ // other, so the build warns and still succeeds, and both routes resolve. Serving one
+ // model under two route names this way is allowed; pointing both routes at one target
+ // is the tidier form.
+ const SAME_MODEL_TWO_ROUTES: &str = r#"
+schema_version = 1
+
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[targets.fast]
+id = "gpt-4o"
+llm_client = "primary"
+
+[targets.smart]
+id = "gpt-4o"
+llm_client = "primary"
+
+[routes.fast]
+id = "switchyard/fast"
+type = "passthrough"
+target = "fast"
+
+[routes.smart]
+id = "switchyard/smart"
+type = "passthrough"
+target = "smart"
+"#;
+ let state = runner_from_toml(SAME_MODEL_TWO_ROUTES)?;
+ assert_eq!(
+ state
+ .models()
+ .map(|model| model.id.as_str())
+ .collect::<Vec<_>>(),
+ ["switchyard/fast", "switchyard/smart"]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> {
+ // The same model id served by two llm clients never collides (each client keys its own
+ // models), so cross-provider A/B builds with no warning; only a repeat within one client
+ // warns.
+ const CROSS_PROVIDER: &str = r#"
+schema_version = 1
+
+[llm_clients.openai]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[llm_clients.azure]
+format = "openai_chat"
+base_url = "https://azure.test/v1"
+
+[targets.openai]
+id = "gpt-4o"
+llm_client = "openai"
+
+[targets.azure]
+id = "gpt-4o"
+llm_client = "azure"
+
+[routes.openai]
+id = "switchyard/openai-gpt4o"
+type = "passthrough"
+target = "openai"
+
+[routes.azure]
+id = "switchyard/azure-gpt4o"
+type = "passthrough"
+target = "azure"
+"#;
+ runner_from_toml(CROSS_PROVIDER)?;
+ Ok(())
+ }
+
+ #[test]
+ fn accepts_relative_weights_and_seed() -> RunnerResult<()> {
+ let weighted = VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",
+ );
+ runner_from_toml(&weighted)?;
+ Ok(())
+ }
+
+ #[test]
+ fn accepts_new_session_trigger_with_message_hash_fallback() -> RunnerResult<()> {
+ let configured = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true",
+ );
+ runner_from_toml(&configured)?;
+ Ok(())
+ }
+
+ #[test]
+ fn target_extra_body_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {
+ let configured = VALID_CONFIG.replacen(
+ "llm_client = \"primary\"",
+ "llm_client = \"primary\"\n\
+ extra_body = { service_tier = \"priority\", \
+ chat_template_kwargs = { enable_thinking = false } }",
+ 1,
+ );
+ let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse config: {error}"))
+ })?;
+ let Some(target) = config.targets.get("classifier") else {
+ return Err(RunnerError::configuration("classifier target is missing"));
+ };
+ let Some(client) = config.llm_clients.get("primary") else {
+ return Err(RunnerError::configuration("primary llm client is missing"));
+ };
+ let backend = build_backend("primary", client, &target.extra_body)?;
+
+ assert_eq!(
+ backend.extra_body().get("service_tier"),
+ Some(&json!("priority"))
+ );
+ assert_eq!(
+ backend
+ .extra_body()
+ .get("chat_template_kwargs")
+ .and_then(|value| value.get("enable_thinking")),
+ Some(&json!(false))
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn retry_budget_defaults_and_accepts_an_override() -> RunnerResult<()> {
+ let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse default config: {error}"))
+ })?;
+ let Some(primary) = default.llm_clients.get("primary") else {
+ return Err(RunnerError::configuration("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);
+
+ let explicit = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = 0",
+ 1,
+ );
+ let config: DeploymentConfig = toml::from_str(&explicit).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse explicit retry config: {error}"))
+ })?;
+ let Some(primary) = config.llm_clients.get("primary") else {
+ return Err(RunnerError::configuration("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, 0);
+
+ let maximum = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ &format!(
+ "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"
+ ),
+ 1,
+ );
+ let config: DeploymentConfig = toml::from_str(&maximum).map_err(|error| {
+ RunnerError::configuration(format!("failed to parse maximum retry config: {error}"))
+ })?;
+ let Some(primary) = config.llm_clients.get("primary") else {
+ return Err(RunnerError::configuration("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);
+ Ok(())
+ }
+
+ #[test]
+ fn rejects_headers_that_switchyard_sets() {
+ let cases = [
+ (
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\n\
+ extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }",
+ "AUTHORIZATION",
+ ),
+ (
+ "base_url = \"https://example.test\"",
+ "base_url = \"https://example.test\"\n\
+ extra_headers = { \"X-Api-Key\" = \"custom-key\" }",
+ "X-Api-Key",
+ ),
+ (
+ "base_url = \"https://example.test\"",
+ "base_url = \"https://example.test\"\n\
+ extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }",
+ "ANTHROPIC-VERSION",
+ ),
+ ];
+
+ for (original, replacement, header) in cases {
+ let configured = VALID_CONFIG.replacen(original, replacement, 1);
+ let error = error_message(&configured);
+ assert!(
+ error.contains(&format!("extra_headers cannot set {header:?}")),
+ "expected {header} to be rejected, got: {error}"
+ );
+ }
+ }
+
+ #[test]
+ fn accepts_additional_headers() -> RunnerResult<()> {
+ let configured = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\n\
+ extra_headers = { X-Inference-Priority = \"batch\" }",
+ 1,
+ );
+
+ runner_from_toml(&configured)?;
+ Ok(())
+ }
+
+ #[test]
+ fn retry_budget_rejects_negative_values() {
+ let invalid = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = -1",
+ 1,
+ );
+ assert!(error_message(&invalid).contains("max_retries"));
+ }
+
+ #[test]
+ fn retry_budget_rejects_excessive_values() {
+ let invalid = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = 11",
+ 1,
+ );
+ assert!(
+ error_message(&invalid).contains("llm client primary max_retries must be at most 10")
+ );
+ }
+
+ #[test]
+ fn api_key_environment_reference_is_validated() {
+ let missing = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",
+ 1,
+ );
+ assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));
+
+ const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";
+ unsafe {
+ // "unsafe" is for concurrent reads and writes, very rare
+ std::env::set_var(EMPTY_KEY_ENV, "");
+ }
+ let empty = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),
+ 1,
+ );
+ let message = error_message(&empty);
+ unsafe {
+ std::env::remove_var(EMPTY_KEY_ENV);
+ }
+ assert!(message.contains("is empty"));
+ }
+
+ #[test]
+ fn forward_auth_rejects_conflicting_credentials() {
+ let competing_auth = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\n\
+ forward_auth = true\n\
+ api_key_env = \"UNUSED_TEST_KEY\"",
+ 1,
+ );
+ assert!(
+ error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env")
+ );
+
+ let static_auth = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test\"",
+ "base_url = \"https://example.test\"\n\
+ forward_auth = true\n\
+ extra_headers = { Authorization = \"static-value\" }",
+ 1,
+ );
+ assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\""));
+
+ let static_beta = static_auth.replace("Authorization", "anthropic-beta");
+ assert!(
+ error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"")
+ );
+
+ for header in ["chatgpt-account-id", "x-openai-fedramp"] {
+ let static_context = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ &format!(
+ "base_url = \"https://example.test/v1\"\n\
+ forward_auth = true\n\
+ extra_headers = {{ \"{header}\" = \"static-value\" }}"
+ ),
+ 1,
+ );
+ assert!(
+ error_message(&static_context)
+ .contains(&format!("extra_headers cannot set \"{header}\""))
+ );
+ }
+ }
+
+ const ADVISOR_CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.anthropic]
+format = "anthropic_messages"
+base_url = "https://example.test"
+
+[targets.executor]
+id = "executor/model"
+llm_client = "anthropic"
+
+[targets.advisor]
+id = "advisor/model"
+llm_client = "anthropic"
+
+[routes.gated]
+id = "switchyard/advisor"
+type = "advisor"
+executor_target = "executor"
+advisor_target = "advisor"
+"#;
+
+ #[test]
+ fn advisor_route_parses_with_defaults_and_builds() -> RunnerResult<()> {
+ let state = runner_from_toml(ADVISOR_CONFIG)?;
+ assert_eq!(
+ state
+ .models()
+ .map(|model| model.id.as_str())
+ .collect::<Vec<_>>(),
+ ["switchyard/advisor"]
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn advisor_route_accepts_every_gate_knob() -> RunnerResult<()> {
+ let tuned = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ concat!(
+ "advisor_target = \"advisor\"\n",
+ "reviewer_system_prompt = \"review it\"\n",
+ "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",
+ "gate_trigger = \"pattern\"\n",
+ "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",
+ "max_reviews = 2\n",
+ "gate_stall_turns = 40\n",
+ "gate_min_tool_results = 1\n",
+ "advisor_max_tokens = 1024\n",
+ "advisor_temperature = 0.0\n",
+ "transcript_max_chars = 100000\n",
+ "fail_open = false\n",
+ "context_window = 200000\n",
+ "tool_calling = true\n",
+ "reasoning = true",
+ ),
+ );
+ runner_from_toml(&tuned)?;
+ Ok(())
+ }
+
+ #[test]
+ fn advisor_route_rejects_unknown_keys() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\nbogus_field = 1",
+ );
+ assert!(error_message(&invalid).contains("bogus_field"));
+ }
+
+ #[test]
+ fn advisor_route_requires_both_targets() {
+ let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");
+ assert!(error_message(&missing).contains("advisor_target"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_unknown_target() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"missing\"",
+ );
+ assert!(error_message(&invalid).contains("missing"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_invalid_pattern() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",
+ );
+ assert!(error_message(&invalid).contains("not a valid regex"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_pattern_without_pattern_trigger() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",
+ );
+ assert!(
+ error_message(&invalid)
+ .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")
+ );
+ }
+
+ #[test]
+ fn advisor_route_pattern_trigger_requires_pattern() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",
+ );
+ assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_zero_max_reviews() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\nmax_reviews = 0",
+ );
+ assert!(error_message(&invalid).contains("max_reviews must be at least 1"));
+ }
+}
diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs
new file mode 100644
index 000000000..d69f2da01
--- /dev/null
+++ b/crates/switchyard-runner/src/lib.rs
@@ -0,0 +1,18 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared configured routing for Switchyard serving surfaces.
+
+mod algorithm;
+mod config;
+mod route;
+mod runner;
+
+pub use algorithm::{
+ AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, ClassifierMode,
+ ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig,
+};
+pub use route::{
+ CallerAuthKind, CountTokensTarget, ModelCapabilities, Route, RunOutput, RunnerError,
+};
+pub use runner::{DecisionDescription, DecisionTarget, ModelInfo, Runner};
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
new file mode 100644
index 000000000..dc4db13ee
--- /dev/null
+++ b/crates/switchyard-runner/src/route.rs
@@ -0,0 +1,263 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! One configured algorithm and the clients that serve its targets.
+
+use std::error::Error;
+use std::sync::Arc;
+
+use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};
+use serde_json::Value;
+use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient};
+use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat};
+use thiserror::Error;
+
+use crate::DecisionTarget;
+
+/// Capabilities that one route advertises on `GET /v1/models`.
+///
+/// An unset capability is undeclared: it serializes as `null` in the OpenAI
+/// `data` entry, and the Codex entry falls back to a safe default for it.
+#[derive(Clone, Copy, Default)]
+pub struct ModelCapabilities {
+ pub context_window: Option<u32>,
+ pub tool_calling: Option<bool>,
+ /// Whether the routed model takes reasoning controls. A serving surface cannot
+ /// probe this, so a route opts in via config; undeclared routes advertise as
+ /// non-reasoning to Codex (fail closed).
+ pub reasoning: Option<bool>,
+}
+
+/// Caller credential family required by a forwarded-auth route.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum CallerAuthKind {
+ Anthropic,
+ OpenAi,
+}
+
+impl CallerAuthKind {
+ /// Stable provider name used by the server compatibility API.
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Anthropic => "anthropic",
+ Self::OpenAi => "openai",
+ }
+ }
+
+ const fn accepts(self, wire_format: WireFormat) -> bool {
+ matches!(
+ (self, wire_format),
+ (Self::Anthropic, WireFormat::AnthropicMessages)
+ | (
+ Self::OpenAi,
+ WireFormat::OpenAiChat | WireFormat::OpenAiResponses
+ )
+ )
+ }
+}
+
+/// Exact upstream model used for Anthropic token counting.
+#[derive(Clone)]
+pub struct CountTokensTarget {
+ pub model: ModelId,
+ pub client: Arc<TranslatingLlmClient>,
+}
+
+/// Error returned while loading or executing configured routes.
+#[derive(Debug, Error)]
+pub enum RunnerError {
+ #[error("{message}")]
+ Configuration {
+ message: String,
+ #[source]
+ source: Option<Box<dyn Error + Send + Sync>>,
+ },
+ #[error("unknown route model {0:?}")]
+ UnknownRouteModel(String),
+ #[error("caller format is incompatible with {} credentials", .0.as_str())]
+ IncompatibleCallerFormat(CallerAuthKind),
+ #[error("route has no Anthropic target for token counting")]
+ CountTokensUnsupported,
+ #[error(transparent)]
+ Algorithm(#[from] LibsyError),
+ #[error(transparent)]
+ Client(#[from] LlmClientError),
+}
+
+impl RunnerError {
+ pub(crate) fn configuration(message: impl Into<String>) -> Self {
+ Self::Configuration {
+ message: message.into(),
+ source: None,
+ }
+ }
+
+ pub(crate) fn configuration_source(
+ message: impl Into<String>,
+ source: impl Error + Send + Sync + 'static,
+ ) -> Self {
+ Self::Configuration {
+ message: message.into(),
+ source: Some(Box::new(source)),
+ }
+ }
+}
+
+/// A configured algorithm and the per-target clients its calls resolve through.
+pub struct Route {
+ algorithm: Arc<dyn Algorithm>,
+ // Resolves each offloaded call to the client configured for the target the algorithm
+ // selected. A route is a synthetic model with no upstream of its own, so this is a
+ // per-target lookup, never one client serving the whole route.
+ clients: ClientRouter,
+ caller_auth: Option<CallerAuthKind>,
+ capabilities: ModelCapabilities,
+ count_tokens_target: Option<CountTokensTarget>,
+ decision_targets: Vec<DecisionTarget>,
+}
+
+/// The selected model and untouched response produced by a route execution.
+pub struct RunOutput {
+ pub selected_model: ModelId,
+ pub response: Response,
+}
+
+impl Route {
+ /// Creates a fully configured execution route.
+ pub fn new(
+ algorithm: Arc<dyn Algorithm>,
+ clients: ClientRouter,
+ caller_auth: Option<CallerAuthKind>,
+ capabilities: ModelCapabilities,
+ count_tokens_target: Option<CountTokensTarget>,
+ decision_targets: Vec<DecisionTarget>,
+ ) -> Self {
+ Self {
+ algorithm,
+ clients,
+ caller_auth,
+ capabilities,
+ count_tokens_target,
+ decision_targets,
+ }
+ }
+
+ /// Returns the configured libsy algorithm name.
+ pub fn algorithm_name(&self) -> &str {
+ self.algorithm.name()
+ }
+
+ /// Returns model-list capability metadata.
+ pub fn capabilities(&self) -> ModelCapabilities {
+ self.capabilities
+ }
+
+ /// Returns the forwarded caller credential family.
+ pub fn caller_auth(&self) -> Option<CallerAuthKind> {
+ self.caller_auth
+ }
+
+ /// Resolves a selected model to this route's non-secret target metadata.
+ pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {
+ self.decision_targets
+ .iter()
+ .find(|target| target.model == *model)
+ .cloned()
+ }
+
+ /// Rejects a caller format incompatible with forwarded credentials.
+ pub fn check_caller_format(&self, input_format: WireFormat) -> Result<(), RunnerError> {
+ if let Some(kind) = self.caller_auth
+ && !kind.accepts(input_format)
+ {
+ return Err(RunnerError::IncompatibleCallerFormat(kind));
+ }
+ Ok(())
+ }
+
+ /// Executes the configured route without consuming or proxying streamed responses.
+ pub async fn execute(
+ &self,
+ request: Request,
+ observer: Option<RunObserver>,
+ ) -> Result<RunOutput, RunnerError> {
+ let (selected_model, response) = switchyard_llm_client::run(
+ Arc::clone(&self.algorithm),
+ self.clients.clone(),
+ request,
+ observer,
+ )
+ .await?;
+ Ok(RunOutput {
+ selected_model,
+ response,
+ })
+ }
+
+ /// Completes routing-time calls without serving the answer target.
+ pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {
+ drive(Arc::clone(&self.algorithm), request, |call| {
+ serve_decision_dependency(self.clients.clone(), call)
+ })
+ .await
+ .map_err(Into::into)
+ }
+
+ /// Counts tokens using the configured Anthropic-capable target.
+ pub async fn count_tokens(&self, request: Request) -> Result<Value, RunnerError> {
+ let target = self
+ .count_tokens_target
+ .as_ref()
+ .ok_or(RunnerError::CountTokensUnsupported)?;
+ target
+ .client
+ .count_tokens(&target.model, request)
+ .await
+ .map_err(Into::into)
+ }
+}
+
+async fn serve_decision_dependency(clients: ClientRouter, call: CallModel) -> libsy::Result<()> {
+ let mut result = Err(LibsyError::NoTargets);
+ for (index, model) in call.models.iter().enumerate() {
+ // The driver stamps only the first candidate, so every fallback must replace it.
+ let mut request = call.request.clone();
+ request.llm_request.model = Some(model.to_string());
+ let response = match clients.route(model) {
+ Ok(client) => client.call(request).await,
+ Err(source) => Err(source),
+ };
+ match response {
+ Ok(response) => {
+ result = Ok(response);
+ break;
+ }
+ Err(source) => {
+ let try_next = index + 1 < call.models.len() && eligible_routing_fallback(&source);
+ result = Err(LibsyError::client_call(model.clone(), source));
+ if !try_next {
+ break;
+ }
+ }
+ }
+ }
+ call.respond(result)
+}
+
+/// Whether a routing-time candidate failure may fall through to the next model.
+fn eligible_routing_fallback(error: &LlmClientError) -> bool {
+ match error {
+ LlmClientError::ContextWindowExceeded { .. }
+ | LlmClientError::Transport { .. }
+ | LlmClientError::Timeout { .. } => true,
+ LlmClientError::UpstreamHttp { status, .. } => {
+ matches!(
+ *status,
+ reqwest::StatusCode::FORBIDDEN
+ | reqwest::StatusCode::REQUEST_TIMEOUT
+ | reqwest::StatusCode::TOO_MANY_REQUESTS
+ ) || status.is_server_error()
+ }
+ _ => false,
+ }
+}
diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs
new file mode 100644
index 000000000..55760f3a3
--- /dev/null
+++ b/crates/switchyard-runner/src/runner.rs
@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Named route table and server-facing route metadata.
+
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use libsy::RoutingOutcome;
+use serde_json::Value;
+use switchyard_protocol::{ModelId, WireFormat};
+
+use crate::config;
+use crate::{ModelCapabilities, Route, RunnerError};
+
+/// Immutable named route table.
+pub struct Runner {
+ routes: Vec<(ModelId, Route)>,
+}
+
+/// Borrowed model metadata returned while listing routes.
+pub struct ModelInfo<'a> {
+ pub id: &'a ModelId,
+ pub algorithm: &'a str,
+ pub capabilities: ModelCapabilities,
+}
+
+/// Fully resolved routing decision.
+pub struct DecisionDescription {
+ pub selected: DecisionTarget,
+ pub fallbacks: Vec<DecisionTarget>,
+}
+
+/// Non-secret configured target details returned by the decision endpoint.
+#[derive(Clone)]
+pub struct DecisionTarget {
+ pub target: String,
+ pub model: ModelId,
+ pub format: WireFormat,
+ pub base_url: String,
+ pub extra_body: BTreeMap<String, Value>,
+}
+
+impl Runner {
+ /// Loads and validates a version-1 deployment TOML file.
+ pub fn load(path: impl AsRef<Path>) -> Result<Self, RunnerError> {
+ config::load_runner(path)
+ }
+
+ /// Builds a runner from named routes in caller-provided order.
+ /// Pre-condition: There must be at least one route.
+ pub fn new(routes: Vec<(ModelId, Route)>) -> Self {
+ Self { routes }
+ }
+
+ /// Returns the route registered for a model.
+ pub fn route(&self, model: &str) -> Option<&Route> {
+ self.routes
+ .iter()
+ .find(|(id, _)| id.as_str() == model)
+ .map(|(_, route)| route)
+ }
+
+ /// Iterates over configured routes in caller-provided order.
+ pub fn models(&self) -> impl Iterator<Item = ModelInfo<'_>> {
+ self.routes.iter().map(|(id, route)| ModelInfo {
+ id,
+ algorithm: route.algorithm_name(),
+ capabilities: route.capabilities(),
+ })
+ }
+
+ /// Resolves an outcome to configured target names and non-secret client settings.
+ pub fn describe_decision(
+ &self,
+ model: &ModelId,
+ outcome: &RoutingOutcome,
+ ) -> Option<DecisionDescription> {
+ let route = self.route(model.as_str())?;
+ let resolve = |selected: &ModelId| route.decision_target(selected);
+ Some(DecisionDescription {
+ selected: resolve(&outcome.selected_model_id)?,
+ fallbacks: outcome
+ .fallback_models
+ .iter()
+ .map(resolve)
+ .collect::<Option<Vec<_>>>()?,
+ })
+ }
+}
diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs
new file mode 100644
index 000000000..71ce57c7e
--- /dev/null
+++ b/crates/switchyard-runner/tests/route.rs
@@ -0,0 +1,164 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::collections::BTreeMap;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex};
+
+use async_trait::async_trait;
+use futures_util::StreamExt;
+use switchyard_llm_client::{ClientRouter, RunObservation};
+use switchyard_protocol::{
+ LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, text_request,
+ text_response,
+};
+use switchyard_runner::{AlgorithmSpec, ModelCapabilities, Route};
+
+struct StubClient;
+
+#[async_trait]
+impl RoutedLlmClient for StubClient {
+ async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ request.llm_request.model.clone(),
+ "plugin response",
+ )),
+ metadata: None,
+ })
+ }
+}
+
+fn plugin_route(client: Arc<dyn RoutedLlmClient>) -> Route {
+ let spec = AlgorithmSpec::Passthrough {
+ target: "semantic-target".to_string(),
+ subagents: None,
+ };
+ let targets = BTreeMap::from([(
+ "semantic-target".to_string(),
+ ModelId::from("semantic-target"),
+ )]);
+ let algorithm = spec
+ .build("switchyard", &targets)
+ .expect("identity target map should build");
+ let clients = ClientRouter::new(
+ BTreeMap::from([(ModelId::from("semantic-target"), client)])
+ .into_iter()
+ .collect(),
+ );
+ Route::new(
+ algorithm,
+ clients,
+ None,
+ ModelCapabilities::default(),
+ None,
+ Vec::new(),
+ )
+}
+
+#[tokio::test]
+async fn plugin_shaped_route_executes_without_runner_model_or_toml() {
+ let route = plugin_route(Arc::new(StubClient));
+ let observations = Arc::new(Mutex::new(Vec::new()));
+ let observer = {
+ let observations = Arc::clone(&observations);
+ Arc::new(move |observation| observations.lock().unwrap().push(observation))
+ };
+ let request = Request {
+ llm_request: text_request(Some("arbitrary-upstream-model".to_string()), "hello"),
+ ..Request::default()
+ };
+
+ let output = route
+ .execute(request, Some(observer))
+ .await
+ .expect("route should execute");
+
+ assert_eq!(output.selected_model, "semantic-target");
+ assert_eq!(
+ output
+ .response
+ .llm_response
+ .as_agg()
+ .unwrap()
+ .model
+ .as_deref(),
+ Some("semantic-target")
+ );
+ assert!(
+ observations
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|observation| { matches!(observation, RunObservation::AnswerCall(_)) })
+ );
+ assert!(
+ observations
+ .lock()
+ .unwrap()
+ .iter()
+ .any(|observation| { matches!(observation, RunObservation::RoutingOverhead(_)) })
+ );
+}
+
+struct LazyStreamClient {
+ polls: Arc<AtomicUsize>,
+}
+
+#[async_trait]
+impl RoutedLlmClient for LazyStreamClient {
+ async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {
+ let polls = Arc::clone(&self.polls);
+ let stream = futures_util::stream::poll_fn(move |_| {
+ polls.fetch_add(1, Ordering::SeqCst);
+ std::task::Poll::Ready(None)
+ })
+ .boxed();
+ Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ })
+ }
+}
+
+#[tokio::test]
+async fn route_returns_stream_without_polling_it() {
+ let polls = Arc::new(AtomicUsize::new(0));
+ let route = plugin_route(Arc::new(LazyStreamClient {
+ polls: Arc::clone(&polls),
+ }));
+ let request = Request {
+ llm_request: text_request(None, "hello"),
+ ..Request::default()
+ };
+
+ let output = route
+ .execute(request, None)
+ .await
+ .expect("stream handle should be returned");
+
+ assert!(matches!(
+ output.response.llm_response,
+ LlmResponse::Stream(_)
+ ));
+ assert_eq!(polls.load(Ordering::SeqCst), 0);
+}
+
+#[test]
+fn algorithm_build_reports_unknown_configured_target() {
+ let spec = AlgorithmSpec::Random {
+ targets: vec!["missing".to_string()],
+ weights: None,
+ seed: None,
+ };
+
+ let error = match spec.build("plugin", &BTreeMap::new()) {
+ Ok(_) => panic!("unknown target should fail"),
+ Err(error) => error,
+ };
+
+ assert_eq!(
+ error.to_string(),
+ "route plugin references unknown target missing"
+ );
+}
diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md
index efd438109..ae40d38eb 100644
--- a/crates/switchyard-server/CONFIGURATION.md
+++ b/crates/switchyard-server/CONFIGURATION.md
@@ -26,11 +26,11 @@ target only when the upstream supports it and would otherwise return the verdict
outside normal assistant `content`.
To support another wire format, add its `ClientFormat` variant and explicit construction match in
-`src/config.rs`. Add a client type only when a second implementation exists.
+`../switchyard-runner/src/config.rs`. Add a client type only when a second implementation exists.
## Add an algorithm
1. Implement and export the algorithm from `libsy`.
-2. Add its TOML fields as an `AlgorithmConfig` variant in `src/config.rs`.
-3. Construct it in the `build_algorithm` match, resolving target names with `resolve_targets`.
+2. Add its fields as an `AlgorithmSpec` variant in `../switchyard-runner/src/algorithm.rs`.
+3. Construct it in that module's builder, resolving names through the caller-supplied target map.
4. Add a parsing test and an end-to-end server test when the algorithm makes LLM calls.
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 5502a0e8c..0c4b47e24 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -32,11 +32,10 @@ opentelemetry-prometheus = "0.32"
opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] }
parking_lot.workspace = true
prometheus = "0.14"
-reqwest.workspace = true
serde.workspace = true
-toml = "1.1"
switchyard-llm-client.workspace = true
switchyard-protocol.workspace = true
+switchyard-runner.workspace = true
serde_json.workspace = true
switchyard-translation.workspace = true
humantime = "2.4"
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 77373959b..ba88b3445 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -1,2262 +1,15 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Typed TOML configuration and explicit construction for the Rust server.
+//! Compatibility wrapper for loading shared runner configuration.
-use std::collections::{BTreeMap, HashMap, HashSet};
-use std::fs;
use std::path::Path;
-use std::sync::Arc;
-use libsy::{
- AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,
- ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig,
- GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop,
- Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter,
- SubagentRouterConfig, TargetPrompts, TaskClassifierConfig,
-};
-use serde::Deserialize;
-use serde_json::Value;
-use switchyard_llm_client::{
- Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
- TranslatingLlmClient,
-};
-use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};
+use switchyard_runner::Runner;
-use crate::{
- CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState,
-};
-
-const SUPPORTED_SCHEMA_VERSION: u32 = 1;
-const MAX_CONFIGURED_RETRIES: u32 = 10;
+use crate::{ServerError, ServerResult, ServerState};
/// Loads a TOML deployment file and constructs the complete server state.
pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {
- let path = path.as_ref();
- let toml = fs::read_to_string(path).map_err(|error| {
- ServerError::new(format!(
- "failed to read server config {}: {error}",
- path.display()
- ))
- })?;
- server_state_from_toml(&toml).map_err(|error| {
- ServerError::new(format!("invalid server config {}: {error}", path.display()))
- })
-}
-
-fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {
- let config: Arc<ServerConfig> = Arc::new(
- toml::from_str(toml)
- .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?,
- );
- let state = config.build()?;
- Ok(state.with_config(config))
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct ServerConfig {
- schema_version: u32,
- #[serde(default)]
- pub(crate) llm_clients: BTreeMap<String, LlmClientConfig>,
- pub(crate) targets: BTreeMap<String, TargetConfig>,
- routes: BTreeMap<String, RouteConfig>,
-}
-
-impl ServerConfig {
- pub(crate) fn routing_target_names(&self, route_name: &str) -> Option<Vec<&str>> {
- self.routes
- .get(route_name)
- .map(RouteConfig::routing_target_names)
- }
-
- fn build(&self) -> ServerResult<ServerState> {
- if self.schema_version != SUPPORTED_SCHEMA_VERSION {
- return Err(ServerError::new(format!(
- "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",
- self.schema_version
- )));
- }
-
- // An llm client keys its models by id, so two targets that share an id on one
- // client end up as one entry: the client keeps one target and drops the other. Warn
- // so the drop is visible at startup instead of silent, and still build both routes.
- // The set only tells us the pair was already seen, not whether the two targets
- // differ, so it warns for a harmless duplicate and for one whose extra_body would be
- // dropped alike. The same id on two different clients never collides: each client
- // keeps its own model.
- let mut seen_client_model_ids = HashSet::new();
- for (target_name, target) in &self.targets {
- validate_value("target name", target_name)?;
- validate_value(&format!("target {target_name} id"), &target.id)?;
- if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) {
- tracing::warn!(
- "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",
- target.id,
- target.llm_client
- );
- }
- }
-
- let clients = self.build_clients()?;
- let targets = self.build_targets()?;
- let mut routes = Vec::with_capacity(self.routes.len());
- for (route_name, config) in &self.routes {
- validate_value("route name", route_name)?;
- validate_value(&format!("route {route_name} id"), config.id())?;
- for target_name in config.callable_target_names() {
- self.targets.get(target_name).ok_or_else(|| {
- ServerError::new(format!("route references unknown target {target_name}"))
- })?;
- }
- let capabilities = config.capabilities();
- if capabilities.context_window == Some(0) {
- return Err(ServerError::new(format!(
- "route {route_name} context_window must be greater than zero"
- )));
- }
- let algorithm = build_algorithm(route_name, config, &targets)?;
- let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?;
- let count_tokens_target = self.build_count_tokens_target(config, &clients);
- routes.push((
- config.id().clone(),
- algorithm,
- client,
- caller_auth,
- capabilities,
- count_tokens_target,
- Some(route_name.clone()),
- ));
- }
- ServerState::new_with_capabilities(routes)
- }
-
- fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {
- let mut models_by_client = self
- .llm_clients
- .keys()
- .map(|name| (name.clone(), Vec::new()))
- .collect::<BTreeMap<String, Vec<ModelConfig>>>();
-
- // Validate every declared client even when no target currently references it.
- for (name, client_config) in &self.llm_clients {
- validate_value("llm client name", name)?;
- build_backend(name, client_config, &BTreeMap::new())?;
- }
- for (target_name, target) in &self.targets {
- let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
- ServerError::new(format!(
- "target {target_name} references unknown llm client {}",
- target.llm_client
- ))
- })?;
- let model_configs = models_by_client
- .get_mut(&target.llm_client)
- .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;
- model_configs.push(ModelConfig::new(
- target.id.clone(),
- build_backend(&target.llm_client, client_config, &target.extra_body)?,
- None,
- ));
- }
-
- let mut clients = BTreeMap::new();
- for (name, model_configs) in models_by_client {
- let client = Arc::new(
- TranslatingLlmClient::new(&model_configs)
- .map_err(|error| ServerError::new(error.to_string()))?,
- );
- clients.insert(name, client);
- }
- Ok(clients)
- }
-
- fn build_targets(&self) -> ServerResult<BTreeMap<String, ModelId>> {
- Ok(self
- .targets
- .iter()
- .map(|(name, config)| (name.clone(), config.id.clone()))
- .collect())
- }
-
- /// Maps every model this route can select to the client configured for that target.
- ///
- /// A route is a synthetic model (`switchyard/random`) with no upstream of its own — the
- /// algorithm picks a real target, and the *target* names the `[llm_clients.*]` section
- /// that serves it. Two targets in one route may therefore sit on different clients, and
- /// one request can emit calls for both.
- ///
- /// The map is built per route because targets are only reachable through the routes that
- /// name them, so the same model id may resolve to different clients in two routes without
- /// colliding.
- fn build_route_clients(
- &self,
- route_name: &str,
- route: &RouteConfig,
- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
- ) -> ServerResult<(ClientRouter, Option<CallerAuthKind>)> {
- let mut by_model = HashMap::new();
- let mut caller_auth = None;
- for name in route.callable_target_names() {
- let target = self.targets.get(name).ok_or_else(|| {
- ServerError::new(format!("route references unknown target {name}"))
- })?;
- let client = clients.get(&target.llm_client).ok_or_else(|| {
- ServerError::new(format!("target {name} has no constructed llm client"))
- })?;
- let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {
- ServerError::new(format!(
- "target {name} references unknown llm client {}",
- target.llm_client
- ))
- })?;
- if client_config.forward_auth {
- let target_auth = client_config.format.caller_auth_kind();
- if caller_auth.is_some_and(|kind| kind != target_auth) {
- return Err(ServerError::new(format!(
- "route {route_name} cannot forward both Anthropic and OpenAI caller credentials"
- )));
- }
- caller_auth = Some(target_auth);
- }
- let client: Arc<dyn RoutedLlmClient> = client.clone();
- by_model.insert(target.id.clone(), client);
- }
- Ok((ClientRouter::new(by_model), caller_auth))
- }
-
- fn build_count_tokens_target(
- &self,
- route_config: &RouteConfig,
- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,
- ) -> Option<CountTokensTarget> {
- route_config
- .routing_target_names()
- .into_iter()
- .enumerate()
- .filter_map(|(index, name)| {
- let target = self.targets.get(name)?;
- let client = clients.get(&target.llm_client)?;
- client.supports_count_tokens(&target.id).then_some((
- count_tokens_priority(name, &target.id),
- index,
- target,
- client,
- ))
- })
- .min_by_key(|(priority, index, _, _)| (*priority, *index))
- .map(|(_, _, target, client)| CountTokensTarget {
- model: target.id.clone(),
- client: client.clone(),
- })
- }
-}
-
-// Prefer known Claude families, then preserve the route's target order.
-fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
- let target_name = target_name.to_ascii_lowercase();
- let model_id = model_id.to_ascii_lowercase();
- ["opus", "sonnet", "haiku"]
- .iter()
- .position(|hint| target_name.contains(hint) || model_id.contains(hint))
- .unwrap_or(3)
-}
-
-/// A client endpoint, parsed when the config loads rather than checked afterwards.
-///
-/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no
-/// later stage has to re-check it or can forget to.
-#[derive(Clone, Debug)]
-pub(crate) struct HttpBaseUrl(reqwest::Url);
-
-impl HttpBaseUrl {
- pub(crate) fn as_str(&self) -> &str {
- self.0.as_str()
- }
-}
-
-impl<'de> Deserialize<'de> for HttpBaseUrl {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: serde::Deserializer<'de>,
- {
- let raw = String::deserialize(deserializer)?;
- let url = reqwest::Url::parse(raw.trim()).map_err(|error| {
- serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}"))
- })?;
- if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {
- return Err(serde::de::Error::custom(
- "base_url must be an absolute HTTP(S) URL",
- ));
- }
- Ok(Self(url))
- }
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct LlmClientConfig {
- pub(crate) format: ClientFormat,
- pub(crate) base_url: HttpBaseUrl,
- api_key_env: Option<String>,
- #[serde(default)]
- forward_auth: bool,
- #[serde(default)]
- extra_headers: BTreeMap<String, String>,
- #[serde(default = "default_max_retries")]
- max_retries: u32,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-pub(crate) struct TargetConfig {
- pub(crate) id: ModelId,
- pub(crate) llm_client: String,
- #[serde(default)]
- pub(crate) extra_body: BTreeMap<String, Value>,
-}
-
-#[derive(Clone, Copy, Debug, Deserialize)]
-pub(crate) enum ClientFormat {
- #[serde(rename = "openai_chat")]
- OpenAiChat,
- #[serde(rename = "openai_responses")]
- OpenAiResponses,
- #[serde(rename = "anthropic_messages")]
- AnthropicMessages,
-}
-
-impl ClientFormat {
- pub(crate) const fn wire_format(self) -> WireFormat {
- match self {
- Self::OpenAiChat => WireFormat::OpenAiChat,
- Self::OpenAiResponses => WireFormat::OpenAiResponses,
- Self::AnthropicMessages => WireFormat::AnthropicMessages,
- }
- }
-
- const fn caller_auth_kind(self) -> CallerAuthKind {
- match self {
- Self::AnthropicMessages => CallerAuthKind::Anthropic,
- Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi,
- }
- }
-}
-
-#[derive(Clone, Debug, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
-enum ClassifierPolicyConfig {
- TargetSelector { selector: String },
-}
-
-#[derive(Clone, Copy, Debug, Deserialize)]
-#[serde(rename_all = "snake_case")]
-enum ClassifierMode {
- Capability,
- Escalation,
- Custom,
-}
-
-impl ClassifierPolicyConfig {
- fn into_libsy(self) -> CustomClassifierPolicy {
- match self {
- Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector),
- }
- }
-}
-
-#[derive(Debug)]
-enum LlmClassifierModeConfig {
- Capability(CapabilityClassifierRouteConfig),
- Escalation(EscalationClassifierRouteConfig),
- Custom(CustomClassifierRouteConfig),
-}
-
-#[derive(Debug)]
-struct CapabilityClassifierRouteConfig {
- strong_target: String,
- weak_target: String,
- base_threshold: f64,
- threshold_step: f64,
- classify_trigger: ClassifyTrigger,
- message_hash_fallback: bool,
- recent_turn_window: Option<usize>,
- prompt: Option<String>,
- response_format_type: ClassifierResponseFormat,
- max_output_tokens: u64,
-}
-
-#[derive(Debug)]
-struct EscalationClassifierRouteConfig {
- strong_target: String,
- weak_target: String,
- prompt: Option<String>,
- response_format_type: ClassifierResponseFormat,
- max_output_tokens: u64,
- judge: EscalationJudgeConfig,
-}
-
-#[derive(Debug)]
-struct CustomClassifierRouteConfig {
- classifier_target: String,
- targets: Vec<String>,
- default_target: String,
- prompt: String,
- response_schema: String,
- policy: ClassifierPolicyConfig,
- classify_trigger: ClassifyTrigger,
- message_hash_fallback: bool,
- recent_turn_window: Option<usize>,
- max_output_tokens: u64,
-}
-
-#[derive(Debug, Default, Deserialize)]
-#[serde(default, deny_unknown_fields)]
-struct LlmClassifierRouteConfig {
- classifier_target: String,
- mode: Option<ClassifierMode>,
- strong_target: Option<String>,
- weak_target: Option<String>,
- base_threshold: Option<f64>,
- threshold_step: Option<f64>,
- classify_trigger: ClassifyTrigger,
- message_hash_fallback: bool,
- recent_turn_window: Option<usize>,
- prompt: Option<String>,
- response_format_type: ClassifierResponseFormat,
- #[serde(default = "default_classifier_max_output_tokens")]
- max_output_tokens: u64,
- escalation: Option<EscalationJudgeConfig>,
- targets: Option<Vec<String>>,
- default_target: Option<String>,
- response_schema: Option<String>,
- policy: Option<ClassifierPolicyConfig>,
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
-enum SubagentRouteConfig {
- Passthrough { target: String },
- LlmClassifier(Box<LlmClassifierRouteConfig>),
-}
-
-impl SubagentRouteConfig {
- fn routing_target_names(&self) -> Vec<&str> {
- match self {
- Self::Passthrough { target } => vec![target],
- Self::LlmClassifier(classifier) => classifier
- .targets
- .iter()
- .flatten()
- .map(String::as_str)
- .collect(),
- }
- }
-
- fn classifier_target_name(&self) -> Option<&str> {
- match self {
- Self::LlmClassifier(classifier) => Some(&classifier.classifier_target),
- Self::Passthrough { .. } => None,
- }
- }
-}
-
-#[derive(Debug, Deserialize)]
-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
-enum RouteConfig {
- Noop {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- },
- Random {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- targets: Vec<String>,
- weights: Option<Vec<f64>>,
- seed: Option<u64>,
- },
- Passthrough {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- target: String,
- #[serde(default)]
- subagents: Option<SubagentRouteConfig>,
- },
- LlmClassifier {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- #[serde(flatten)]
- config: LlmClassifierRouteConfig,
- },
- StageRouter {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- capable_target: String,
- efficient_target: String,
- /// Tier a turn falls back to when the signals are not confident.
- picker: PickerMode,
- confidence_threshold: f64,
- /// Trailing tool results the signals are computed over.
- #[serde(default)]
- recent_turn_window: Option<usize>,
- /// Note handed to the model a signal-driven switch routes to.
- #[serde(default)]
- handoff_notes: Option<HandoffNoteConfig>,
- /// System prompt handed to each tier on every turn it serves.
- #[serde(default)]
- capable_system_prompt: Option<String>,
- #[serde(default)]
- efficient_system_prompt: Option<String>,
- /// Capability judge consulted on turns the signals leave undecided.
- #[serde(default)]
- classifier: Option<StageClassifierConfig>,
- /// Optional routing applied only to delegated sub-agent work.
- #[serde(default)]
- subagents: Option<SubagentRouteConfig>,
- },
- Advisor {
- id: ModelId,
- #[serde(default)]
- context_window: Option<u32>,
- #[serde(default)]
- tool_calling: Option<bool>,
- #[serde(default)]
- reasoning: Option<bool>,
- /// Serves every client-visible turn; also the count_tokens target.
- executor_target: String,
- /// Reviews the executor's first terminal turn. Judge-only, never a
- /// routing destination.
- advisor_target: String,
- #[serde(default)]
- reviewer_system_prompt: Option<String>,
- #[serde(default)]
- redo_feedback_prefix: Option<String>,
- #[serde(default)]
- gate_trigger: AdvisorTriggerConfig,
- #[serde(default)]
- gate_trigger_pattern: Option<String>,
- #[serde(default = "default_max_reviews")]
- max_reviews: u32,
- #[serde(default)]
- gate_stall_turns: u32,
- #[serde(default)]
- gate_min_tool_results: u32,
- #[serde(default = "default_advisor_max_tokens")]
- advisor_max_tokens: u64,
- #[serde(default)]
- advisor_temperature: Option<f64>,
- #[serde(default = "default_transcript_max_chars")]
- transcript_max_chars: usize,
- #[serde(default = "default_fail_open")]
- fail_open: bool,
- },
-}
-
-/// What fires an advisor route's review.
-#[derive(Debug, Default, Deserialize, PartialEq)]
-#[serde(rename_all = "snake_case")]
-enum AdvisorTriggerConfig {
- /// The executor's first turn without tool calls.
- #[default]
- NoToolCall,
- /// The first turn whose text matches `gate_trigger_pattern`.
- Pattern,
-}
-
-/// The judge a `stage_router` route falls through to, and how it routes.
-#[derive(Debug, Deserialize)]
-#[serde(deny_unknown_fields)]
-struct StageClassifierConfig {
- /// Target the judge is called through. Not a routing destination.
- target: String,
- base_threshold: f64,
- #[serde(default)]
- threshold_step: f64,
- #[serde(default)]
- classify_trigger: ClassifyTrigger,
- #[serde(default)]
- message_hash_fallback: bool,
- #[serde(default)]
- recent_turn_window: Option<usize>,
- #[serde(default)]
- prompt: Option<String>,
- #[serde(default)]
- response_format_type: ClassifierResponseFormat,
- #[serde(default = "default_classifier_max_output_tokens")]
- max_output_tokens: u64,
-}
-
-impl StageClassifierConfig {
- fn task_classifier_config(&self) -> TaskClassifierConfig {
- TaskClassifierConfig {
- base_threshold: self.base_threshold,
- threshold_step: self.threshold_step,
- classify_trigger: self.classify_trigger,
- message_hash_fallback: self.message_hash_fallback,
- recent_turn_window: self.recent_turn_window,
- contract: classifier_contract(self.prompt.as_deref())
- .with_response_format_type(self.response_format_type),
- max_output_tokens: self.max_output_tokens,
- }
- }
-}
-
-impl RouteConfig {
- fn id(&self) -> &ModelId {
- use RouteConfig::*;
- match self {
- Noop { id, .. }
- | Random { id, .. }
- | LlmClassifier { id, .. }
- | Passthrough { id, .. }
- | StageRouter { id, .. }
- | Advisor { id, .. } => id,
- }
- }
-
- // Completion targets in algorithm order; judge-only targets are excluded.
- fn routing_target_names(&self) -> Vec<&str> {
- match self {
- Self::Noop { .. } => Vec::new(),
- Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(),
- Self::Passthrough {
- target, subagents, ..
- } => {
- let mut names = vec![target.as_str()];
- if let Some(subagents) = subagents {
- names.extend(subagents.routing_target_names());
- }
- names
- }
- Self::LlmClassifier { config, .. } => {
- match config.mode.unwrap_or(if config.escalation.is_some() {
- ClassifierMode::Escalation
- } else {
- ClassifierMode::Capability
- }) {
- ClassifierMode::Capability => config
- .weak_target
- .iter()
- .chain(&config.strong_target)
- .map(String::as_str)
- .collect(),
- ClassifierMode::Escalation => config
- .strong_target
- .iter()
- .chain(&config.weak_target)
- .map(String::as_str)
- .collect(),
- ClassifierMode::Custom => config
- .targets
- .iter()
- .flatten()
- .map(String::as_str)
- .collect(),
- }
- }
- Self::StageRouter {
- capable_target,
- efficient_target,
- subagents,
- ..
- } => {
- let mut names = vec![capable_target.as_str(), efficient_target.as_str()];
- if let Some(subagents) = subagents {
- names.extend(subagents.routing_target_names());
- }
- names
- }
- // The advisor is judge-only: reviews go through its own client,
- // so it is not a completion (or count_tokens) destination.
- Self::Advisor {
- executor_target, ..
- } => vec![executor_target],
- }
- }
-
- /// Every target the algorithm may call, including judge-only targets.
- ///
- /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;
- /// a classifier also calls its judge, and that call needs a client too.
- fn callable_target_names(&self) -> Vec<&str> {
- let mut names = self.routing_target_names();
- match self {
- Self::LlmClassifier { config, .. } => names.push(&config.classifier_target),
- Self::Passthrough {
- subagents: Some(subagents),
- ..
- } => names.extend(subagents.classifier_target_name()),
- Self::StageRouter {
- classifier,
- subagents,
- ..
- } => {
- if let Some(classifier) = classifier {
- names.push(&classifier.target);
- }
- if let Some(subagents) = subagents {
- names.extend(subagents.classifier_target_name());
- }
- }
- Self::Advisor { advisor_target, .. } => names.push(advisor_target),
- _ => {}
- }
- names
- }
-
- fn capabilities(&self) -> ModelCapabilities {
- use RouteConfig::*;
- match self {
- Noop {
- context_window,
- tool_calling,
- reasoning,
- ..
- }
- | Random {
- context_window,
- tool_calling,
- reasoning,
- ..
- }
- | Passthrough {
- context_window,
- tool_calling,
- reasoning,
- ..
- }
- | LlmClassifier {
- context_window,
- tool_calling,
- reasoning,
- ..
- }
- | StageRouter {
- context_window,
- tool_calling,
- reasoning,
- ..
- }
- | Advisor {
- context_window,
- tool_calling,
- reasoning,
- ..
- } => ModelCapabilities {
- context_window: *context_window,
- tool_calling: *tool_calling,
- reasoning: *reasoning,
- },
- }
- }
-}
-
-impl LlmClassifierRouteConfig {
- fn classifier_mode(&self, route_name: &str) -> ServerResult<LlmClassifierModeConfig> {
- let Self {
- classifier_target,
- mode,
- strong_target,
- weak_target,
- base_threshold,
- threshold_step,
- classify_trigger,
- message_hash_fallback,
- recent_turn_window,
- prompt,
- response_format_type,
- max_output_tokens,
- escalation,
- targets,
- default_target,
- response_schema,
- policy,
- } = self;
-
- let selected_mode = match (mode, escalation.is_some()) {
- (Some(mode), _) => *mode,
- (None, true) => ClassifierMode::Escalation,
- (None, false) => ClassifierMode::Capability,
- };
-
- match selected_mode {
- ClassifierMode::Capability => {
- if escalation.is_some() {
- return Err(classifier_field_error(
- route_name,
- "escalation",
- "capability",
- ));
- }
- reject_custom_fields(
- route_name,
- "capability",
- targets,
- default_target,
- response_schema,
- policy,
- )?;
- Ok(LlmClassifierModeConfig::Capability(
- CapabilityClassifierRouteConfig {
- strong_target: required_classifier_field(
- route_name,
- "strong_target",
- strong_target,
- )?,
- weak_target: required_classifier_field(
- route_name,
- "weak_target",
- weak_target,
- )?,
- base_threshold: required_classifier_field(
- route_name,
- "base_threshold",
- base_threshold,
- )?,
- threshold_step: threshold_step.unwrap_or_default(),
- classify_trigger: *classify_trigger,
- message_hash_fallback: *message_hash_fallback,
- recent_turn_window: *recent_turn_window,
- prompt: prompt.clone(),
- response_format_type: *response_format_type,
- max_output_tokens: *max_output_tokens,
- },
- ))
- }
- ClassifierMode::Escalation => {
- reject_custom_fields(
- route_name,
- "escalation",
- targets,
- default_target,
- response_schema,
- policy,
- )?;
- if *classify_trigger != ClassifyTrigger::EveryRequest {
- return Err(ServerError::new(format!(
- "llm_classifier route {route_name} mode escalation cannot use classify_trigger"
- )));
- }
- if mode.is_some()
- && (base_threshold.is_some()
- || threshold_step.is_some()
- || *message_hash_fallback
- || recent_turn_window.is_some())
- {
- return Err(ServerError::new(format!(
- "llm_classifier route {route_name} mode escalation cannot use capability routing settings"
- )));
- }
- Ok(LlmClassifierModeConfig::Escalation(
- EscalationClassifierRouteConfig {
- strong_target: required_classifier_field(
- route_name,
- "strong_target",
- strong_target,
- )?,
- weak_target: required_classifier_field(
- route_name,
- "weak_target",
- weak_target,
- )?,
- prompt: prompt.clone(),
- response_format_type: *response_format_type,
- max_output_tokens: *max_output_tokens,
- judge: required_classifier_field(route_name, "escalation", escalation)?,
- },
- ))
- }
- ClassifierMode::Custom => {
- if strong_target.is_some()
- || weak_target.is_some()
- || base_threshold.is_some()
- || threshold_step.is_some()
- || escalation.is_some()
- || *response_format_type != ClassifierResponseFormat::JsonSchema
- {
- return Err(ServerError::new(format!(
- "llm_classifier route {route_name} mode custom cannot use capability or escalation fields"
- )));
- }
- Ok(LlmClassifierModeConfig::Custom(
- CustomClassifierRouteConfig {
- classifier_target: classifier_target.clone(),
- targets: required_classifier_field(route_name, "targets", targets)?,
- default_target: required_classifier_field(
- route_name,
- "default_target",
- default_target,
- )?,
- prompt: required_classifier_field(route_name, "prompt", prompt)?,
- response_schema: required_classifier_field(
- route_name,
- "response_schema",
- response_schema,
- )?,
- policy: required_classifier_field(route_name, "policy", policy)?,
- classify_trigger: *classify_trigger,
- message_hash_fallback: *message_hash_fallback,
- recent_turn_window: *recent_turn_window,
- max_output_tokens: *max_output_tokens,
- },
- ))
- }
- }
- }
-}
-
-fn reject_custom_fields(
- route_name: &str,
- mode: &str,
- targets: &Option<Vec<String>>,
- default_target: &Option<String>,
- response_schema: &Option<String>,
- policy: &Option<ClassifierPolicyConfig>,
-) -> ServerResult<()> {
- if targets.is_some()
- || default_target.is_some()
- || response_schema.is_some()
- || policy.is_some()
- {
- return Err(ServerError::new(format!(
- "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields"
- )));
- }
- Ok(())
-}
-
-fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> ServerError {
- ServerError::new(format!(
- "llm_classifier route {route_name} mode {mode} cannot use {field}"
- ))
-}
-
-fn required_classifier_field<T: Clone>(
- route_name: &str,
- field: &str,
- value: &Option<T>,
-) -> ServerResult<T> {
- value.clone().ok_or_else(|| {
- ServerError::new(format!(
- "llm_classifier route {route_name} requires {field}"
- ))
- })
-}
-
-fn build_backend(
- client_name: &str,
- config: &LlmClientConfig,
- extra_body: &BTreeMap<String, Value>,
-) -> ServerResult<Backend> {
- if config.max_retries > MAX_CONFIGURED_RETRIES {
- return Err(ServerError::new(format!(
- "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"
- )));
- }
- if config.forward_auth && config.api_key_env.is_some() {
- return Err(ServerError::new(format!(
- "llm client {client_name} cannot set both forward_auth and api_key_env"
- )));
- }
- let api_key = config
- .api_key_env
- .as_deref()
- .map(|variable| {
- if variable.trim().is_empty() {
- return Err(ServerError::new(format!(
- "llm client {client_name} api_key_env must not be empty"
- )));
- }
- let api_key = std::env::var(variable).map_err(|error| {
- ServerError::new(format!(
- "llm client {client_name} could not read api_key_env {variable}: {error}"
- ))
- })?;
- if api_key.trim().is_empty() {
- return Err(ServerError::new(format!(
- "llm client {client_name} api_key_env {variable} is empty"
- )));
- }
- Ok(api_key)
- })
- .transpose()?;
- let http = HttpBackendConfig {
- base_url: config.base_url.as_str().to_string(),
- api_key,
- forward_auth: config.forward_auth,
- extra_headers: config.extra_headers.clone(),
- extra_body: extra_body.clone(),
- max_retries: config.max_retries,
- };
- let backend = match config.format {
- ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
- ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),
- ClientFormat::AnthropicMessages => Backend::Anthropic(http),
- };
- Ok(backend)
-}
-
-const fn default_max_retries() -> u32 {
- DEFAULT_MAX_RETRIES
-}
-
-// Resolve nested policy once so every supported parent builds the same child route.
-fn build_subagent_router_config(
- route_name: &str,
- config: &SubagentRouteConfig,
- targets: &BTreeMap<String, ModelId>,
-) -> ServerResult<SubagentRouterConfig> {
- match config {
- SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target(
- resolve_target_model_id(route_name, target, targets)?,
- )),
- SubagentRouteConfig::LlmClassifier(config) => {
- let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)?
- else {
- return Err(ServerError::new(format!(
- "route {route_name}: subagents llm_classifier only supports mode custom"
- )));
- };
- let judge_target =
- resolve_target_model_id(route_name, &config.classifier_target, targets)?;
- let resolved_targets = config
- .targets
- .iter()
- .map(|name| {
- resolve_target_model_id(route_name, name, targets)
- .map(|target| (name.clone(), target))
- })
- .collect::<ServerResult<Vec<_>>>()?;
- let default_target = resolved_targets
- .iter()
- .find(|(name, _)| *name == config.default_target)
- .map(|(_, target)| target.clone())
- .ok_or_else(|| {
- ServerError::new(format!(
- "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets",
- config.default_target
- ))
- })?;
- let response_schema =
- serde_json::from_str(&config.response_schema).map_err(|error| {
- ServerError::new(format!(
- "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}"
- ))
- })?;
- let mut classifier_config = CustomClassifierConfig::new(
- config.prompt,
- response_schema,
- config.policy.into_libsy(),
- );
- classifier_config.recent_turn_window = config.recent_turn_window;
- classifier_config.max_output_tokens = config.max_output_tokens;
- let subagent_targets = resolved_targets
- .iter()
- .map(|(_, target)| target.clone())
- .collect();
- let classifier = Arc::new(
- LlmTaskClassifier::new(LlmClassifierConfig::Custom {
- judge_target,
- targets: resolved_targets,
- default_target: config.default_target,
- config: classifier_config,
- })
- .map_err(|error| {
- ServerError::new(format!(
- "route {route_name}: subagents llm_classifier: {error}"
- ))
- })?,
- );
- Ok(SubagentRouterConfig {
- targets: subagent_targets,
- classifier,
- default_target,
- classify_trigger: config.classify_trigger,
- message_hash_fallback: config.message_hash_fallback,
- })
- }
- }
-}
-
-fn attach_subagent_router(
- route_name: &str,
- parent: Arc<dyn Algorithm>,
- config: Option<&SubagentRouteConfig>,
- targets: &BTreeMap<String, ModelId>,
-) -> ServerResult<Arc<dyn Algorithm>> {
- let Some(config) = config else {
- return Ok(parent);
- };
- let config = build_subagent_router_config(route_name, config, targets)?;
- let algorithm = SubagentRouter::new(parent, config).map_err(|error| {
- ServerError::new(format!("route {route_name}: subagent routing: {error}"))
- })?;
- Ok(Arc::new(algorithm))
-}
-
-fn build_algorithm(
- route_name: &str,
- config: &RouteConfig,
- targets: &BTreeMap<String, ModelId>,
-) -> ServerResult<Arc<dyn Algorithm>> {
- match config {
- RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
- RouteConfig::Random {
- targets: names,
- weights,
- seed,
- ..
- } => {
- let target_set =
- resolve_targets(route_name, names.iter().map(String::as_str), targets)?;
- let algorithm = Random::new(target_set, weights.clone(), *seed)
- .map_err(|error| ServerError::new(format!("random route {route_name}: {error}")))?;
- Ok(Arc::new(algorithm))
- }
- RouteConfig::Passthrough {
- target, subagents, ..
- } => {
- let parent_target = resolve_target_model_id(route_name, target, targets)?;
- let algorithm = Passthrough::new(parent_target);
- let parent: Arc<dyn Algorithm> = Arc::new(algorithm);
- attach_subagent_router(route_name, parent, subagents.as_ref(), targets)
- }
- RouteConfig::LlmClassifier {
- config: classifier_config,
- ..
- } => {
- let classifier =
- resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?;
- let mode = classifier_config.classifier_mode(route_name)?;
- let algorithm = match mode {
- LlmClassifierModeConfig::Capability(config) => {
- let strong =
- resolve_target_model_id(route_name, &config.strong_target, targets)?;
- let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
- let classifier_config = TaskClassifierConfig {
- base_threshold: config.base_threshold,
- threshold_step: config.threshold_step,
- classify_trigger: config.classify_trigger,
- message_hash_fallback: config.message_hash_fallback,
- recent_turn_window: config.recent_turn_window,
- contract: classifier_contract(config.prompt.as_deref())
- .with_response_format_type(config.response_format_type),
- max_output_tokens: config.max_output_tokens,
- };
- LlmTaskClassifier::new(LlmClassifierConfig::Capability {
- judge_target: classifier,
- efficient_target: weak,
- capable_target: strong,
- config: classifier_config,
- })
- }
- LlmClassifierModeConfig::Escalation(config) => {
- let strong =
- resolve_target_model_id(route_name, &config.strong_target, targets)?;
- let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;
- LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
- judge_target: classifier,
- efficient_target: weak,
- capable_target: strong,
- contract: classifier_contract(config.prompt.as_deref())
- .with_response_format_type(config.response_format_type),
- config: config.judge,
- max_output_tokens: config.max_output_tokens,
- })
- }
- LlmClassifierModeConfig::Custom(config) => {
- let resolved_targets = config
- .targets
- .iter()
- .map(|name| {
- resolve_target_model_id(route_name, name, targets)
- .map(|target| (name.clone(), target))
- })
- .collect::<ServerResult<Vec<_>>>()?;
- let response_schema = serde_json::from_str(&config.response_schema).map_err(
- |error| {
- ServerError::new(format!(
- "llm_classifier route {route_name}: response_schema is invalid JSON: {error}"
- ))
- },
- )?;
- let mut classifier_config = CustomClassifierConfig::new(
- config.prompt,
- response_schema,
- config.policy.into_libsy(),
- );
- classifier_config.classify_trigger = config.classify_trigger;
- classifier_config.message_hash_fallback = config.message_hash_fallback;
- classifier_config.recent_turn_window = config.recent_turn_window;
- classifier_config.max_output_tokens = config.max_output_tokens;
- LlmTaskClassifier::new(LlmClassifierConfig::Custom {
- judge_target: classifier,
- targets: resolved_targets,
- default_target: config.default_target,
- config: classifier_config,
- })
- }
- }
- .map_err(|error| {
- ServerError::new(format!("llm_classifier route {route_name}: {error}"))
- })?;
- Ok(Arc::new(algorithm))
- }
- RouteConfig::StageRouter {
- capable_target,
- efficient_target,
- picker,
- confidence_threshold,
- recent_turn_window,
- handoff_notes,
- capable_system_prompt,
- efficient_system_prompt,
- classifier,
- subagents,
- ..
- } => {
- if matches!(picker, PickerMode::CapableFirst) {
- tracing::warn!(
- "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."
- );
- }
- let capable = resolve_target_model_id(route_name, capable_target, targets)?;
- let efficient = resolve_target_model_id(route_name, efficient_target, targets)?;
- let mut config = StageRouterConfig::new(*picker, *confidence_threshold);
- config.recent_window = *recent_turn_window;
- config.handoff_notes = handoff_notes.clone();
- config.tier_prompts = tier_prompts(
- &capable,
- capable_system_prompt.as_deref(),
- &efficient,
- efficient_system_prompt.as_deref(),
- );
- // The judge is called through its own target, so it is not a routing
- // destination and stays out of the tier pair.
- config.llm_fallback = classifier
- .as_ref()
- .map(|classifier| {
- resolve_target_model_id(route_name, &classifier.target, targets).map(
- |judge_target| LlmFallback {
- judge_target,
- config: classifier.task_classifier_config(),
- },
- )
- })
- .transpose()?;
- let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| {
- ServerError::new(format!("stage_router route {route_name}: {error}"))
- })?;
- let parent: Arc<dyn Algorithm> = Arc::new(algorithm);
- attach_subagent_router(route_name, parent, subagents.as_ref(), targets)
- }
- RouteConfig::Advisor {
- executor_target,
- advisor_target,
- reviewer_system_prompt,
- redo_feedback_prefix,
- gate_trigger,
- gate_trigger_pattern,
- max_reviews,
- gate_stall_turns,
- gate_min_tool_results,
- advisor_max_tokens,
- advisor_temperature,
- transcript_max_chars,
- fail_open,
- ..
- } => {
- let executor = resolve_target_model_id(route_name, executor_target, targets)?;
- let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;
- // A pattern set under the default trigger would be silently
- // ignored; reject the misconfiguration instead.
- if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {
- return Err(ServerError::new(format!(
- "advisor route {route_name}: gate_trigger_pattern requires \
- gate_trigger = \"pattern\""
- )));
- }
- let mut config = AdvisorGateConfig::default();
- if let Some(prompt) = reviewer_system_prompt {
- config.reviewer_system_prompt = prompt.clone();
- }
- if let Some(prefix) = redo_feedback_prefix {
- config.redo_feedback_prefix = prefix.clone();
- }
- config.gate_trigger = match gate_trigger {
- AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,
- AdvisorTriggerConfig::Pattern => {
- GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())
- }
- };
- config.max_reviews = *max_reviews;
- config.gate_stall_turns = *gate_stall_turns;
- config.gate_min_tool_results = *gate_min_tool_results;
- config.advisor_max_tokens = *advisor_max_tokens;
- config.advisor_temperature = *advisor_temperature;
- config.transcript_max_chars = *transcript_max_chars;
- config.fail_open = *fail_open;
- let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {
- ServerError::new(format!("advisor route {route_name}: {error}"))
- })?;
- Ok(Arc::new(algorithm))
- }
- }
-}
-
-const fn default_max_reviews() -> u32 {
- 1
-}
-
-const fn default_advisor_max_tokens() -> u64 {
- 2048
-}
-
-const fn default_transcript_max_chars() -> usize {
- 200_000
-}
-
-const fn default_fail_open() -> bool {
- true
-}
-
-fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {
- prompt.map_or_else(ClassifierContractConfig::default, |prompt| {
- ClassifierContractConfig::default().with_prompt(prompt)
- })
-}
-
-fn default_classifier_max_output_tokens() -> u64 {
- TaskClassifierConfig::default().max_output_tokens
-}
-
-/// Keys each configured system prompt by the target it belongs to.
-fn tier_prompts(
- capable: &str,
- capable_prompt: Option<&str>,
- efficient: &str,
- efficient_prompt: Option<&str>,
-) -> TargetPrompts {
- let mut prompts = TargetPrompts::default();
- if let Some(prompt) = capable_prompt {
- prompts = prompts.with(capable, prompt);
- }
- if let Some(prompt) = efficient_prompt {
- prompts = prompts.with(efficient, prompt);
- }
- prompts
-}
-
-fn resolve_targets<'a>(
- route_name: &str,
- names: impl IntoIterator<Item = &'a str>,
- targets: &BTreeMap<String, ModelId>,
-) -> ServerResult<Vec<ModelId>> {
- names
- .into_iter()
- .map(|name| resolve_target_model_id(route_name, name, targets))
- .collect()
-}
-
-fn resolve_target_model_id(
- route_name: &str,
- name: &str,
- targets: &BTreeMap<String, ModelId>,
-) -> ServerResult<ModelId> {
- targets.get(name).cloned().ok_or_else(|| {
- ServerError::new(format!(
- "route {route_name} references unknown target {name}"
- ))
- })
-}
-
-fn validate_value(label: &str, value: &str) -> ServerResult<()> {
- if value.trim().is_empty() || value.trim() != value {
- return Err(ServerError::new(format!(
- "{label} must be non-empty and have no surrounding whitespace"
- )));
- }
- Ok(())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- const VALID_CONFIG: &str = r#"
-schema_version = 1
-
-[llm_clients.primary]
-format = "openai_chat"
-base_url = "https://example.test/v1"
-
-[llm_clients.responses]
-format = "openai_responses"
-base_url = "https://example.test/v1"
-
-[llm_clients.anthropic]
-format = "anthropic_messages"
-base_url = "https://example.test"
-
-[targets.classifier]
-id = "classifier/model"
-llm_client = "primary"
-
-[targets.strong]
-id = "strong/model"
-llm_client = "responses"
-
-[targets.weak]
-id = "weak/model"
-llm_client = "anthropic"
-
-[routes.noop]
-id = "switchyard/noop"
-type = "noop"
-
-[routes.random]
-id = "switchyard/random"
-type = "random"
-targets = ["strong", "weak"]
-
-[routes.classifier]
-id = "switchyard/classifier"
-type = "llm_classifier"
-classifier_target = "classifier"
-strong_target = "strong"
-weak_target = "weak"
-base_threshold = 0.5
-
-[routes.passthrough]
-id = "switchyard/passthrough"
-type = "passthrough"
-target = "weak"
-"#;
-
- fn error_message(toml: &str) -> String {
- match server_state_from_toml(toml) {
- Ok(_) => "configuration unexpectedly succeeded".to_string(),
- Err(error) => error.to_string(),
- }
- }
-
- fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {
- let mut configured = config.to_string();
- configured.push_str(&format!("\n[routes.{route}.subagents]\n"));
- configured.push_str(
- r#"type = "llm_classifier"
-mode = "custom"
-classifier_target = "classifier"
-targets = ["strong", "weak"]
-default_target = "weak"
-prompt = "Select a target for this delegated task."
-response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}'
-policy = { type = "target_selector", selector = "/target" }
-classify_trigger = "new_session""#,
- );
- configured.push_str(extra);
- configured
- }
-
- fn with_subagent_passthrough(config: &str, route: &str) -> String {
- format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")
- }
-
- fn stage_config() -> String {
- format!(
- r#"{VALID_CONFIG}
-[targets.stage_judge]
-id = "stage-judge/model"
-llm_client = "primary"
-
-[routes.stage]
-id = "switchyard/stage"
-type = "stage_router"
-capable_target = "strong"
-efficient_target = "weak"
-picker = "efficient_first"
-confidence_threshold = 1.0
-
-[routes.stage.classifier]
-target = "stage_judge"
-base_threshold = 0.5
-"#
- )
- }
-
- #[test]
- fn builds_all_supported_algorithm_types() -> ServerResult<()> {
- let state = server_state_from_toml(VALID_CONFIG)?;
- // The model id array is sorted alphabetically
- assert_eq!(
- state.models().collect::<Vec<_>>(),
- [
- "switchyard/classifier",
- "switchyard/noop",
- "switchyard/passthrough",
- "switchyard/random",
- ]
- );
- Ok(())
- }
-
- #[test]
- fn passthrough_and_stage_accept_subagent_routing() -> ServerResult<()> {
- let stage = stage_config();
- let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", "");
- let parsed: ServerConfig = toml::from_str(&stage_with_classifier)
- .map_err(|error| ServerError::new(format!("failed to parse stage config: {error}")))?;
- let Some(stage_route) = parsed.routes.get("stage") else {
- return Err(ServerError::new("stage route is missing"));
- };
- let callable_targets = stage_route.callable_target_names();
- for expected in ["strong", "weak", "stage_judge", "classifier"] {
- assert!(callable_targets.contains(&expected));
- }
-
- for configured in [
- with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""),
- with_subagent_passthrough(VALID_CONFIG, "passthrough"),
- stage_with_classifier,
- with_subagent_passthrough(&stage, "stage"),
- ] {
- server_state_from_toml(&configured)?;
- }
- Ok(())
- }
-
- #[test]
- fn rejects_invalid_unreferenced_llm_client() {
- let invalid = format!(
- "{VALID_CONFIG}\n\
- [llm_clients.unused]\n\
- format = \"openai_chat\"\n\
- base_url = \"not a url\"\n"
- );
- let message = error_message(&invalid);
- assert!(
- message.contains("base_url must be an absolute HTTP(S) URL"),
- "unexpected error: {message}"
- );
- }
-
- #[test]
- fn an_escalation_table_switches_the_classifier_route_to_escalation() -> ServerResult<()> {
- // Present: the classifier target judges the weak tier's reply each turn instead of
- // picking a tier ahead of it. The route builds either way, so the assertion is that
- // the knob parses and its settings reach the algorithm's validation.
- let escalating = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nescalation = { confirmations = 2 }",
- );
- server_state_from_toml(&escalating)?;
-
- // A setting that would starve the judge is rejected here rather than on the first
- // request, the same as any other unusable route configuration.
- let starved = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nescalation = { confirmations = 0 }",
- );
- assert!(error_message(&starved).contains("confirmations must be at least 1"));
- Ok(())
- }
-
- #[test]
- fn classifier_judge_completion_caps_are_configurable() -> ServerResult<()> {
- let capability = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nmax_output_tokens = 512",
- );
- server_state_from_toml(&capability)?;
-
- let escalation = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",
- );
- server_state_from_toml(&escalation)?;
- Ok(())
- }
-
- #[test]
- fn classifier_prompts_are_configurable_in_both_modes() -> ServerResult<()> {
- let capability = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nprompt = \"custom capability rubric\"",
- );
- server_state_from_toml(&capability)?;
-
- let escalation = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }",
- );
- server_state_from_toml(&escalation)?;
-
- let empty = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nprompt = \" \"",
- );
- assert!(error_message(&empty).contains("classifier prompt must not be empty"));
-
- let schema_placeholder = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",
- );
- assert!(
- error_message(&schema_placeholder)
- .contains("Switchyard supplies the schema automatically")
- );
- Ok(())
- }
-
- #[test]
- fn mode_custom_rejects_capability_fields() {
- let mixed = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "mode = \"custom\"\nbase_threshold = 0.5",
- );
-
- assert!(
- error_message(&mixed)
- .contains("mode custom cannot use capability or escalation fields")
- );
- }
-
- #[test]
- fn rejects_unknown_fields_and_algorithm_types() {
- let unknown_field =
- VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");
- assert!(error_message(&unknown_field).contains("unknown field"));
-
- let nested_completion_cap = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",
- );
- assert!(error_message(&nested_completion_cap).contains("unknown field"));
-
- let unknown_classifier_field = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nclassifier_magic = true",
- );
- assert!(error_message(&unknown_classifier_field).contains("unknown field"));
-
- let target_capability = VALID_CONFIG.replace(
- "llm_client = \"responses\"",
- "llm_client = \"responses\"\ncontext_window = 1000000",
- );
- assert!(error_message(&target_capability).contains("unknown field `context_window`"));
-
- let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
- assert!(error_message(&unknown_algorithm).contains("unknown variant"));
- }
-
- #[test]
- fn rejects_unknown_stage_classifier_fields() {
- // Nested classifier typos must fail instead of silently using a default.
- let config = format!(
- r#"{VALID_CONFIG}
-
-[routes.stage]
-id = "switchyard/stage"
-type = "stage_router"
-capable_target = "strong"
-efficient_target = "weak"
-picker = "efficient_first"
-confidence_threshold = 1.0
-
-[routes.stage.classifier]
-target = "classifier"
-base_threshold = 0.5
-classifier_magic = true
-"#
- );
-
- let error = error_message(&config);
- assert!(
- error.contains("unknown field `classifier_magic`"),
- "{error}"
- );
- }
-
- #[test]
- fn rejects_invalid_references_and_parameters() {
- let cases = [
- (
- VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),
- "unknown llm client missing",
- ),
- (
- VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"missing\"]",
- ),
- "unknown target missing",
- ),
- (
- VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"strong\"]",
- ),
- "random targets must be unique",
- ),
- (
- VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"weak\"]\nweights = [1]",
- ),
- "expected 2 weights, got 1",
- ),
- (
- VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",
- ),
- "at least one weight must be positive",
- ),
- (
- VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),
- "base_threshold must be between 0 and 1",
- ),
- (
- VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""),
- "route references unknown target",
- ),
- (
- VALID_CONFIG.replace(
- "classifier_target = \"classifier\"",
- "classifier_target = \"\"",
- ),
- "route references unknown target",
- ),
- (
- VALID_CONFIG.replace(
- "classifier_target = \"classifier\"",
- "classifier_target = \" \"",
- ),
- "route references unknown target",
- ),
- (
- VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nthreshold_step = -0.1",
- ),
- "threshold_step must be finite and greater than or equal to 0",
- ),
- (
- VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.8\nthreshold_step = 0.11",
- ),
- "base_threshold + 2 * threshold_step must be at most 1",
- ),
- (
- VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",
- ),
- "max_output_tokens must be at least 1",
- ),
- (
- VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.5\nmessage_hash_fallback = true",
- ),
- "message_hash_fallback requires classify_trigger = new_session",
- ),
- (
- with_subagent_llm_classifier(
- VALID_CONFIG,
- "passthrough",
- "\nmessage_hash_fallback = true",
- ),
- "cannot use message_hash_fallback",
- ),
- (
- with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "")
- .replace("mode = \"custom\"", "mode = \"capability\""),
- "mode capability cannot use custom classifier fields",
- ),
- (
- VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"",
- ),
- "mode escalation cannot use classify_trigger",
- ),
- (
- VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),
- "unsupported schema_version 2",
- ),
- (
- VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),
- "target name must be non-empty and have no surrounding whitespace",
- ),
- (
- VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"weak\"]\ncontext_window = 0",
- ),
- "route random context_window must be greater than zero",
- ),
- ];
-
- for (toml, expected) in cases {
- let error = error_message(&toml);
- assert!(
- error.contains(expected),
- "expected error containing {expected}, got {error}"
- );
- }
- }
-
- #[test]
- fn accepts_duplicate_target_model_ids_on_one_client() -> ServerResult<()> {
- // Two targets share one model id on one client. The client keeps one and drops the
- // other, so the build warns and still succeeds, and both routes resolve. Serving one
- // model under two route names this way is allowed; pointing both routes at one target
- // is the tidier form.
- const SAME_MODEL_TWO_ROUTES: &str = r#"
-schema_version = 1
-
-[llm_clients.primary]
-format = "openai_chat"
-base_url = "https://example.test/v1"
-
-[targets.fast]
-id = "gpt-4o"
-llm_client = "primary"
-
-[targets.smart]
-id = "gpt-4o"
-llm_client = "primary"
-
-[routes.fast]
-id = "switchyard/fast"
-type = "passthrough"
-target = "fast"
-
-[routes.smart]
-id = "switchyard/smart"
-type = "passthrough"
-target = "smart"
-"#;
- let state = server_state_from_toml(SAME_MODEL_TWO_ROUTES)?;
- assert_eq!(
- state.models().collect::<Vec<_>>(),
- ["switchyard/fast", "switchyard/smart"]
- );
- Ok(())
- }
-
- #[test]
- fn accepts_same_model_id_on_different_llm_clients() -> ServerResult<()> {
- // The same model id served by two llm clients never collides (each client keys its own
- // models), so cross-provider A/B builds with no warning; only a repeat within one client
- // warns.
- const CROSS_PROVIDER: &str = r#"
-schema_version = 1
-
-[llm_clients.openai]
-format = "openai_chat"
-base_url = "https://example.test/v1"
-
-[llm_clients.azure]
-format = "openai_chat"
-base_url = "https://azure.test/v1"
-
-[targets.openai]
-id = "gpt-4o"
-llm_client = "openai"
-
-[targets.azure]
-id = "gpt-4o"
-llm_client = "azure"
-
-[routes.openai]
-id = "switchyard/openai-gpt4o"
-type = "passthrough"
-target = "openai"
-
-[routes.azure]
-id = "switchyard/azure-gpt4o"
-type = "passthrough"
-target = "azure"
-"#;
- server_state_from_toml(CROSS_PROVIDER)?;
- Ok(())
- }
-
- #[test]
- fn accepts_relative_weights_and_seed() -> ServerResult<()> {
- let weighted = VALID_CONFIG.replace(
- "targets = [\"strong\", \"weak\"]",
- "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",
- );
- server_state_from_toml(&weighted)?;
- Ok(())
- }
-
- #[test]
- fn accepts_new_session_trigger_with_message_hash_fallback() -> ServerResult<()> {
- let configured = VALID_CONFIG.replace(
- "base_threshold = 0.5",
- "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true",
- );
- server_state_from_toml(&configured)?;
- Ok(())
- }
-
- #[test]
- fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> {
- let configured = VALID_CONFIG.replacen(
- "llm_client = \"primary\"",
- "llm_client = \"primary\"\n\
- extra_body = { service_tier = \"priority\", \
- chat_template_kwargs = { enable_thinking = false } }",
- 1,
- );
- let config: ServerConfig = toml::from_str(&configured)
- .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?;
- let Some(target) = config.targets.get("classifier") else {
- return Err(ServerError::new("classifier target is missing"));
- };
- let Some(client) = config.llm_clients.get("primary") else {
- return Err(ServerError::new("primary llm client is missing"));
- };
- let backend = build_backend("primary", client, &target.extra_body)?;
-
- assert_eq!(
- backend.extra_body().get("service_tier"),
- Some(&json!("priority"))
- );
- assert_eq!(
- backend
- .extra_body()
- .get("chat_template_kwargs")
- .and_then(|value| value.get("enable_thinking")),
- Some(&json!(false))
- );
- Ok(())
- }
-
- #[test]
- fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> {
- let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| {
- ServerError::new(format!("failed to parse default config: {error}"))
- })?;
- let Some(primary) = default.llm_clients.get("primary") else {
- return Err(ServerError::new("primary llm client is missing"));
- };
- assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);
-
- let explicit = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\nmax_retries = 0",
- 1,
- );
- let config: ServerConfig = toml::from_str(&explicit).map_err(|error| {
- ServerError::new(format!("failed to parse explicit retry config: {error}"))
- })?;
- let Some(primary) = config.llm_clients.get("primary") else {
- return Err(ServerError::new("primary llm client is missing"));
- };
- assert_eq!(primary.max_retries, 0);
-
- let maximum = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- &format!(
- "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"
- ),
- 1,
- );
- let config: ServerConfig = toml::from_str(&maximum).map_err(|error| {
- ServerError::new(format!("failed to parse maximum retry config: {error}"))
- })?;
- let Some(primary) = config.llm_clients.get("primary") else {
- return Err(ServerError::new("primary llm client is missing"));
- };
- assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);
- Ok(())
- }
-
- #[test]
- fn rejects_headers_that_switchyard_sets() {
- let cases = [
- (
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\n\
- extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }",
- "AUTHORIZATION",
- ),
- (
- "base_url = \"https://example.test\"",
- "base_url = \"https://example.test\"\n\
- extra_headers = { \"X-Api-Key\" = \"custom-key\" }",
- "X-Api-Key",
- ),
- (
- "base_url = \"https://example.test\"",
- "base_url = \"https://example.test\"\n\
- extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }",
- "ANTHROPIC-VERSION",
- ),
- ];
-
- for (original, replacement, header) in cases {
- let configured = VALID_CONFIG.replacen(original, replacement, 1);
- let error = error_message(&configured);
- assert!(
- error.contains(&format!("extra_headers cannot set {header:?}")),
- "expected {header} to be rejected, got: {error}"
- );
- }
- }
-
- #[test]
- fn accepts_additional_headers() -> ServerResult<()> {
- let configured = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\n\
- extra_headers = { X-Inference-Priority = \"batch\" }",
- 1,
- );
-
- server_state_from_toml(&configured)?;
- Ok(())
- }
-
- #[test]
- fn retry_budget_rejects_negative_values() {
- let invalid = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\nmax_retries = -1",
- 1,
- );
- assert!(error_message(&invalid).contains("max_retries"));
- }
-
- #[test]
- fn retry_budget_rejects_excessive_values() {
- let invalid = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\nmax_retries = 11",
- 1,
- );
- assert!(
- error_message(&invalid).contains("llm client primary max_retries must be at most 10")
- );
- }
-
- #[test]
- fn api_key_environment_reference_is_validated() {
- let missing = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",
- 1,
- );
- assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));
-
- const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";
- unsafe {
- // "unsafe" is for concurrent reads and writes, very rare
- std::env::set_var(EMPTY_KEY_ENV, "");
- }
- let empty = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),
- 1,
- );
- let message = error_message(&empty);
- unsafe {
- std::env::remove_var(EMPTY_KEY_ENV);
- }
- assert!(message.contains("is empty"));
- }
-
- #[test]
- fn forward_auth_rejects_conflicting_credentials() {
- let competing_auth = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- "base_url = \"https://example.test/v1\"\n\
- forward_auth = true\n\
- api_key_env = \"UNUSED_TEST_KEY\"",
- 1,
- );
- assert!(
- error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env")
- );
-
- let static_auth = VALID_CONFIG.replacen(
- "base_url = \"https://example.test\"",
- "base_url = \"https://example.test\"\n\
- forward_auth = true\n\
- extra_headers = { Authorization = \"static-value\" }",
- 1,
- );
- assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\""));
-
- let static_beta = static_auth.replace("Authorization", "anthropic-beta");
- assert!(
- error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"")
- );
-
- for header in ["chatgpt-account-id", "x-openai-fedramp"] {
- let static_context = VALID_CONFIG.replacen(
- "base_url = \"https://example.test/v1\"",
- &format!(
- "base_url = \"https://example.test/v1\"\n\
- forward_auth = true\n\
- extra_headers = {{ \"{header}\" = \"static-value\" }}"
- ),
- 1,
- );
- assert!(
- error_message(&static_context)
- .contains(&format!("extra_headers cannot set \"{header}\""))
- );
- }
- }
-
- const ADVISOR_CONFIG: &str = r#"
-schema_version = 1
-
-[llm_clients.anthropic]
-format = "anthropic_messages"
-base_url = "https://example.test"
-
-[targets.executor]
-id = "executor/model"
-llm_client = "anthropic"
-
-[targets.advisor]
-id = "advisor/model"
-llm_client = "anthropic"
-
-[routes.gated]
-id = "switchyard/advisor"
-type = "advisor"
-executor_target = "executor"
-advisor_target = "advisor"
-"#;
-
- #[test]
- fn advisor_route_parses_with_defaults_and_builds() -> ServerResult<()> {
- let state = server_state_from_toml(ADVISOR_CONFIG)?;
- assert_eq!(state.models().collect::<Vec<_>>(), ["switchyard/advisor"]);
- Ok(())
- }
-
- #[test]
- fn advisor_route_accepts_every_gate_knob() -> ServerResult<()> {
- let tuned = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- concat!(
- "advisor_target = \"advisor\"\n",
- "reviewer_system_prompt = \"review it\"\n",
- "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",
- "gate_trigger = \"pattern\"\n",
- "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",
- "max_reviews = 2\n",
- "gate_stall_turns = 40\n",
- "gate_min_tool_results = 1\n",
- "advisor_max_tokens = 1024\n",
- "advisor_temperature = 0.0\n",
- "transcript_max_chars = 100000\n",
- "fail_open = false\n",
- "context_window = 200000\n",
- "tool_calling = true\n",
- "reasoning = true",
- ),
- );
- server_state_from_toml(&tuned)?;
- Ok(())
- }
-
- #[test]
- fn advisor_route_rejects_unknown_keys() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"advisor\"\nbogus_field = 1",
- );
- assert!(error_message(&invalid).contains("bogus_field"));
- }
-
- #[test]
- fn advisor_route_requires_both_targets() {
- let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");
- assert!(error_message(&missing).contains("advisor_target"));
- }
-
- #[test]
- fn advisor_route_rejects_unknown_target() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"missing\"",
- );
- assert!(error_message(&invalid).contains("missing"));
- }
-
- #[test]
- fn advisor_route_rejects_invalid_pattern() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",
- );
- assert!(error_message(&invalid).contains("not a valid regex"));
- }
-
- #[test]
- fn advisor_route_rejects_pattern_without_pattern_trigger() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",
- );
- assert!(
- error_message(&invalid)
- .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")
- );
- }
-
- #[test]
- fn advisor_route_pattern_trigger_requires_pattern() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",
- );
- assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));
- }
-
- #[test]
- fn advisor_route_rejects_zero_max_reviews() {
- let invalid = ADVISOR_CONFIG.replace(
- "advisor_target = \"advisor\"",
- "advisor_target = \"advisor\"\nmax_reviews = 0",
- );
- assert!(error_message(&invalid).contains("max_reviews must be at least 1"));
- }
+ ServerState::from_runner(Runner::load(path).map_err(ServerError::from)?)
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 80f75b7fe..914fb6c8a 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -32,19 +32,21 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};
+use libsy::{Algorithm, LibsyError, RoutingOutcome};
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
-use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};
+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};
+use switchyard_runner::{
+ CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,
+};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
use tracing::{Instrument, Level};
use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};
-use crate::config::ServerConfig;
use crate::response::into_http_response;
use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};
@@ -86,106 +88,51 @@ impl Display for ServerError {
impl Error for ServerError {}
+impl From<RunnerError> for ServerError {
+ fn from(error: RunnerError) -> Self {
+ let mut message = error.to_string();
+ let mut source = error.source();
+ while let Some(error) = source {
+ message.push_str(": ");
+ message.push_str(&error.to_string());
+ source = error.source();
+ }
+ Self::new(message)
+ }
+}
+
/// Result returned by server setup and lifecycle operations.
pub type ServerResult<T> = std::result::Result<T, ServerError>;
-/// Capabilities that one route advertises on `GET /v1/models`.
-///
-/// An unset capability is undeclared: it serializes as `null` in the OpenAI
-/// `data` entry, and the Codex entry falls back to a safe default for it.
-#[derive(Clone, Copy, Default)]
-struct ModelCapabilities {
- context_window: Option<u32>,
- tool_calling: Option<bool>,
- // Whether the routed model takes reasoning controls. The server cannot probe
- // this, so a route opts in via config; undeclared routes advertise as
- // non-reasoning to Codex (fail closed).
- reasoning: Option<bool>,
-}
-
-/// A registered algorithm route and its server-owned endpoint metadata.
-struct RouteEntry {
- algorithm: Arc<dyn Algorithm>,
- /// Resolves each offloaded call to the client configured for the target the algorithm
- /// selected. A route is a synthetic model with no upstream of its own, so this is a
- /// per-target lookup, never one client serving the whole route.
- target_clients: ClientRouter,
- caller_auth: Option<CallerAuthKind>,
- capabilities: ModelCapabilities,
- count_tokens_target: Option<CountTokensTarget>,
- config_name: Option<String>,
-}
-
/// Decision result using the same selected/fallback order as libsy.
#[derive(Serialize)]
-struct DecisionResponse<'a> {
- selected: DecisionTargetResponse<'a>,
- fallbacks: Vec<DecisionTargetResponse<'a>>,
+struct DecisionResponse {
+ selected: DecisionTargetResponse,
+ fallbacks: Vec<DecisionTargetResponse>,
#[serde(skip_serializing_if = "Option::is_none")]
response: Option<Value>,
}
-/// Borrowed response view over one target's existing configuration.
+/// Response view over one target's existing configuration.
#[derive(Serialize)]
-struct DecisionTargetResponse<'a> {
- target: &'a str,
- model: &'a ModelId,
- llm_client: DecisionLlmClientResponse<'a>,
- extra_body: &'a BTreeMap<String, Value>,
+struct DecisionTargetResponse {
+ target: String,
+ model: ModelId,
+ llm_client: DecisionLlmClientResponse,
+ extra_body: BTreeMap<String, Value>,
}
/// Non-secret client settings needed to call a selected model.
#[derive(Serialize)]
-struct DecisionLlmClientResponse<'a> {
+struct DecisionLlmClientResponse {
format: WireFormat,
- base_url: &'a str,
-}
-
-/// Caller credential family required by forwarded-auth backends.
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub(crate) enum CallerAuthKind {
- Anthropic,
- OpenAi,
-}
-
-impl CallerAuthKind {
- const fn as_str(self) -> &'static str {
- match self {
- Self::Anthropic => "anthropic",
- Self::OpenAi => "openai",
- }
- }
-
- const fn accepts(self, wire_format: WireFormat) -> bool {
- matches!(
- (self, wire_format),
- (Self::Anthropic, WireFormat::AnthropicMessages)
- | (
- Self::OpenAi,
- WireFormat::OpenAiChat | WireFormat::OpenAiResponses
- )
- )
- }
-}
-
-/// Exact upstream model used by the server's Anthropic token-count endpoint.
-#[derive(Clone)]
-struct CountTokensTarget {
- model: ModelId,
- client: Arc<TranslatingLlmClient>,
-}
-
-impl CountTokensTarget {
- async fn count_tokens(&self, request: Request) -> Result<Value, LlmClientError> {
- self.client.count_tokens(&self.model, request).await
- }
+ base_url: String,
}
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- routes: Arc<BTreeMap<ModelId, RouteEntry>>,
- config: Option<Arc<ServerConfig>>,
+ runner: Arc<Runner>,
metrics: prometheus::Registry,
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
@@ -227,75 +174,37 @@ impl SharedRoutingLog {
}
impl ServerState {
- /// Creates server state from route model IDs, their libsy algorithms, and the
- /// per-target client routing each route's calls are resolved through.
- pub fn new(
- routes: impl IntoIterator<Item = (ModelId, Arc<dyn Algorithm>, ClientRouter)>,
- ) -> ServerResult<Self> {
- Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| {
- (
- model,
- algorithm,
- clients,
- None,
- ModelCapabilities::default(),
- None,
- None,
- )
- }))
+ /// Creates server state from route model IDs, algorithms, and per-target clients.
+ pub fn new(routes: Vec<(ModelId, Arc<dyn Algorithm>, ClientRouter)>) -> ServerResult<Self> {
+ let routes = routes
+ .into_iter()
+ .map(|(model, algorithm, clients)| {
+ (
+ model,
+ Route::new(
+ algorithm,
+ clients,
+ None,
+ ModelCapabilities::default(),
+ None,
+ Vec::new(),
+ ),
+ )
+ })
+ .collect();
+ let runner = Runner::new(routes);
+ Self::from_runner(runner)
}
- fn new_with_capabilities(
- routes: impl IntoIterator<
- Item = (
- ModelId,
- Arc<dyn Algorithm>,
- ClientRouter,
- Option<CallerAuthKind>,
- ModelCapabilities,
- Option<CountTokensTarget>,
- Option<String>,
- ),
- >,
- ) -> ServerResult<Self> {
- let mut entries = BTreeMap::new();
- for (
- model,
- algorithm,
- target_clients,
- caller_auth,
- capabilities,
- count_tokens_target,
- config_name,
- ) in routes
- {
- let model = ModelId::from(model.trim());
- if model.is_empty() {
- return Err(ServerError::new("route model must not be empty"));
- }
- let entry = RouteEntry {
- algorithm,
- target_clients,
- caller_auth,
- capabilities,
- count_tokens_target,
- config_name,
- };
- if entries.insert(model.clone(), entry).is_some() {
- return Err(ServerError::new(format!("duplicate route model {model}")));
- }
- }
- if entries.is_empty() {
- return Err(ServerError::new("at least one algorithm route is required"));
- }
+ /// Creates HTTP-server state around an already configured runner.
+ pub fn from_runner(runner: Runner) -> ServerResult<Self> {
let metrics = metrics::registry().map_err(ServerError::new)?;
let stats = StatsAccumulator::new(
metrics.clone(),
- entries.values().map(|entry| entry.algorithm.name()),
+ runner.models().map(|model| model.algorithm),
);
Ok(Self {
- routes: Arc::new(entries),
- config: None,
+ runner: Arc::new(runner),
metrics,
stats,
routing_log: None,
@@ -303,12 +212,6 @@ impl ServerState {
})
}
- /// Retains the validated configuration used to describe decision results.
- fn with_config(mut self, config: Arc<ServerConfig>) -> Self {
- self.config = Some(config);
- self
- }
-
/// Enables durable per-request routing records at `path`.
pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
self.routing_log = Some(SharedRoutingLog::new(path.into())?);
@@ -317,57 +220,44 @@ impl ServerState {
/// Returns the route model IDs served by the configured algorithms.
pub fn models(&self) -> impl Iterator<Item = &str> {
- self.routes.keys().map(ModelId::as_str)
+ self.runner.models().map(|model| model.id.as_str())
}
/// Returns the caller credential family used by `model`, if any.
- pub fn caller_auth_kind(&self, model: &str) -> ServerResult<Option<&'static str>> {
- self.route_for_model(model)
- .map(|entry| entry.caller_auth.map(CallerAuthKind::as_str))
- .ok_or_else(|| ServerError::new(format!("unknown route model {model:?}")))
+ pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> {
+ self.runner
+ .route(model)
+ .and_then(|route| route.caller_auth())
+ .map(|kind| kind.as_str())
}
- fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {
- self.routes.get(model)
+ fn route_for_model(&self, model: &str) -> Option<&Route> {
+ self.runner.route(model)
}
- /// Resolves the routing outcome through the target names configured for its route.
- fn decision_response<'a>(
- &'a self,
- route: &'a RouteEntry,
+ fn decision_response(
+ &self,
+ route_model: &ModelId,
outcome: &RoutingOutcome,
response: Option<Value>,
- ) -> Option<DecisionResponse<'a>> {
- let config = self.config.as_ref()?;
- let target_names = config.routing_target_names(route.config_name.as_ref()?)?;
- let resolve = |model: &ModelId| {
- let (target_name, target) = target_names
- .iter()
- .filter_map(|name| config.targets.get_key_value(*name))
- .find(|(_, target)| target.id == *model)?;
- let client = config.llm_clients.get(&target.llm_client)?;
- Some(DecisionTargetResponse {
- target: target_name,
- model: &target.id,
- llm_client: DecisionLlmClientResponse {
- format: client.format.wire_format(),
- base_url: client.base_url.as_str(),
- },
- extra_body: &target.extra_body,
- })
+ ) -> Option<DecisionResponse> {
+ let description = self.runner.describe_decision(route_model, outcome)?;
+ let convert = |target: DecisionTarget| DecisionTargetResponse {
+ target: target.target,
+ model: target.model,
+ llm_client: DecisionLlmClientResponse {
+ format: target.format,
+ base_url: target.base_url,
+ },
+ extra_body: target.extra_body,
};
Some(DecisionResponse {
- selected: resolve(&outcome.selected_model_id)?,
- fallbacks: outcome
- .fallback_models
- .iter()
- .map(resolve)
- .collect::<Option<Vec<_>>>()?,
+ selected: convert(description.selected),
+ fallbacks: description.fallbacks.into_iter().map(convert).collect(),
response,
})
}
}
-
/// Runtime options shared by server entry points.
#[derive(Clone, Debug)]
pub struct ServerRunOptions {
@@ -682,9 +572,15 @@ async fn decision(
Ok(resolved) => resolved,
Err(response) => return response,
};
- let mut outcome = match run_decision_only(route, request).await {
+ let route_model = request
+ .llm_request
+ .model
+ .as_deref()
+ .map(ModelId::from)
+ .unwrap_or_default();
+ let mut outcome = match route.decide(request).await {
Ok(outcome) => outcome,
- Err(error) => return algorithm_error(error),
+ Err(error) => return runner_error(error),
};
let response = match outcome.response.take() {
Some(response) => {
@@ -706,7 +602,7 @@ async fn decision(
}
None => None,
};
- match state.decision_response(route, &outcome, response) {
+ match state.decision_response(&route_model, &outcome, response) {
Some(response) => Json(response).into_response(),
None => {
server_error("routing outcome contains a model with no callable target configuration")
@@ -714,27 +610,6 @@ async fn decision(
}
}
-/// Completes routing-time calls and returns the outcome without serving its answer target.
-async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result<RoutingOutcome> {
- drive(Arc::clone(&route.algorithm), request, |call| {
- serve_decision_dependency(route, call)
- })
- .await
-}
-
-/// Serves a classifier or judge call that the decision depends on.
-async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> {
- let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
- let result = match route.target_clients.route(&model) {
- Ok(client) => client
- .call(call.request.clone())
- .await
- .map_err(|source| LibsyError::client_call(model, source)),
- Err(source) => Err(LibsyError::client_call(model, source)),
- };
- call.respond(result)
-}
-
/// Anthropic token counting against the route's explicitly configured target.
async fn anthropic_count_tokens(
State(state): State<ServerState>,
@@ -756,17 +631,16 @@ async fn anthropic_count_tokens(
Ok(resolved) => resolved,
Err(response) => return anthropic_error_response(response),
};
- let Some(target) = route.count_tokens_target.as_ref() else {
- return anthropic_error_response(error_response(
+ anthropic_error_response(match route.count_tokens(request).await {
+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
+ Err(RunnerError::CountTokensUnsupported) => error_response(
StatusCode::BAD_REQUEST,
"route has no Anthropic target for token counting",
"invalid_request_error",
"count_tokens_unsupported",
- ));
- };
- anthropic_error_response(match target.count_tokens(request).await {
- Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),
- Err(error) => count_tokens_error(error),
+ ),
+ Err(RunnerError::Client(error)) => count_tokens_error(error),
+ Err(error) => server_error(error.to_string()),
})
}
@@ -867,7 +741,7 @@ fn resolve_route(
metadata: Metadata,
body: Value,
wire_format: WireFormat,
-) -> std::result::Result<(&RouteEntry, Request), Response> {
+) -> std::result::Result<(&Route, Request), Response> {
let llm_request = decode_request(wire_format, &body)
.map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;
let requested_model = llm_request
@@ -890,8 +764,8 @@ fn resolve_route(
"model_not_found",
)
})?;
- if let Some(caller_auth) = route.caller_auth
- && !caller_auth.accepts(wire_format)
+ if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) =
+ route.check_caller_format(wire_format)
{
let (provider, expected_endpoint) = match caller_auth {
CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"),
@@ -929,17 +803,18 @@ async fn handle_llm_request(
};
// Only the Codex namespace mapping is needed downstream, not the whole request.
let request_extensions = request.llm_request.extensions.clone();
- let algorithm = Arc::clone(&route.algorithm);
- let client_router = route.target_clients.clone();
let observer = stats_observer(
state.stats.clone(),
state.routing_log.clone().zip(routing_log_context.clone()),
);
- let (selected_model, response) =
- match switchyard_llm_client::run(algorithm, client_router, request, Some(observer)).await {
- Ok(result) => result,
- Err(error) => return algorithm_error(error),
- };
+ let output = match route.execute(request, Some(observer)).await {
+ Ok(output) => output,
+ Err(error) => return runner_error(error),
+ };
+ let RunOutput {
+ selected_model,
+ response,
+ } = output;
// The response carries the candidate that actually served it. Fall back to the routing
// selection for algorithms that return a response without an offloaded model call.
let served_model = response.served_model().cloned().or(Some(selected_model));
@@ -1071,6 +946,14 @@ fn algorithm_error(error: LibsyError) -> Response {
client_error(source)
}
+fn runner_error(error: RunnerError) -> Response {
+ match error {
+ RunnerError::Algorithm(error) => algorithm_error(error),
+ RunnerError::Client(error) => client_error(&error),
+ error => server_error(error.to_string()),
+ }
+}
+
fn client_error(error: &LlmClientError) -> Response {
match error {
LlmClientError::InvalidRequest { message }
@@ -1235,9 +1118,9 @@ fn error_response(
async fn models(State(state): State<ServerState>) -> Json<Value> {
Json(model_list_payload(
state
- .routes
- .iter()
- .map(|(model, entry)| (model.as_str(), entry.capabilities)),
+ .runner
+ .models()
+ .map(|model| (model.id.as_str(), model.capabilities)),
))
}
@@ -1321,7 +1204,8 @@ async fn not_found() -> Response {
fn model_list_payload<'a>(
entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
) -> Value {
- let entries = entries.into_iter().collect::<Vec<_>>();
+ let mut entries = entries.into_iter().collect::<Vec<_>>();
+ entries.sort_unstable_by_key(|(model_id, _)| *model_id);
let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
let first_id = model_ids.first().copied();
let last_id = model_ids.last().copied();