Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: Extract from switchyard-server into switchyard-runner (#517)
Graham King8888492a0e5e
diff --git a/Cargo.lock b/Cargo.lockindex a5093f441..fde36135f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2343,6 +2343,24 @@ dependencies = ["tokio",]+[[package]]+name = "switchyard-runner"+version = "0.2.0"+dependencies = [+ "async-trait",+ "futures-util",+ "reqwest",+ "serde",+ "serde_json",+ "switchyard-libsy",+ "switchyard-llm-client",+ "switchyard-protocol",+ "thiserror 2.0.18",+ "tokio",+ "toml",+ "tracing",+]+[[package]]name = "switchyard-server"version = "0.2.0"@@ -2362,17 +2380,16 @@ dependencies = ["opentelemetry_sdk","parking_lot","prometheus",- "reqwest","rustls","serde","serde_json","switchyard-libsy","switchyard-llm-client","switchyard-protocol",+ "switchyard-runner","switchyard-translation","tempfile","tokio",- "toml","tower","tracing","tracing-opentelemetry",diff --git a/Cargo.toml b/Cargo.tomlindex 0fc5030a7..c1b8d1a94 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -8,6 +8,7 @@ members = ["crates/libsy-llm-client","crates/switchyard-py","crates/protocol",+ "crates/switchyard-runner","crates/switchyard-server","crates/switchyard-skill-distillation","crates/switchyard-soak",@@ -40,6 +41,7 @@ serde_json = { version = "1", features = ["preserve_order"] }switchyard-libsy = { path = "crates/libsy", version = "0.2.0" }switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.2.0" }switchyard-protocol = { path = "crates/protocol", version = "0.2.0" }+switchyard-runner = { path = "crates/switchyard-runner", version = "0.2.0" }switchyard-server = { path = "crates/switchyard-server", version = "0.2.0" }switchyard-translation = { path = "crates/switchyard-translation", version = "0.2.0" }thiserror = "2"diff --git a/crates/switchyard-py/src/server_bindings.rs b/crates/switchyard-py/src/server_bindings.rsindex 12b4395d6..1005d49f6 100644--- a/crates/switchyard-py/src/server_bindings.rs+++ b/crates/switchyard-py/src/server_bindings.rs@@ -44,13 +44,8 @@ impl PyServer {.map_err(|error| ServerConfigError::new_err(error.to_string()))?;let caller_auth_by_model = state.models()- .map(|model| {- state- .caller_auth_kind(model)- .map(|kind| (model.to_string(), kind))- })- .collect::<ServerResult<HashMap<_, _>>>()- .map_err(server_error)?;+ .map(|model| (model.to_string(), state.caller_auth_kind(model)))+ .collect::<HashMap<_, _>>();let runtime = pyo3_async_runtimes::tokio::get_runtime();let server = {let _guard = runtime.enter();diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.tomlnew file mode 100644index 000000000..5545f2c19--- /dev/null+++ b/crates/switchyard-runner/Cargo.toml@@ -0,0 +1,30 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++[package]+name = "switchyard-runner"+version.workspace = true+description = "Configured routing execution for Switchyard serving surfaces"+authors.workspace = true+edition.workspace = true+homepage = "https://github.com/NVIDIA-NeMo/Switchyard"+license.workspace = true+repository.workspace = true+rust-version.workspace = true+publish = ["crates-io"]++[dependencies]+libsy = { package = "switchyard-libsy", path = "../libsy", version = "0.2.0" }+reqwest.workspace = true+serde.workspace = true+serde_json.workspace = true+switchyard-llm-client.workspace = true+switchyard-protocol.workspace = true+thiserror.workspace = true+toml = "1.1"+tracing.workspace = true++[dev-dependencies]+async-trait.workspace = true+futures-util.workspace = true+tokio.workspace = truediff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsnew file mode 100644index 000000000..b4d400d74--- /dev/null+++ b/crates/switchyard-runner/src/algorithm.rs@@ -0,0 +1,1063 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Schema-neutral algorithm configuration and construction.++use std::collections::BTreeMap;+use std::error::Error;+use std::fmt::{Display, Formatter};+use std::sync::Arc;++use libsy::{+ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,+ ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig,+ GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop,+ Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter,+ SubagentRouterConfig, TargetPrompts, TaskClassifierConfig,+};+use serde::Deserialize;+use switchyard_protocol::ModelId;++/// Error returned when an algorithm description cannot be constructed.+#[derive(Debug)]+pub struct AlgorithmConfigError {+ message: String,+ source: Option<Box<dyn Error + Send + Sync>>,+}++impl AlgorithmConfigError {+ fn new(message: impl Into<String>) -> Self {+ Self {+ message: message.into(),+ source: None,+ }+ }++ fn with_source(message: impl Into<String>, source: impl Error + Send + Sync + 'static) -> Self {+ Self {+ message: message.into(),+ source: Some(Box::new(source)),+ }+ }+}++impl Display for AlgorithmConfigError {+ fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {+ formatter.write_str(&self.message)+ }+}++impl Error for AlgorithmConfigError {+ fn source(&self) -> Option<&(dyn Error + 'static)> {+ self.source.as_deref().map(|source| source as _)+ }+}++type AlgorithmResult<T> = Result<T, AlgorithmConfigError>;++/// How a custom classifier turns the judge's JSON verdict into a target.+#[derive(Clone, Debug, Deserialize)]+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]+pub enum ClassifierPolicyConfig {+ /// Reads the target name straight out of the judge's verdict.+ TargetSelector {+ /// JSON Pointer to the name, such as `/decision/target`.+ selector: String,+ },+}++/// Which of the three `llm_classifier` behaviors a route uses.+#[derive(Clone, Copy, Debug, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ClassifierMode {+ /// Judges the request first, then serves the strong or weak target.+ Capability,+ /// Serves the weak target first and judges the finished turn, moving to the+ /// strong target once the session latches.+ Escalation,+ /// Judges against your own JSON schema and routes to any configured target.+ Custom,+}++impl ClassifierPolicyConfig {+ fn into_libsy(self) -> CustomClassifierPolicy {+ match self {+ Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector),+ }+ }+}++#[derive(Clone, Debug)]+enum LlmClassifierModeConfig {+ Capability(CapabilityClassifierRouteConfig),+ Escalation(EscalationClassifierRouteConfig),+ Custom(CustomClassifierRouteConfig),+}++#[derive(Clone, Debug)]+struct CapabilityClassifierRouteConfig {+ strong_target: String,+ weak_target: String,+ base_threshold: f64,+ threshold_step: f64,+ classify_trigger: ClassifyTrigger,+ message_hash_fallback: bool,+ recent_turn_window: Option<usize>,+ prompt: Option<String>,+ response_format_type: ClassifierResponseFormat,+ max_output_tokens: u64,+}++#[derive(Clone, Debug)]+struct EscalationClassifierRouteConfig {+ strong_target: String,+ weak_target: String,+ prompt: Option<String>,+ response_format_type: ClassifierResponseFormat,+ max_output_tokens: u64,+ judge: EscalationJudgeConfig,+}++#[derive(Clone, Debug)]+struct CustomClassifierRouteConfig {+ classifier_target: String,+ targets: Vec<String>,+ default_target: String,+ prompt: String,+ response_schema: String,+ policy: ClassifierPolicyConfig,+ classify_trigger: ClassifyTrigger,+ message_hash_fallback: bool,+ recent_turn_window: Option<usize>,+ max_output_tokens: u64,+}++/// Settings for an `llm_classifier` route. Which fields are required depends on+/// the [`ClassifierMode`]; using a field from the wrong mode is an error.+#[derive(Clone, Debug, Default, Deserialize)]+#[serde(default, deny_unknown_fields)]+pub struct LlmClassifierRouteConfig {+ /// Target the judge is called through. Never a routing destination itself.+ pub classifier_target: String,+ /// Mode to run. Defaults to escalation when `escalation` is set, otherwise capability.+ pub mode: Option<ClassifierMode>,+ /// Capability and escalation modes: the capable tier.+ pub strong_target: Option<String>,+ /// Capability and escalation modes: the efficient tier.+ pub weak_target: Option<String>,+ /// Capability mode: lowest solve probability that still routes to the weak+ /// target, from 0 to 1.+ pub base_threshold: Option<f64>,+ /// Capability mode: how much to raise the threshold when the judge is+ /// uncertain. Added once for an uncertain verdict and twice for unsupported.+ pub threshold_step: Option<f64>,+ /// How often the judge runs: every request, once per user turn, or once per session.+ pub classify_trigger: ClassifyTrigger,+ /// Reuses the session's target by hashing the first user message when no+ /// session ID is available. Needs `classify_trigger = "new_session"`.+ pub message_hash_fallback: bool,+ /// How many trailing turns the judge sees. Unset shows it the opening task+ /// and the latest user follow-up only.+ pub recent_turn_window: Option<usize>,+ /// Replaces the packaged judge prompt. Required in custom mode.+ pub prompt: Option<String>,+ /// How the judge is asked for structured output. Use `json_object` when the+ /// provider cannot do JSON Schema.+ pub response_format_type: ClassifierResponseFormat,+ /// Most completion tokens the judge verdict may use.+ #[serde(default = "default_classifier_max_output_tokens")]+ pub max_output_tokens: u64,+ /// Escalation mode: how many escalate verdicts latch the session, and how+ /// much of the transcript the judge sees.+ pub escalation: Option<EscalationJudgeConfig>,+ /// Custom mode: the target names the policy may pick from.+ pub targets: Option<Vec<String>>,+ /// Custom mode: target used when the judge fails or its verdict cannot be routed.+ pub default_target: Option<String>,+ /// Custom mode: JSON Schema the verdict must match, written as a string.+ pub response_schema: Option<String>,+ /// Custom mode: how to read the chosen target out of the verdict.+ pub policy: Option<ClassifierPolicyConfig>,+}++/// Routing policy applied only to delegated sub-agent work, nested inside a+/// `passthrough` or `stage_router` route.+#[derive(Clone, Debug, Deserialize)]+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]+pub enum SubagentRouteConfig {+ /// Sends all sub-agent work to one target.+ Passthrough {+ /// Target that serves sub-agent requests.+ target: String,+ },+ /// Judges each sub-agent request. Only [`ClassifierMode::Custom`] is supported here.+ LlmClassifier(Box<LlmClassifierRouteConfig>),+}++impl SubagentRouteConfig {+ fn routing_target_names(&self) -> Vec<&str> {+ match self {+ Self::Passthrough { target } => vec![target],+ Self::LlmClassifier(classifier) => classifier+ .targets+ .iter()+ .flatten()+ .map(String::as_str)+ .collect(),+ }+ }++ fn classifier_target_name(&self) -> Option<&str> {+ match self {+ Self::LlmClassifier(classifier) => Some(&classifier.classifier_target),+ Self::Passthrough { .. } => None,+ }+ }+}++/// A routing algorithm described by configured target names.+#[derive(Clone, Debug, Deserialize)]+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]+pub enum AlgorithmSpec {+ /// Replies `OK` without calling any model. Useful for smoke tests.+ Noop {},+ /// Splits traffic across several targets.+ Random {+ /// Target names to choose from.+ targets: Vec<String>,+ /// Relative weights in `targets` order. Equal weights when unset.+ weights: Option<Vec<f64>>,+ /// Makes the sequence of choices repeatable.+ seed: Option<u64>,+ },+ /// Sends every request to one target.+ Passthrough {+ /// Target that serves the request.+ target: String,+ /// Separate policy for delegated sub-agent work.+ #[serde(default)]+ subagents: Option<SubagentRouteConfig>,+ },+ /// Asks a judge model which target should serve the request.+ LlmClassifier {+ /// Judge and tier settings, written directly in the route table.+ #[serde(flatten)]+ config: LlmClassifierRouteConfig,+ },+ /// Picks a tier per turn by scoring signals from recent tool results.+ StageRouter {+ /// The capable tier.+ capable_target: String,+ /// The efficient tier.+ efficient_target: String,+ /// Tier to use when the signals are not confident.+ picker: PickerMode,+ /// How much agreement a decisive pick needs, from 0 to 1.+ confidence_threshold: f64,+ /// How many trailing tool results the signals are scored over.+ #[serde(default)]+ recent_turn_window: Option<usize>,+ /// Notes handed to a tier when the router switches to it.+ #[serde(default)]+ handoff_notes: Option<HandoffNoteConfig>,+ /// System prompt handed to the capable tier.+ #[serde(default)]+ capable_system_prompt: Option<String>,+ /// System prompt handed to the efficient tier.+ #[serde(default)]+ efficient_system_prompt: Option<String>,+ /// Judge consulted for turns the tool signals cannot decide.+ #[serde(default)]+ classifier: Option<StageClassifierConfig>,+ /// Separate policy for delegated sub-agent work.+ #[serde(default)]+ subagents: Option<SubagentRouteConfig>,+ },+ /// Serves every turn from one target, and has a second model review some of+ /// those turns before the caller sees them.+ Advisor {+ /// Serves every client-visible turn.+ executor_target: String,+ /// Reviews gated turns. Never a routing destination.+ advisor_target: String,+ /// Replaces the built-in APPROVE/REDO reviewer prompt.+ #[serde(default)]+ reviewer_system_prompt: Option<String>,+ /// Replaces the built-in text put in front of a REDO plan.+ #[serde(default)]+ redo_feedback_prefix: Option<String>,+ /// What fires a review.+ #[serde(default)]+ gate_trigger: AdvisorTriggerConfig,+ /// Regular expression for the `pattern` trigger. Required by it, and unused otherwise.+ #[serde(default)]+ gate_trigger_pattern: Option<String>,+ /// How many reviews one session may spend.+ #[serde(default = "default_max_reviews")]+ max_reviews: u32,+ /// Reviews a turn after this many assistant turns, as a mid-task+ /// checkpoint. Zero turns the checkpoint off.+ #[serde(default)]+ gate_stall_turns: u32,+ /// Tool results a turn needs before it can be reviewed. Skips early chatty turns.+ #[serde(default)]+ gate_min_tool_results: u32,+ /// Most output tokens one review may use.+ #[serde(default = "default_advisor_max_tokens")]+ advisor_max_tokens: u64,+ /// Sampling temperature for reviews. Left off the request when unset.+ #[serde(default)]+ advisor_temperature: Option<f64>,+ /// Size cap on the transcript sent to the advisor. Longer transcripts+ /// are trimmed from the middle.+ #[serde(default = "default_transcript_max_chars")]+ transcript_max_chars: usize,+ /// Lets the turn through when the advisor fails, instead of erroring.+ #[serde(default = "default_fail_open")]+ fail_open: bool,+ },+}++/// What fires an advisor route's review.+#[derive(Clone, Debug, Default, Deserialize, PartialEq)]+#[serde(rename_all = "snake_case")]+pub enum AdvisorTriggerConfig {+ /// The executor's first turn without tool calls.+ #[default]+ NoToolCall,+ /// The first turn whose text matches `gate_trigger_pattern`.+ Pattern,+}++/// The judge a `stage_router` route falls through to, and how it routes.+#[derive(Clone, Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct StageClassifierConfig {+ /// Target the judge is called through. Not a routing destination.+ pub target: String,+ /// Lowest solve probability that still routes to the efficient tier, from 0 to 1.+ pub base_threshold: f64,+ /// How much to raise the threshold when the judge is uncertain. Added once+ /// for an uncertain verdict and twice for unsupported.+ #[serde(default)]+ pub threshold_step: f64,+ /// How often the judge runs. `new_session` has no effect here.+ #[serde(default)]+ pub classify_trigger: ClassifyTrigger,+ /// Reuses the session's target by hashing the first user message when no+ /// session ID is available.+ #[serde(default)]+ pub message_hash_fallback: bool,+ /// How many trailing turns the judge sees. Unset shows it the opening task+ /// and the latest user follow-up only.+ #[serde(default)]+ pub recent_turn_window: Option<usize>,+ /// Replaces the packaged judge prompt.+ #[serde(default)]+ pub prompt: Option<String>,+ /// How the judge is asked for structured output. Use `json_object` when the+ /// provider cannot do JSON Schema.+ #[serde(default)]+ pub response_format_type: ClassifierResponseFormat,+ /// Most completion tokens the judge verdict may use.+ #[serde(default = "default_classifier_max_output_tokens")]+ pub max_output_tokens: u64,+}++impl StageClassifierConfig {+ fn task_classifier_config(&self) -> TaskClassifierConfig {+ TaskClassifierConfig {+ base_threshold: self.base_threshold,+ threshold_step: self.threshold_step,+ classify_trigger: self.classify_trigger,+ message_hash_fallback: self.message_hash_fallback,+ recent_turn_window: self.recent_turn_window,+ contract: classifier_contract(self.prompt.as_deref())+ .with_response_format_type(self.response_format_type),+ max_output_tokens: self.max_output_tokens,+ }+ }+}++impl AlgorithmSpec {+ /// Completion targets in algorithm order; judge-only targets are excluded.+ pub fn routing_target_names(&self) -> Vec<&str> {+ match self {+ Self::Noop { .. } => Vec::new(),+ Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(),+ Self::Passthrough {+ target, subagents, ..+ } => {+ let mut names = vec![target.as_str()];+ if let Some(subagents) = subagents {+ names.extend(subagents.routing_target_names());+ }+ names+ }+ Self::LlmClassifier { config, .. } => {+ match config.mode.unwrap_or(if config.escalation.is_some() {+ ClassifierMode::Escalation+ } else {+ ClassifierMode::Capability+ }) {+ ClassifierMode::Capability => config+ .weak_target+ .iter()+ .chain(&config.strong_target)+ .map(String::as_str)+ .collect(),+ ClassifierMode::Escalation => config+ .strong_target+ .iter()+ .chain(&config.weak_target)+ .map(String::as_str)+ .collect(),+ ClassifierMode::Custom => config+ .targets+ .iter()+ .flatten()+ .map(String::as_str)+ .collect(),+ }+ }+ Self::StageRouter {+ capable_target,+ efficient_target,+ subagents,+ ..+ } => {+ let mut names = vec![capable_target.as_str(), efficient_target.as_str()];+ if let Some(subagents) = subagents {+ names.extend(subagents.routing_target_names());+ }+ names+ }+ // The advisor is judge-only: reviews go through its own client,+ // so it is not a completion (or count_tokens) destination.+ Self::Advisor {+ executor_target, ..+ } => vec![executor_target],+ }+ }++ /// Every target the algorithm may call, including judge-only targets.+ ///+ /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;+ /// a classifier also calls its judge, and that call needs a client too.+ pub fn callable_target_names(&self) -> Vec<&str> {+ let mut names = self.routing_target_names();+ match self {+ Self::LlmClassifier { config, .. } => names.push(&config.classifier_target),+ Self::Passthrough {+ subagents: Some(subagents),+ ..+ } => names.extend(subagents.classifier_target_name()),+ Self::StageRouter {+ classifier,+ subagents,+ ..+ } => {+ if let Some(classifier) = classifier {+ names.push(&classifier.target);+ }+ if let Some(subagents) = subagents {+ names.extend(subagents.classifier_target_name());+ }+ }+ Self::Advisor { advisor_target, .. } => names.push(advisor_target),+ _ => {}+ }+ names+ }+ /// Builds this algorithm after resolving configured target names.+ pub fn build(+ &self,+ context: &str,+ targets: &BTreeMap<String, ModelId>,+ ) -> AlgorithmResult<Arc<dyn Algorithm>> {+ build_algorithm(context, self, targets)+ }+}+impl LlmClassifierRouteConfig {+ fn classifier_mode(&self, route_name: &str) -> AlgorithmResult<LlmClassifierModeConfig> {+ let Self {+ classifier_target,+ mode,+ strong_target,+ weak_target,+ base_threshold,+ threshold_step,+ classify_trigger,+ message_hash_fallback,+ recent_turn_window,+ prompt,+ response_format_type,+ max_output_tokens,+ escalation,+ targets,+ default_target,+ response_schema,+ policy,+ } = self;++ let selected_mode = match (mode, escalation.is_some()) {+ (Some(mode), _) => *mode,+ (None, true) => ClassifierMode::Escalation,+ (None, false) => ClassifierMode::Capability,+ };++ match selected_mode {+ ClassifierMode::Capability => {+ if escalation.is_some() {+ return Err(classifier_field_error(+ route_name,+ "escalation",+ "capability",+ ));+ }+ reject_custom_fields(+ route_name,+ "capability",+ targets,+ default_target,+ response_schema,+ policy,+ )?;+ Ok(LlmClassifierModeConfig::Capability(+ CapabilityClassifierRouteConfig {+ strong_target: required_classifier_field(+ route_name,+ "strong_target",+ strong_target,+ )?,+ weak_target: required_classifier_field(+ route_name,+ "weak_target",+ weak_target,+ )?,+ base_threshold: required_classifier_field(+ route_name,+ "base_threshold",+ base_threshold,+ )?,+ threshold_step: threshold_step.unwrap_or_default(),+ classify_trigger: *classify_trigger,+ message_hash_fallback: *message_hash_fallback,+ recent_turn_window: *recent_turn_window,+ prompt: prompt.clone(),+ response_format_type: *response_format_type,+ max_output_tokens: *max_output_tokens,+ },+ ))+ }+ ClassifierMode::Escalation => {+ reject_custom_fields(+ route_name,+ "escalation",+ targets,+ default_target,+ response_schema,+ policy,+ )?;+ if *classify_trigger != ClassifyTrigger::EveryRequest {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} mode escalation cannot use classify_trigger"+ )));+ }+ if mode.is_some()+ && (base_threshold.is_some()+ || threshold_step.is_some()+ || *message_hash_fallback+ || recent_turn_window.is_some())+ {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} mode escalation cannot use capability routing settings"+ )));+ }+ Ok(LlmClassifierModeConfig::Escalation(+ EscalationClassifierRouteConfig {+ strong_target: required_classifier_field(+ route_name,+ "strong_target",+ strong_target,+ )?,+ weak_target: required_classifier_field(+ route_name,+ "weak_target",+ weak_target,+ )?,+ prompt: prompt.clone(),+ response_format_type: *response_format_type,+ max_output_tokens: *max_output_tokens,+ judge: required_classifier_field(route_name, "escalation", escalation)?,+ },+ ))+ }+ ClassifierMode::Custom => {+ if strong_target.is_some()+ || weak_target.is_some()+ || base_threshold.is_some()+ || threshold_step.is_some()+ || escalation.is_some()+ || *response_format_type != ClassifierResponseFormat::JsonSchema+ {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} mode custom cannot use capability or escalation fields and response_format_type must be 'json_schema'"+ )));+ }+ Ok(LlmClassifierModeConfig::Custom(+ CustomClassifierRouteConfig {+ classifier_target: classifier_target.clone(),+ targets: required_classifier_field(route_name, "targets", targets)?,+ default_target: required_classifier_field(+ route_name,+ "default_target",+ default_target,+ )?,+ prompt: required_classifier_field(route_name, "prompt", prompt)?,+ response_schema: required_classifier_field(+ route_name,+ "response_schema",+ response_schema,+ )?,+ policy: required_classifier_field(route_name, "policy", policy)?,+ classify_trigger: *classify_trigger,+ message_hash_fallback: *message_hash_fallback,+ recent_turn_window: *recent_turn_window,+ max_output_tokens: *max_output_tokens,+ },+ ))+ }+ }+ }+}++fn reject_custom_fields(+ route_name: &str,+ mode: &str,+ targets: &Option<Vec<String>>,+ default_target: &Option<String>,+ response_schema: &Option<String>,+ policy: &Option<ClassifierPolicyConfig>,+) -> AlgorithmResult<()> {+ if targets.is_some()+ || default_target.is_some()+ || response_schema.is_some()+ || policy.is_some()+ {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields"+ )));+ }+ Ok(())+}++fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> AlgorithmConfigError {+ AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} mode {mode} cannot use {field}"+ ))+}++fn required_classifier_field<T: Clone>(+ route_name: &str,+ field: &str,+ value: &Option<T>,+) -> AlgorithmResult<T> {+ value.clone().ok_or_else(|| {+ AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name} requires {field}"+ ))+ })+}++fn build_subagent_router_config(+ route_name: &str,+ config: &SubagentRouteConfig,+ targets: &BTreeMap<String, ModelId>,+) -> AlgorithmResult<SubagentRouterConfig> {+ match config {+ SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target(+ resolve_target_model_id(route_name, target, targets)?,+ )),+ SubagentRouteConfig::LlmClassifier(config) => {+ let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)?+ else {+ return Err(AlgorithmConfigError::new(format!(+ "route {route_name}: subagents llm_classifier only supports mode custom"+ )));+ };+ let judge_target =+ resolve_target_model_id(route_name, &config.classifier_target, targets)?;+ let resolved_targets = config+ .targets+ .iter()+ .map(|name| {+ resolve_target_model_id(route_name, name, targets)+ .map(|target| (name.clone(), target))+ })+ .collect::<AlgorithmResult<Vec<_>>>()?;+ let default_target = resolved_targets+ .iter()+ .find(|(name, _)| *name == config.default_target)+ .map(|(_, target)| target.clone())+ .ok_or_else(|| {+ AlgorithmConfigError::new(format!(+ "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets",+ config.default_target+ ))+ })?;+ let response_schema =+ serde_json::from_str(&config.response_schema).map_err(|error| {+ AlgorithmConfigError::with_source(+ format!(+ "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}"+ ),+ error,+ )+ })?;+ let mut classifier_config = CustomClassifierConfig::new(+ config.prompt,+ response_schema,+ config.policy.into_libsy(),+ );+ classifier_config.recent_turn_window = config.recent_turn_window;+ classifier_config.max_output_tokens = config.max_output_tokens;+ let subagent_targets = resolved_targets+ .iter()+ .map(|(_, target)| target.clone())+ .collect();+ let classifier = Arc::new(+ LlmTaskClassifier::new(LlmClassifierConfig::Custom {+ judge_target,+ targets: resolved_targets,+ default_target: config.default_target,+ config: classifier_config,+ })+ .map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("route {route_name}: subagents llm_classifier: {error}"),+ error,+ )+ })?,+ );+ Ok(SubagentRouterConfig {+ targets: subagent_targets,+ classifier,+ default_target,+ classify_trigger: config.classify_trigger,+ message_hash_fallback: config.message_hash_fallback,+ })+ }+ }+}++fn attach_subagent_router(+ route_name: &str,+ parent: Arc<dyn Algorithm>,+ config: Option<&SubagentRouteConfig>,+ targets: &BTreeMap<String, ModelId>,+) -> AlgorithmResult<Arc<dyn Algorithm>> {+ let Some(config) = config else {+ return Ok(parent);+ };+ let config = build_subagent_router_config(route_name, config, targets)?;+ let algorithm = SubagentRouter::new(parent, config).map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("route {route_name}: subagent routing: {error}"),+ error,+ )+ })?;+ Ok(Arc::new(algorithm))+}++fn build_algorithm(+ route_name: &str,+ config: &AlgorithmSpec,+ targets: &BTreeMap<String, ModelId>,+) -> AlgorithmResult<Arc<dyn Algorithm>> {+ match config {+ AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})),+ AlgorithmSpec::Random {+ targets: names,+ weights,+ seed,+ ..+ } => {+ let target_set =+ resolve_targets(route_name, names.iter().map(String::as_str), targets)?;+ let algorithm = Random::new(target_set, weights.clone(), *seed).map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("random route {route_name}: {error}"),+ error,+ )+ })?;+ Ok(Arc::new(algorithm))+ }+ AlgorithmSpec::Passthrough {+ target, subagents, ..+ } => {+ let parent_target = resolve_target_model_id(route_name, target, targets)?;+ let algorithm = Passthrough::new(parent_target);+ let parent: Arc<dyn Algorithm> = Arc::new(algorithm);+ attach_subagent_router(route_name, parent, subagents.as_ref(), targets)+ }+ AlgorithmSpec::LlmClassifier {+ config: classifier_config,+ ..+ } => {+ let classifier =+ resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?;+ let mode = classifier_config.classifier_mode(route_name)?;+ let algorithm = match mode {+ LlmClassifierModeConfig::Capability(config) => {+ let strong =+ resolve_target_model_id(route_name, &config.strong_target, targets)?;+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;+ let classifier_config = TaskClassifierConfig {+ base_threshold: config.base_threshold,+ threshold_step: config.threshold_step,+ classify_trigger: config.classify_trigger,+ message_hash_fallback: config.message_hash_fallback,+ recent_turn_window: config.recent_turn_window,+ contract: classifier_contract(config.prompt.as_deref())+ .with_response_format_type(config.response_format_type),+ max_output_tokens: config.max_output_tokens,+ };+ LlmTaskClassifier::new(LlmClassifierConfig::Capability {+ judge_target: classifier,+ efficient_target: weak,+ capable_target: strong,+ config: classifier_config,+ })+ }+ LlmClassifierModeConfig::Escalation(config) => {+ let strong =+ resolve_target_model_id(route_name, &config.strong_target, targets)?;+ let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;+ LlmTaskClassifier::new(LlmClassifierConfig::Escalation {+ judge_target: classifier,+ efficient_target: weak,+ capable_target: strong,+ contract: classifier_contract(config.prompt.as_deref())+ .with_response_format_type(config.response_format_type),+ config: config.judge,+ max_output_tokens: config.max_output_tokens,+ })+ }+ LlmClassifierModeConfig::Custom(config) => {+ let resolved_targets = config+ .targets+ .iter()+ .map(|name| {+ resolve_target_model_id(route_name, name, targets)+ .map(|target| (name.clone(), target))+ })+ .collect::<AlgorithmResult<Vec<_>>>()?;+ let response_schema = serde_json::from_str(&config.response_schema).map_err(+ |error| {+ AlgorithmConfigError::with_source(+ format!(+ "llm_classifier route {route_name}: response_schema is invalid JSON: {error}"+ ),+ error,+ )+ },+ )?;+ let mut classifier_config = CustomClassifierConfig::new(+ config.prompt,+ response_schema,+ config.policy.into_libsy(),+ );+ classifier_config.classify_trigger = config.classify_trigger;+ classifier_config.message_hash_fallback = config.message_hash_fallback;+ classifier_config.recent_turn_window = config.recent_turn_window;+ classifier_config.max_output_tokens = config.max_output_tokens;+ LlmTaskClassifier::new(LlmClassifierConfig::Custom {+ judge_target: classifier,+ targets: resolved_targets,+ default_target: config.default_target,+ config: classifier_config,+ })+ }+ }+ .map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("llm_classifier route {route_name}: {error}"),+ error,+ )+ })?;+ Ok(Arc::new(algorithm))+ }+ AlgorithmSpec::StageRouter {+ capable_target,+ efficient_target,+ picker,+ confidence_threshold,+ recent_turn_window,+ handoff_notes,+ capable_system_prompt,+ efficient_system_prompt,+ classifier,+ subagents,+ ..+ } => {+ if matches!(picker, PickerMode::CapableFirst) {+ tracing::warn!(+ "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."+ );+ }+ let capable = resolve_target_model_id(route_name, capable_target, targets)?;+ let efficient = resolve_target_model_id(route_name, efficient_target, targets)?;+ let mut config = StageRouterConfig::new(*picker, *confidence_threshold);+ config.recent_window = *recent_turn_window;+ config.handoff_notes = handoff_notes.clone();+ config.tier_prompts = tier_prompts(+ &capable,+ capable_system_prompt.as_deref(),+ &efficient,+ efficient_system_prompt.as_deref(),+ );+ // The judge is called through its own target, so it is not a routing+ // destination and stays out of the tier pair.+ config.llm_fallback = classifier+ .as_ref()+ .map(|classifier| {+ resolve_target_model_id(route_name, &classifier.target, targets).map(+ |judge_target| LlmFallback {+ judge_target,+ config: classifier.task_classifier_config(),+ },+ )+ })+ .transpose()?;+ let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("stage_router route {route_name}: {error}"),+ error,+ )+ })?;+ let parent: Arc<dyn Algorithm> = Arc::new(algorithm);+ attach_subagent_router(route_name, parent, subagents.as_ref(), targets)+ }+ AlgorithmSpec::Advisor {+ executor_target,+ advisor_target,+ reviewer_system_prompt,+ redo_feedback_prefix,+ gate_trigger,+ gate_trigger_pattern,+ max_reviews,+ gate_stall_turns,+ gate_min_tool_results,+ advisor_max_tokens,+ advisor_temperature,+ transcript_max_chars,+ fail_open,+ ..+ } => {+ let executor = resolve_target_model_id(route_name, executor_target, targets)?;+ let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;+ // A pattern set under the default trigger would be silently+ // ignored; reject the misconfiguration instead.+ if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {+ return Err(AlgorithmConfigError::new(format!(+ "advisor route {route_name}: gate_trigger_pattern requires \+ gate_trigger = \"pattern\""+ )));+ }+ let mut config = AdvisorGateConfig::default();+ if let Some(prompt) = reviewer_system_prompt {+ config.reviewer_system_prompt = prompt.clone();+ }+ if let Some(prefix) = redo_feedback_prefix {+ config.redo_feedback_prefix = prefix.clone();+ }+ config.gate_trigger = match gate_trigger {+ AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,+ AdvisorTriggerConfig::Pattern => {+ GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())+ }+ };+ config.max_reviews = *max_reviews;+ config.gate_stall_turns = *gate_stall_turns;+ config.gate_min_tool_results = *gate_min_tool_results;+ config.advisor_max_tokens = *advisor_max_tokens;+ config.advisor_temperature = *advisor_temperature;+ config.transcript_max_chars = *transcript_max_chars;+ config.fail_open = *fail_open;+ let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {+ AlgorithmConfigError::with_source(+ format!("advisor route {route_name}: {error}"),+ error,+ )+ })?;+ Ok(Arc::new(algorithm))+ }+ }+}++const fn default_max_reviews() -> u32 {+ 1+}++const fn default_advisor_max_tokens() -> u64 {+ 2048+}++const fn default_transcript_max_chars() -> usize {+ 200_000+}++const fn default_fail_open() -> bool {+ true+}++fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {+ prompt.map_or_else(ClassifierContractConfig::default, |prompt| {+ ClassifierContractConfig::default().with_prompt(prompt)+ })+}++fn default_classifier_max_output_tokens() -> u64 {+ TaskClassifierConfig::default().max_output_tokens+}++/// Keys each configured system prompt by the target it belongs to.+fn tier_prompts(+ capable: &str,+ capable_prompt: Option<&str>,+ efficient: &str,+ efficient_prompt: Option<&str>,+) -> TargetPrompts {+ let mut prompts = TargetPrompts::default();+ if let Some(prompt) = capable_prompt {+ prompts = prompts.with(capable, prompt);+ }+ if let Some(prompt) = efficient_prompt {+ prompts = prompts.with(efficient, prompt);+ }+ prompts+}++fn resolve_targets<'a>(+ route_name: &str,+ names: impl IntoIterator<Item = &'a str>,+ targets: &BTreeMap<String, ModelId>,+) -> AlgorithmResult<Vec<ModelId>> {+ names+ .into_iter()+ .map(|name| resolve_target_model_id(route_name, name, targets))+ .collect()+}++fn resolve_target_model_id(+ route_name: &str,+ name: &str,+ targets: &BTreeMap<String, ModelId>,+) -> AlgorithmResult<ModelId> {+ targets.get(name).cloned().ok_or_else(|| {+ AlgorithmConfigError::new(format!(+ "route {route_name} references unknown target {name}"+ ))+ })+}diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsnew file mode 100644index 000000000..7a90c7bba--- /dev/null+++ b/crates/switchyard-runner/src/config.rs@@ -0,0 +1,1401 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Version-1 TOML deployment loading for the shared runner.++use std::collections::{BTreeMap, HashMap, HashSet};+use std::fs;+use std::path::Path;+use std::sync::Arc;++use serde::de::DeserializeOwned;+use serde::{Deserialize, Deserializer};+use serde_json::Value;+use switchyard_llm_client::{+ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,+ TranslatingLlmClient,+};+use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};++use crate::{+ AlgorithmSpec, CallerAuthKind, CountTokensTarget, DecisionTarget, ModelCapabilities, Route,+ Runner, RunnerError,+};++const SUPPORTED_SCHEMA_VERSION: u32 = 1;+const MAX_CONFIGURED_RETRIES: u32 = 10;++type RunnerResult<T> = Result<T, RunnerError>;++pub(crate) fn load_runner(path: impl AsRef<Path>) -> RunnerResult<Runner> {+ let path = path.as_ref();+ let source = fs::read_to_string(path).map_err(|error| {+ RunnerError::configuration_source(+ format!("failed to read server config {}: {error}", path.display()),+ error,+ )+ })?;+ runner_from_toml(&source).map_err(|error| {+ RunnerError::configuration_source(+ format!("invalid server config {}: {error}", path.display()),+ error,+ )+ })+}++fn runner_from_toml(source: &str) -> RunnerResult<Runner> {+ let config: DeploymentConfig = toml::from_str(source).map_err(|error| {+ RunnerError::configuration_source(format!("failed to parse TOML: {error}"), error)+ })?;+ config.build()+}++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub(crate) struct DeploymentConfig {+ schema_version: u32,+ #[serde(default)]+ llm_clients: BTreeMap<String, LlmClientConfig>,+ targets: BTreeMap<String, TargetConfig>,+ routes: BTreeMap<String, RouteConfig>,+}++#[derive(Debug)]+struct RouteConfig {+ id: ModelId,+ context_window: Option<u32>,+ tool_calling: Option<bool>,+ reasoning: Option<bool>,+ algorithm: AlgorithmSpec,+}++impl<'de> Deserialize<'de> for RouteConfig {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: Deserializer<'de>,+ {+ let mut table = toml::Table::deserialize(deserializer)?;+ let id = take_required(&mut table, "id")?;+ let context_window = take_optional(&mut table, "context_window")?;+ let tool_calling = take_optional(&mut table, "tool_calling")?;+ let reasoning = take_optional(&mut table, "reasoning")?;+ let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))+ .map_err(serde::de::Error::custom)?;+ Ok(Self {+ id,+ context_window,+ tool_calling,+ reasoning,+ algorithm,+ })+ }+}++fn take_required<T, E>(table: &mut toml::Table, name: &'static str) -> Result<T, E>+where+ T: DeserializeOwned,+ E: serde::de::Error,+{+ let value = table.remove(name).ok_or_else(|| E::missing_field(name))?;+ T::deserialize(value).map_err(E::custom)+}++fn take_optional<T, E>(table: &mut toml::Table, name: &'static str) -> Result<Option<T>, E>+where+ T: DeserializeOwned,+ E: serde::de::Error,+{+ table+ .remove(name)+ .map(|value| T::deserialize(value).map_err(E::custom))+ .transpose()+}++impl RouteConfig {+ fn capabilities(&self) -> ModelCapabilities {+ ModelCapabilities {+ context_window: self.context_window,+ tool_calling: self.tool_calling,+ reasoning: self.reasoning,+ }+ }++ fn routing_target_names(&self) -> Vec<&str> {+ self.algorithm.routing_target_names()+ }++ fn callable_target_names(&self) -> Vec<&str> {+ self.algorithm.callable_target_names()+ }+}++impl DeploymentConfig {+ fn decision_target(&self, name: &str) -> Option<DecisionTarget> {+ let target = self.targets.get(name)?;+ let client = self.llm_clients.get(&target.llm_client)?;+ Some(DecisionTarget {+ target: name.to_string(),+ model: target.id.clone(),+ format: client.format.wire_format(),+ base_url: client.base_url.as_str().to_string(),+ extra_body: target.extra_body.clone(),+ })+ }++ fn build(self) -> RunnerResult<Runner> {+ if self.schema_version != SUPPORTED_SCHEMA_VERSION {+ return Err(RunnerError::configuration(format!(+ "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",+ self.schema_version+ )));+ }++ let mut seen_client_model_ids = HashSet::new();+ for (target_name, target) in &self.targets {+ validate_value("target name", target_name)?;+ validate_value(&format!("target {target_name} id"), &target.id)?;+ if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) {+ tracing::warn!(+ "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",+ target.id,+ target.llm_client+ );+ }+ }++ let clients = self.build_clients()?;+ let targets = self.build_targets();+ let mut routes = Vec::with_capacity(self.routes.len());+ for (route_name, config) in &self.routes {+ validate_value("route name", route_name)?;+ validate_value(&format!("route {route_name} id"), &config.id)?;+ for target_name in config.callable_target_names() {+ self.targets.get(target_name).ok_or_else(|| {+ RunnerError::configuration(format!(+ "route references unknown target {target_name}"+ ))+ })?;+ }+ let capabilities = config.capabilities();+ if capabilities.context_window == Some(0) {+ return Err(RunnerError::configuration(format!(+ "route {route_name} context_window must be greater than zero"+ )));+ }+ let algorithm = config+ .algorithm+ .build(route_name, &targets)+ .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;+ let (route_clients, caller_auth) =+ self.build_route_clients(route_name, config, &clients)?;+ let count_tokens_target = self.build_count_tokens_target(config, &clients);+ let decision_targets = config+ .routing_target_names()+ .into_iter()+ .filter_map(|name| self.decision_target(name))+ .collect();+ let route = Route::new(+ algorithm,+ route_clients,+ caller_auth,+ capabilities,+ count_tokens_target,+ decision_targets,+ );+ routes.push((config.id.clone(), route));+ }+ Ok(Runner::new(routes))+ }++ fn build_clients(&self) -> RunnerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {+ let mut models_by_client = self+ .llm_clients+ .keys()+ .map(|name| (name.clone(), Vec::new()))+ .collect::<BTreeMap<String, Vec<ModelConfig>>>();++ for (name, client_config) in &self.llm_clients {+ validate_value("llm client name", name)?;+ build_backend(name, client_config, &BTreeMap::new())?;+ }+ for (target_name, target) in &self.targets {+ let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {+ RunnerError::configuration(format!(+ "target {target_name} references unknown llm client {}",+ target.llm_client+ ))+ })?;+ let model_configs = models_by_client+ .get_mut(&target.llm_client)+ .ok_or_else(|| {+ RunnerError::configuration("validated llm client was not initialized")+ })?;+ model_configs.push(ModelConfig::new(+ target.id.clone(),+ build_backend(&target.llm_client, client_config, &target.extra_body)?,+ None,+ ));+ }++ let mut clients = BTreeMap::new();+ for (name, model_configs) in models_by_client {+ let client = Arc::new(+ TranslatingLlmClient::new(&model_configs)+ .map_err(|error| RunnerError::configuration(error.to_string()))?,+ );+ clients.insert(name, client);+ }+ Ok(clients)+ }++ fn build_targets(&self) -> BTreeMap<String, ModelId> {+ self.targets+ .iter()+ .map(|(name, config)| (name.clone(), config.id.clone()))+ .collect()+ }++ fn build_route_clients(+ &self,+ route_name: &str,+ route: &RouteConfig,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ ) -> RunnerResult<(ClientRouter, Option<CallerAuthKind>)> {+ let mut by_model = HashMap::new();+ let mut caller_auth = None;+ for name in route.callable_target_names() {+ let target = self.targets.get(name).ok_or_else(|| {+ RunnerError::configuration(format!("route references unknown target {name}"))+ })?;+ let client = clients.get(&target.llm_client).ok_or_else(|| {+ RunnerError::configuration(format!("target {name} has no constructed llm client"))+ })?;+ let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {+ RunnerError::configuration(format!(+ "target {name} references unknown llm client {}",+ target.llm_client+ ))+ })?;+ if client_config.forward_auth {+ let target_auth = client_config.format.caller_auth_kind();+ if caller_auth.is_some_and(|kind| kind != target_auth) {+ return Err(RunnerError::configuration(format!(+ "route {route_name} cannot forward both Anthropic and OpenAI caller credentials"+ )));+ }+ caller_auth = Some(target_auth);+ }+ let client: Arc<dyn RoutedLlmClient> = client.clone();+ by_model.insert(target.id.clone(), client);+ }+ Ok((ClientRouter::new(by_model), caller_auth))+ }++ fn build_count_tokens_target(+ &self,+ route: &RouteConfig,+ clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ ) -> Option<CountTokensTarget> {+ route+ .routing_target_names()+ .into_iter()+ .enumerate()+ .filter_map(|(index, name)| {+ let target = self.targets.get(name)?;+ let client = clients.get(&target.llm_client)?;+ client.supports_count_tokens(&target.id).then_some((+ count_tokens_priority(name, &target.id),+ index,+ target,+ client,+ ))+ })+ .min_by_key(|(priority, index, _, _)| (*priority, *index))+ .map(|(_, _, target, client)| CountTokensTarget {+ model: target.id.clone(),+ client: client.clone(),+ })+ }+}++fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {+ let target_name = target_name.to_ascii_lowercase();+ let model_id = model_id.to_ascii_lowercase();+ ["opus", "sonnet", "haiku"]+ .iter()+ .position(|hint| target_name.contains(hint) || model_id.contains(hint))+ .unwrap_or(3)+}++/// A client endpoint, parsed when the config loads rather than checked afterwards.+///+/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no+/// later stage has to re-check it or can forget to.+#[derive(Clone, Debug)]+struct HttpBaseUrl(reqwest::Url);++impl HttpBaseUrl {+ fn as_str(&self) -> &str {+ self.0.as_str()+ }+}++impl<'de> Deserialize<'de> for HttpBaseUrl {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ let raw = String::deserialize(deserializer)?;+ let url = reqwest::Url::parse(raw.trim()).map_err(|error| {+ serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}"))+ })?;+ if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {+ return Err(serde::de::Error::custom(+ "base_url must be an absolute HTTP(S) URL",+ ));+ }+ Ok(Self(url))+ }+}++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+struct LlmClientConfig {+ format: ClientFormat,+ base_url: HttpBaseUrl,+ api_key_env: Option<String>,+ #[serde(default)]+ forward_auth: bool,+ #[serde(default)]+ extra_headers: BTreeMap<String, String>,+ #[serde(default = "default_max_retries")]+ max_retries: u32,+}++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+struct TargetConfig {+ id: ModelId,+ llm_client: String,+ #[serde(default)]+ extra_body: BTreeMap<String, Value>,+}++#[derive(Clone, Copy, Debug, Deserialize)]+enum ClientFormat {+ #[serde(rename = "openai_chat")]+ OpenAiChat,+ #[serde(rename = "openai_responses")]+ OpenAiResponses,+ #[serde(rename = "anthropic_messages")]+ AnthropicMessages,+}++impl ClientFormat {+ const fn wire_format(self) -> WireFormat {+ match self {+ Self::OpenAiChat => WireFormat::OpenAiChat,+ Self::OpenAiResponses => WireFormat::OpenAiResponses,+ Self::AnthropicMessages => WireFormat::AnthropicMessages,+ }+ }++ const fn caller_auth_kind(self) -> CallerAuthKind {+ match self {+ Self::AnthropicMessages => CallerAuthKind::Anthropic,+ Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi,+ }+ }+}+fn build_backend(+ client_name: &str,+ config: &LlmClientConfig,+ extra_body: &BTreeMap<String, Value>,+) -> RunnerResult<Backend> {+ if config.max_retries > MAX_CONFIGURED_RETRIES {+ return Err(RunnerError::configuration(format!(+ "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"+ )));+ }+ if config.forward_auth && config.api_key_env.is_some() {+ return Err(RunnerError::configuration(format!(+ "llm client {client_name} cannot set both forward_auth and api_key_env"+ )));+ }+ let api_key = config+ .api_key_env+ .as_deref()+ .map(|variable| {+ if variable.trim().is_empty() {+ return Err(RunnerError::configuration(format!(+ "llm client {client_name} api_key_env must not be empty"+ )));+ }+ let api_key = std::env::var(variable).map_err(|error| {+ RunnerError::configuration(format!(+ "llm client {client_name} could not read api_key_env {variable}: {error}"+ ))+ })?;+ if api_key.trim().is_empty() {+ return Err(RunnerError::configuration(format!(+ "llm client {client_name} api_key_env {variable} is empty"+ )));+ }+ Ok(api_key)+ })+ .transpose()?;+ let http = HttpBackendConfig {+ base_url: config.base_url.as_str().to_string(),+ api_key,+ forward_auth: config.forward_auth,+ extra_headers: config.extra_headers.clone(),+ extra_body: extra_body.clone(),+ max_retries: config.max_retries,+ };+ let backend = match config.format {+ ClientFormat::OpenAiChat => Backend::OpenAiChat(http),+ ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),+ ClientFormat::AnthropicMessages => Backend::Anthropic(http),+ };+ Ok(backend)+}++// A function so that serde default can use it.+const fn default_max_retries() -> u32 {+ DEFAULT_MAX_RETRIES+}++fn validate_value(label: &str, value: &str) -> RunnerResult<()> {+ if value.trim().is_empty() || value.trim() != value {+ return Err(RunnerError::configuration(format!(+ "{label} must be non-empty and have no surrounding whitespace"+ )));+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn route_presentation_fields_are_split_from_the_algorithm() {+ let route: RouteConfig = toml::from_str(+ r#"+type = "random"+id = "switchyard/random"+context_window = 128000+tool_calling = true+reasoning = false+targets = ["fast", "strong"]+weights = [1.0, 2.0]+seed = 7+"#,+ )+ .expect("route should deserialize");++ assert_eq!(route.id, "switchyard/random");+ assert_eq!(route.context_window, Some(128_000));+ assert_eq!(route.tool_calling, Some(true));+ assert_eq!(route.reasoning, Some(false));+ assert_eq!(route.algorithm.routing_target_names(), ["fast", "strong"]);+ }++ #[test]+ fn route_algorithm_unknown_fields_are_rejected() {+ let error = toml::from_str::<RouteConfig>(+ r#"+type = "passthrough"+id = "switchyard/fast"+target = "fast"+bogus = true+"#,+ )+ .expect_err("unknown route keys must be rejected");++ assert!(error.to_string().contains("bogus"), "{error}");+ }++ #[test]+ fn deployment_errors_keep_one_path_context() {+ let path = Path::new("/definitely/missing/switchyard-routes.toml");+ let error = match load_runner(path) {+ Ok(_) => panic!("missing config should fail"),+ Err(error) => error,+ };+ let message = error.to_string();++ assert_eq!(message.matches(&path.display().to_string()).count(), 1);+ assert!(message.starts_with("failed to read server config"));+ }+}+#[cfg(test)]+mod deployment_tests {+ use super::*;+ use serde_json::json;++ const VALID_CONFIG: &str = r#"+schema_version = 1++[llm_clients.primary]+format = "openai_chat"+base_url = "https://example.test/v1"++[llm_clients.responses]+format = "openai_responses"+base_url = "https://example.test/v1"++[llm_clients.anthropic]+format = "anthropic_messages"+base_url = "https://example.test"++[targets.classifier]+id = "classifier/model"+llm_client = "primary"++[targets.strong]+id = "strong/model"+llm_client = "responses"++[targets.weak]+id = "weak/model"+llm_client = "anthropic"++[routes.noop]+id = "switchyard/noop"+type = "noop"++[routes.random]+id = "switchyard/random"+type = "random"+targets = ["strong", "weak"]++[routes.classifier]+id = "switchyard/classifier"+type = "llm_classifier"+classifier_target = "classifier"+strong_target = "strong"+weak_target = "weak"+base_threshold = 0.5++[routes.passthrough]+id = "switchyard/passthrough"+type = "passthrough"+target = "weak"+"#;++ fn error_message(toml: &str) -> String {+ match runner_from_toml(toml) {+ Ok(_) => "configuration unexpectedly succeeded".to_string(),+ Err(error) => error.to_string(),+ }+ }++ fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {+ let mut configured = config.to_string();+ configured.push_str(&format!("\n[routes.{route}.subagents]\n"));+ configured.push_str(+ r#"type = "llm_classifier"+mode = "custom"+classifier_target = "classifier"+targets = ["strong", "weak"]+default_target = "weak"+prompt = "Select a target for this delegated task."+response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}'+policy = { type = "target_selector", selector = "/target" }+classify_trigger = "new_session""#,+ );+ configured.push_str(extra);+ configured+ }++ fn with_subagent_passthrough(config: &str, route: &str) -> String {+ format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")+ }++ fn stage_config() -> String {+ format!(+ r#"{VALID_CONFIG}+[targets.stage_judge]+id = "stage-judge/model"+llm_client = "primary"++[routes.stage]+id = "switchyard/stage"+type = "stage_router"+capable_target = "strong"+efficient_target = "weak"+picker = "efficient_first"+confidence_threshold = 1.0++[routes.stage.classifier]+target = "stage_judge"+base_threshold = 0.5+"#+ )+ }++ #[test]+ fn builds_all_supported_algorithm_types() -> RunnerResult<()> {+ let state = runner_from_toml(VALID_CONFIG)?;+ // The model id array is sorted alphabetically+ assert_eq!(+ state+ .models()+ .map(|model| model.id.as_str())+ .collect::<Vec<_>>(),+ [+ "switchyard/classifier",+ "switchyard/noop",+ "switchyard/passthrough",+ "switchyard/random",+ ]+ );+ Ok(())+ }++ #[test]+ fn passthrough_and_stage_accept_subagent_routing() -> RunnerResult<()> {+ let stage = stage_config();+ let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", "");+ let parsed: DeploymentConfig = toml::from_str(&stage_with_classifier).map_err(|error| {+ RunnerError::configuration(format!("failed to parse stage config: {error}"))+ })?;+ let Some(stage_route) = parsed.routes.get("stage") else {+ return Err(RunnerError::configuration("stage route is missing"));+ };+ let callable_targets = stage_route.callable_target_names();+ for expected in ["strong", "weak", "stage_judge", "classifier"] {+ assert!(callable_targets.contains(&expected));+ }++ for configured in [+ with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""),+ with_subagent_passthrough(VALID_CONFIG, "passthrough"),+ stage_with_classifier,+ with_subagent_passthrough(&stage, "stage"),+ ] {+ runner_from_toml(&configured)?;+ }+ Ok(())+ }++ #[test]+ fn rejects_invalid_unreferenced_llm_client() {+ let invalid = format!(+ "{VALID_CONFIG}\n\+ [llm_clients.unused]\n\+ format = \"openai_chat\"\n\+ base_url = \"not a url\"\n"+ );+ let message = error_message(&invalid);+ assert!(+ message.contains("base_url must be an absolute HTTP(S) URL"),+ "unexpected error: {message}"+ );+ }++ #[test]+ fn an_escalation_table_switches_the_classifier_route_to_escalation() -> RunnerResult<()> {+ // Present: the classifier target judges the weak tier's reply each turn instead of+ // picking a tier ahead of it. The route builds either way, so the assertion is that+ // the knob parses and its settings reach the algorithm's validation.+ let escalating = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nescalation = { confirmations = 2 }",+ );+ runner_from_toml(&escalating)?;++ // A setting that would starve the judge is rejected here rather than on the first+ // request, the same as any other unusable route configuration.+ let starved = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nescalation = { confirmations = 0 }",+ );+ assert!(error_message(&starved).contains("confirmations must be at least 1"));+ Ok(())+ }++ #[test]+ fn classifier_judge_completion_caps_are_configurable() -> RunnerResult<()> {+ let capability = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 512",+ );+ runner_from_toml(&capability)?;++ let escalation = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",+ );+ runner_from_toml(&escalation)?;+ Ok(())+ }++ #[test]+ fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> {+ let capability = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nprompt = \"custom capability rubric\"",+ );+ runner_from_toml(&capability)?;++ let escalation = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }",+ );+ runner_from_toml(&escalation)?;++ let empty = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nprompt = \" \"",+ );+ assert!(error_message(&empty).contains("classifier prompt must not be empty"));++ let schema_placeholder = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",+ );+ assert!(+ error_message(&schema_placeholder)+ .contains("Switchyard supplies the schema automatically")+ );+ Ok(())+ }++ #[test]+ fn mode_custom_rejects_capability_fields() {+ let mixed = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "mode = \"custom\"\nbase_threshold = 0.5",+ );++ assert!(+ error_message(&mixed)+ .contains("mode custom cannot use capability or escalation fields")+ );+ }++ #[test]+ fn rejects_unknown_fields_and_algorithm_types() {+ let unknown_field =+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");+ assert!(error_message(&unknown_field).contains("unknown field"));++ let nested_completion_cap = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",+ );+ assert!(error_message(&nested_completion_cap).contains("unknown field"));++ let unknown_classifier_field = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nclassifier_magic = true",+ );+ assert!(error_message(&unknown_classifier_field).contains("unknown field"));++ let target_capability = VALID_CONFIG.replace(+ "llm_client = \"responses\"",+ "llm_client = \"responses\"\ncontext_window = 1000000",+ );+ assert!(error_message(&target_capability).contains("unknown field `context_window`"));++ let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");+ assert!(error_message(&unknown_algorithm).contains("unknown variant"));+ }++ #[test]+ fn rejects_unknown_stage_classifier_fields() {+ // Nested classifier typos must fail instead of silently using a default.+ let config = format!(+ r#"{VALID_CONFIG}++[routes.stage]+id = "switchyard/stage"+type = "stage_router"+capable_target = "strong"+efficient_target = "weak"+picker = "efficient_first"+confidence_threshold = 1.0++[routes.stage.classifier]+target = "classifier"+base_threshold = 0.5+classifier_magic = true+"#+ );++ let error = error_message(&config);+ assert!(+ error.contains("unknown field `classifier_magic`"),+ "{error}"+ );+ }++ #[test]+ fn rejects_invalid_references_and_parameters() {+ let cases = [+ (+ VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),+ "unknown llm client missing",+ ),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"missing\"]",+ ),+ "unknown target missing",+ ),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"strong\"]",+ ),+ "random targets must be unique",+ ),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"weak\"]\nweights = [1]",+ ),+ "expected 2 weights, got 1",+ ),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",+ ),+ "at least one weight must be positive",+ ),+ (+ VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),+ "base_threshold must be between 0 and 1",+ ),+ (+ VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""),+ "route references unknown target",+ ),+ (+ VALID_CONFIG.replace(+ "classifier_target = \"classifier\"",+ "classifier_target = \"\"",+ ),+ "route references unknown target",+ ),+ (+ VALID_CONFIG.replace(+ "classifier_target = \"classifier\"",+ "classifier_target = \" \"",+ ),+ "route references unknown target",+ ),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nthreshold_step = -0.1",+ ),+ "threshold_step must be finite and greater than or equal to 0",+ ),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.8\nthreshold_step = 0.11",+ ),+ "base_threshold + 2 * threshold_step must be at most 1",+ ),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",+ ),+ "max_output_tokens must be at least 1",+ ),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nmessage_hash_fallback = true",+ ),+ "message_hash_fallback requires classify_trigger = new_session",+ ),+ (+ with_subagent_llm_classifier(+ VALID_CONFIG,+ "passthrough",+ "\nmessage_hash_fallback = true",+ ),+ "cannot use message_hash_fallback",+ ),+ (+ with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "")+ .replace("mode = \"custom\"", "mode = \"capability\""),+ "mode capability cannot use custom classifier fields",+ ),+ (+ VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"",+ ),+ "mode escalation cannot use classify_trigger",+ ),+ (+ VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),+ "unsupported schema_version 2",+ ),+ (+ VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),+ "target name must be non-empty and have no surrounding whitespace",+ ),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"weak\"]\ncontext_window = 0",+ ),+ "route random context_window must be greater than zero",+ ),+ ];++ for (toml, expected) in cases {+ let error = error_message(&toml);+ assert!(+ error.contains(expected),+ "expected error containing {expected}, got {error}"+ );+ }+ }++ #[test]+ fn accepts_duplicate_target_model_ids_on_one_client() -> RunnerResult<()> {+ // Two targets share one model id on one client. The client keeps one and drops the+ // other, so the build warns and still succeeds, and both routes resolve. Serving one+ // model under two route names this way is allowed; pointing both routes at one target+ // is the tidier form.+ const SAME_MODEL_TWO_ROUTES: &str = r#"+schema_version = 1++[llm_clients.primary]+format = "openai_chat"+base_url = "https://example.test/v1"++[targets.fast]+id = "gpt-4o"+llm_client = "primary"++[targets.smart]+id = "gpt-4o"+llm_client = "primary"++[routes.fast]+id = "switchyard/fast"+type = "passthrough"+target = "fast"++[routes.smart]+id = "switchyard/smart"+type = "passthrough"+target = "smart"+"#;+ let state = runner_from_toml(SAME_MODEL_TWO_ROUTES)?;+ assert_eq!(+ state+ .models()+ .map(|model| model.id.as_str())+ .collect::<Vec<_>>(),+ ["switchyard/fast", "switchyard/smart"]+ );+ Ok(())+ }++ #[test]+ fn accepts_same_model_id_on_different_llm_clients() -> RunnerResult<()> {+ // The same model id served by two llm clients never collides (each client keys its own+ // models), so cross-provider A/B builds with no warning; only a repeat within one client+ // warns.+ const CROSS_PROVIDER: &str = r#"+schema_version = 1++[llm_clients.openai]+format = "openai_chat"+base_url = "https://example.test/v1"++[llm_clients.azure]+format = "openai_chat"+base_url = "https://azure.test/v1"++[targets.openai]+id = "gpt-4o"+llm_client = "openai"++[targets.azure]+id = "gpt-4o"+llm_client = "azure"++[routes.openai]+id = "switchyard/openai-gpt4o"+type = "passthrough"+target = "openai"++[routes.azure]+id = "switchyard/azure-gpt4o"+type = "passthrough"+target = "azure"+"#;+ runner_from_toml(CROSS_PROVIDER)?;+ Ok(())+ }++ #[test]+ fn accepts_relative_weights_and_seed() -> RunnerResult<()> {+ let weighted = VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",+ );+ runner_from_toml(&weighted)?;+ Ok(())+ }++ #[test]+ fn accepts_new_session_trigger_with_message_hash_fallback() -> RunnerResult<()> {+ let configured = VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true",+ );+ runner_from_toml(&configured)?;+ Ok(())+ }++ #[test]+ fn target_extra_body_is_parsed_and_applied_to_its_backend() -> RunnerResult<()> {+ let configured = VALID_CONFIG.replacen(+ "llm_client = \"primary\"",+ "llm_client = \"primary\"\n\+ extra_body = { service_tier = \"priority\", \+ chat_template_kwargs = { enable_thinking = false } }",+ 1,+ );+ let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| {+ RunnerError::configuration(format!("failed to parse config: {error}"))+ })?;+ let Some(target) = config.targets.get("classifier") else {+ return Err(RunnerError::configuration("classifier target is missing"));+ };+ let Some(client) = config.llm_clients.get("primary") else {+ return Err(RunnerError::configuration("primary llm client is missing"));+ };+ let backend = build_backend("primary", client, &target.extra_body)?;++ assert_eq!(+ backend.extra_body().get("service_tier"),+ Some(&json!("priority"))+ );+ assert_eq!(+ backend+ .extra_body()+ .get("chat_template_kwargs")+ .and_then(|value| value.get("enable_thinking")),+ Some(&json!(false))+ );+ Ok(())+ }++ #[test]+ fn retry_budget_defaults_and_accepts_an_override() -> RunnerResult<()> {+ let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| {+ RunnerError::configuration(format!("failed to parse default config: {error}"))+ })?;+ let Some(primary) = default.llm_clients.get("primary") else {+ return Err(RunnerError::configuration("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);++ let explicit = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = 0",+ 1,+ );+ let config: DeploymentConfig = toml::from_str(&explicit).map_err(|error| {+ RunnerError::configuration(format!("failed to parse explicit retry config: {error}"))+ })?;+ let Some(primary) = config.llm_clients.get("primary") else {+ return Err(RunnerError::configuration("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, 0);++ let maximum = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ &format!(+ "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"+ ),+ 1,+ );+ let config: DeploymentConfig = toml::from_str(&maximum).map_err(|error| {+ RunnerError::configuration(format!("failed to parse maximum retry config: {error}"))+ })?;+ let Some(primary) = config.llm_clients.get("primary") else {+ return Err(RunnerError::configuration("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);+ Ok(())+ }++ #[test]+ fn rejects_headers_that_switchyard_sets() {+ let cases = [+ (+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\n\+ extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }",+ "AUTHORIZATION",+ ),+ (+ "base_url = \"https://example.test\"",+ "base_url = \"https://example.test\"\n\+ extra_headers = { \"X-Api-Key\" = \"custom-key\" }",+ "X-Api-Key",+ ),+ (+ "base_url = \"https://example.test\"",+ "base_url = \"https://example.test\"\n\+ extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }",+ "ANTHROPIC-VERSION",+ ),+ ];++ for (original, replacement, header) in cases {+ let configured = VALID_CONFIG.replacen(original, replacement, 1);+ let error = error_message(&configured);+ assert!(+ error.contains(&format!("extra_headers cannot set {header:?}")),+ "expected {header} to be rejected, got: {error}"+ );+ }+ }++ #[test]+ fn accepts_additional_headers() -> RunnerResult<()> {+ let configured = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\n\+ extra_headers = { X-Inference-Priority = \"batch\" }",+ 1,+ );++ runner_from_toml(&configured)?;+ Ok(())+ }++ #[test]+ fn retry_budget_rejects_negative_values() {+ let invalid = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = -1",+ 1,+ );+ assert!(error_message(&invalid).contains("max_retries"));+ }++ #[test]+ fn retry_budget_rejects_excessive_values() {+ let invalid = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = 11",+ 1,+ );+ assert!(+ error_message(&invalid).contains("llm client primary max_retries must be at most 10")+ );+ }++ #[test]+ fn api_key_environment_reference_is_validated() {+ let missing = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",+ 1,+ );+ assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));++ const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";+ unsafe {+ // "unsafe" is for concurrent reads and writes, very rare+ std::env::set_var(EMPTY_KEY_ENV, "");+ }+ let empty = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),+ 1,+ );+ let message = error_message(&empty);+ unsafe {+ std::env::remove_var(EMPTY_KEY_ENV);+ }+ assert!(message.contains("is empty"));+ }++ #[test]+ fn forward_auth_rejects_conflicting_credentials() {+ let competing_auth = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\n\+ forward_auth = true\n\+ api_key_env = \"UNUSED_TEST_KEY\"",+ 1,+ );+ assert!(+ error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env")+ );++ let static_auth = VALID_CONFIG.replacen(+ "base_url = \"https://example.test\"",+ "base_url = \"https://example.test\"\n\+ forward_auth = true\n\+ extra_headers = { Authorization = \"static-value\" }",+ 1,+ );+ assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\""));++ let static_beta = static_auth.replace("Authorization", "anthropic-beta");+ assert!(+ error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"")+ );++ for header in ["chatgpt-account-id", "x-openai-fedramp"] {+ let static_context = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ &format!(+ "base_url = \"https://example.test/v1\"\n\+ forward_auth = true\n\+ extra_headers = {{ \"{header}\" = \"static-value\" }}"+ ),+ 1,+ );+ assert!(+ error_message(&static_context)+ .contains(&format!("extra_headers cannot set \"{header}\""))+ );+ }+ }++ const ADVISOR_CONFIG: &str = r#"+schema_version = 1++[llm_clients.anthropic]+format = "anthropic_messages"+base_url = "https://example.test"++[targets.executor]+id = "executor/model"+llm_client = "anthropic"++[targets.advisor]+id = "advisor/model"+llm_client = "anthropic"++[routes.gated]+id = "switchyard/advisor"+type = "advisor"+executor_target = "executor"+advisor_target = "advisor"+"#;++ #[test]+ fn advisor_route_parses_with_defaults_and_builds() -> RunnerResult<()> {+ let state = runner_from_toml(ADVISOR_CONFIG)?;+ assert_eq!(+ state+ .models()+ .map(|model| model.id.as_str())+ .collect::<Vec<_>>(),+ ["switchyard/advisor"]+ );+ Ok(())+ }++ #[test]+ fn advisor_route_accepts_every_gate_knob() -> RunnerResult<()> {+ let tuned = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ concat!(+ "advisor_target = \"advisor\"\n",+ "reviewer_system_prompt = \"review it\"\n",+ "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",+ "gate_trigger = \"pattern\"\n",+ "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",+ "max_reviews = 2\n",+ "gate_stall_turns = 40\n",+ "gate_min_tool_results = 1\n",+ "advisor_max_tokens = 1024\n",+ "advisor_temperature = 0.0\n",+ "transcript_max_chars = 100000\n",+ "fail_open = false\n",+ "context_window = 200000\n",+ "tool_calling = true\n",+ "reasoning = true",+ ),+ );+ runner_from_toml(&tuned)?;+ Ok(())+ }++ #[test]+ fn advisor_route_rejects_unknown_keys() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\nbogus_field = 1",+ );+ assert!(error_message(&invalid).contains("bogus_field"));+ }++ #[test]+ fn advisor_route_requires_both_targets() {+ let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");+ assert!(error_message(&missing).contains("advisor_target"));+ }++ #[test]+ fn advisor_route_rejects_unknown_target() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"missing\"",+ );+ assert!(error_message(&invalid).contains("missing"));+ }++ #[test]+ fn advisor_route_rejects_invalid_pattern() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",+ );+ assert!(error_message(&invalid).contains("not a valid regex"));+ }++ #[test]+ fn advisor_route_rejects_pattern_without_pattern_trigger() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",+ );+ assert!(+ error_message(&invalid)+ .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")+ );+ }++ #[test]+ fn advisor_route_pattern_trigger_requires_pattern() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",+ );+ assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));+ }++ #[test]+ fn advisor_route_rejects_zero_max_reviews() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\nmax_reviews = 0",+ );+ assert!(error_message(&invalid).contains("max_reviews must be at least 1"));+ }+}diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rsnew file mode 100644index 000000000..d69f2da01--- /dev/null+++ b/crates/switchyard-runner/src/lib.rs@@ -0,0 +1,18 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Shared configured routing for Switchyard serving surfaces.++mod algorithm;+mod config;+mod route;+mod runner;++pub use algorithm::{+ AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, ClassifierMode,+ ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig,+};+pub use route::{+ CallerAuthKind, CountTokensTarget, ModelCapabilities, Route, RunOutput, RunnerError,+};+pub use runner::{DecisionDescription, DecisionTarget, ModelInfo, Runner};diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rsnew file mode 100644index 000000000..dc4db13ee--- /dev/null+++ b/crates/switchyard-runner/src/route.rs@@ -0,0 +1,263 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! One configured algorithm and the clients that serve its targets.++use std::error::Error;+use std::sync::Arc;++use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};+use serde_json::Value;+use switchyard_llm_client::{ClientRouter, RunObserver, TranslatingLlmClient};+use switchyard_protocol::{LlmClientError, ModelId, Request, Response, WireFormat};+use thiserror::Error;++use crate::DecisionTarget;++/// Capabilities that one route advertises on `GET /v1/models`.+///+/// An unset capability is undeclared: it serializes as `null` in the OpenAI+/// `data` entry, and the Codex entry falls back to a safe default for it.+#[derive(Clone, Copy, Default)]+pub struct ModelCapabilities {+ pub context_window: Option<u32>,+ pub tool_calling: Option<bool>,+ /// Whether the routed model takes reasoning controls. A serving surface cannot+ /// probe this, so a route opts in via config; undeclared routes advertise as+ /// non-reasoning to Codex (fail closed).+ pub reasoning: Option<bool>,+}++/// Caller credential family required by a forwarded-auth route.+#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum CallerAuthKind {+ Anthropic,+ OpenAi,+}++impl CallerAuthKind {+ /// Stable provider name used by the server compatibility API.+ pub const fn as_str(self) -> &'static str {+ match self {+ Self::Anthropic => "anthropic",+ Self::OpenAi => "openai",+ }+ }++ const fn accepts(self, wire_format: WireFormat) -> bool {+ matches!(+ (self, wire_format),+ (Self::Anthropic, WireFormat::AnthropicMessages)+ | (+ Self::OpenAi,+ WireFormat::OpenAiChat | WireFormat::OpenAiResponses+ )+ )+ }+}++/// Exact upstream model used for Anthropic token counting.+#[derive(Clone)]+pub struct CountTokensTarget {+ pub model: ModelId,+ pub client: Arc<TranslatingLlmClient>,+}++/// Error returned while loading or executing configured routes.+#[derive(Debug, Error)]+pub enum RunnerError {+ #[error("{message}")]+ Configuration {+ message: String,+ #[source]+ source: Option<Box<dyn Error + Send + Sync>>,+ },+ #[error("unknown route model {0:?}")]+ UnknownRouteModel(String),+ #[error("caller format is incompatible with {} credentials", .0.as_str())]+ IncompatibleCallerFormat(CallerAuthKind),+ #[error("route has no Anthropic target for token counting")]+ CountTokensUnsupported,+ #[error(transparent)]+ Algorithm(#[from] LibsyError),+ #[error(transparent)]+ Client(#[from] LlmClientError),+}++impl RunnerError {+ pub(crate) fn configuration(message: impl Into<String>) -> Self {+ Self::Configuration {+ message: message.into(),+ source: None,+ }+ }++ pub(crate) fn configuration_source(+ message: impl Into<String>,+ source: impl Error + Send + Sync + 'static,+ ) -> Self {+ Self::Configuration {+ message: message.into(),+ source: Some(Box::new(source)),+ }+ }+}++/// A configured algorithm and the per-target clients its calls resolve through.+pub struct Route {+ algorithm: Arc<dyn Algorithm>,+ // Resolves each offloaded call to the client configured for the target the algorithm+ // selected. A route is a synthetic model with no upstream of its own, so this is a+ // per-target lookup, never one client serving the whole route.+ clients: ClientRouter,+ caller_auth: Option<CallerAuthKind>,+ capabilities: ModelCapabilities,+ count_tokens_target: Option<CountTokensTarget>,+ decision_targets: Vec<DecisionTarget>,+}++/// The selected model and untouched response produced by a route execution.+pub struct RunOutput {+ pub selected_model: ModelId,+ pub response: Response,+}++impl Route {+ /// Creates a fully configured execution route.+ pub fn new(+ algorithm: Arc<dyn Algorithm>,+ clients: ClientRouter,+ caller_auth: Option<CallerAuthKind>,+ capabilities: ModelCapabilities,+ count_tokens_target: Option<CountTokensTarget>,+ decision_targets: Vec<DecisionTarget>,+ ) -> Self {+ Self {+ algorithm,+ clients,+ caller_auth,+ capabilities,+ count_tokens_target,+ decision_targets,+ }+ }++ /// Returns the configured libsy algorithm name.+ pub fn algorithm_name(&self) -> &str {+ self.algorithm.name()+ }++ /// Returns model-list capability metadata.+ pub fn capabilities(&self) -> ModelCapabilities {+ self.capabilities+ }++ /// Returns the forwarded caller credential family.+ pub fn caller_auth(&self) -> Option<CallerAuthKind> {+ self.caller_auth+ }++ /// Resolves a selected model to this route's non-secret target metadata.+ pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {+ self.decision_targets+ .iter()+ .find(|target| target.model == *model)+ .cloned()+ }++ /// Rejects a caller format incompatible with forwarded credentials.+ pub fn check_caller_format(&self, input_format: WireFormat) -> Result<(), RunnerError> {+ if let Some(kind) = self.caller_auth+ && !kind.accepts(input_format)+ {+ return Err(RunnerError::IncompatibleCallerFormat(kind));+ }+ Ok(())+ }++ /// Executes the configured route without consuming or proxying streamed responses.+ pub async fn execute(+ &self,+ request: Request,+ observer: Option<RunObserver>,+ ) -> Result<RunOutput, RunnerError> {+ let (selected_model, response) = switchyard_llm_client::run(+ Arc::clone(&self.algorithm),+ self.clients.clone(),+ request,+ observer,+ )+ .await?;+ Ok(RunOutput {+ selected_model,+ response,+ })+ }++ /// Completes routing-time calls without serving the answer target.+ pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {+ drive(Arc::clone(&self.algorithm), request, |call| {+ serve_decision_dependency(self.clients.clone(), call)+ })+ .await+ .map_err(Into::into)+ }++ /// Counts tokens using the configured Anthropic-capable target.+ pub async fn count_tokens(&self, request: Request) -> Result<Value, RunnerError> {+ let target = self+ .count_tokens_target+ .as_ref()+ .ok_or(RunnerError::CountTokensUnsupported)?;+ target+ .client+ .count_tokens(&target.model, request)+ .await+ .map_err(Into::into)+ }+}++async fn serve_decision_dependency(clients: ClientRouter, call: CallModel) -> libsy::Result<()> {+ let mut result = Err(LibsyError::NoTargets);+ for (index, model) in call.models.iter().enumerate() {+ // The driver stamps only the first candidate, so every fallback must replace it.+ let mut request = call.request.clone();+ request.llm_request.model = Some(model.to_string());+ let response = match clients.route(model) {+ Ok(client) => client.call(request).await,+ Err(source) => Err(source),+ };+ match response {+ Ok(response) => {+ result = Ok(response);+ break;+ }+ Err(source) => {+ let try_next = index + 1 < call.models.len() && eligible_routing_fallback(&source);+ result = Err(LibsyError::client_call(model.clone(), source));+ if !try_next {+ break;+ }+ }+ }+ }+ call.respond(result)+}++/// Whether a routing-time candidate failure may fall through to the next model.+fn eligible_routing_fallback(error: &LlmClientError) -> bool {+ match error {+ LlmClientError::ContextWindowExceeded { .. }+ | LlmClientError::Transport { .. }+ | LlmClientError::Timeout { .. } => true,+ LlmClientError::UpstreamHttp { status, .. } => {+ matches!(+ *status,+ reqwest::StatusCode::FORBIDDEN+ | reqwest::StatusCode::REQUEST_TIMEOUT+ | reqwest::StatusCode::TOO_MANY_REQUESTS+ ) || status.is_server_error()+ }+ _ => false,+ }+}diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rsnew file mode 100644index 000000000..55760f3a3--- /dev/null+++ b/crates/switchyard-runner/src/runner.rs@@ -0,0 +1,90 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Named route table and server-facing route metadata.++use std::collections::BTreeMap;+use std::path::Path;++use libsy::RoutingOutcome;+use serde_json::Value;+use switchyard_protocol::{ModelId, WireFormat};++use crate::config;+use crate::{ModelCapabilities, Route, RunnerError};++/// Immutable named route table.+pub struct Runner {+ routes: Vec<(ModelId, Route)>,+}++/// Borrowed model metadata returned while listing routes.+pub struct ModelInfo<'a> {+ pub id: &'a ModelId,+ pub algorithm: &'a str,+ pub capabilities: ModelCapabilities,+}++/// Fully resolved routing decision.+pub struct DecisionDescription {+ pub selected: DecisionTarget,+ pub fallbacks: Vec<DecisionTarget>,+}++/// Non-secret configured target details returned by the decision endpoint.+#[derive(Clone)]+pub struct DecisionTarget {+ pub target: String,+ pub model: ModelId,+ pub format: WireFormat,+ pub base_url: String,+ pub extra_body: BTreeMap<String, Value>,+}++impl Runner {+ /// Loads and validates a version-1 deployment TOML file.+ pub fn load(path: impl AsRef<Path>) -> Result<Self, RunnerError> {+ config::load_runner(path)+ }++ /// Builds a runner from named routes in caller-provided order.+ /// Pre-condition: There must be at least one route.+ pub fn new(routes: Vec<(ModelId, Route)>) -> Self {+ Self { routes }+ }++ /// Returns the route registered for a model.+ pub fn route(&self, model: &str) -> Option<&Route> {+ self.routes+ .iter()+ .find(|(id, _)| id.as_str() == model)+ .map(|(_, route)| route)+ }++ /// Iterates over configured routes in caller-provided order.+ pub fn models(&self) -> impl Iterator<Item = ModelInfo<'_>> {+ self.routes.iter().map(|(id, route)| ModelInfo {+ id,+ algorithm: route.algorithm_name(),+ capabilities: route.capabilities(),+ })+ }++ /// Resolves an outcome to configured target names and non-secret client settings.+ pub fn describe_decision(+ &self,+ model: &ModelId,+ outcome: &RoutingOutcome,+ ) -> Option<DecisionDescription> {+ let route = self.route(model.as_str())?;+ let resolve = |selected: &ModelId| route.decision_target(selected);+ Some(DecisionDescription {+ selected: resolve(&outcome.selected_model_id)?,+ fallbacks: outcome+ .fallback_models+ .iter()+ .map(resolve)+ .collect::<Option<Vec<_>>>()?,+ })+ }+}diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rsnew file mode 100644index 000000000..71ce57c7e--- /dev/null+++ b/crates/switchyard-runner/tests/route.rs@@ -0,0 +1,164 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::collections::BTreeMap;+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};++use async_trait::async_trait;+use futures_util::StreamExt;+use switchyard_llm_client::{ClientRouter, RunObservation};+use switchyard_protocol::{+ LlmClientError, LlmResponse, ModelId, Request, Response, RoutedLlmClient, text_request,+ text_response,+};+use switchyard_runner::{AlgorithmSpec, ModelCapabilities, Route};++struct StubClient;++#[async_trait]+impl RoutedLlmClient for StubClient {+ async fn call(&self, request: Request) -> Result<Response, LlmClientError> {+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ request.llm_request.model.clone(),+ "plugin response",+ )),+ metadata: None,+ })+ }+}++fn plugin_route(client: Arc<dyn RoutedLlmClient>) -> Route {+ let spec = AlgorithmSpec::Passthrough {+ target: "semantic-target".to_string(),+ subagents: None,+ };+ let targets = BTreeMap::from([(+ "semantic-target".to_string(),+ ModelId::from("semantic-target"),+ )]);+ let algorithm = spec+ .build("switchyard", &targets)+ .expect("identity target map should build");+ let clients = ClientRouter::new(+ BTreeMap::from([(ModelId::from("semantic-target"), client)])+ .into_iter()+ .collect(),+ );+ Route::new(+ algorithm,+ clients,+ None,+ ModelCapabilities::default(),+ None,+ Vec::new(),+ )+}++#[tokio::test]+async fn plugin_shaped_route_executes_without_runner_model_or_toml() {+ let route = plugin_route(Arc::new(StubClient));+ let observations = Arc::new(Mutex::new(Vec::new()));+ let observer = {+ let observations = Arc::clone(&observations);+ Arc::new(move |observation| observations.lock().unwrap().push(observation))+ };+ let request = Request {+ llm_request: text_request(Some("arbitrary-upstream-model".to_string()), "hello"),+ ..Request::default()+ };++ let output = route+ .execute(request, Some(observer))+ .await+ .expect("route should execute");++ assert_eq!(output.selected_model, "semantic-target");+ assert_eq!(+ output+ .response+ .llm_response+ .as_agg()+ .unwrap()+ .model+ .as_deref(),+ Some("semantic-target")+ );+ assert!(+ observations+ .lock()+ .unwrap()+ .iter()+ .any(|observation| { matches!(observation, RunObservation::AnswerCall(_)) })+ );+ assert!(+ observations+ .lock()+ .unwrap()+ .iter()+ .any(|observation| { matches!(observation, RunObservation::RoutingOverhead(_)) })+ );+}++struct LazyStreamClient {+ polls: Arc<AtomicUsize>,+}++#[async_trait]+impl RoutedLlmClient for LazyStreamClient {+ async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {+ let polls = Arc::clone(&self.polls);+ let stream = futures_util::stream::poll_fn(move |_| {+ polls.fetch_add(1, Ordering::SeqCst);+ std::task::Poll::Ready(None)+ })+ .boxed();+ Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ })+ }+}++#[tokio::test]+async fn route_returns_stream_without_polling_it() {+ let polls = Arc::new(AtomicUsize::new(0));+ let route = plugin_route(Arc::new(LazyStreamClient {+ polls: Arc::clone(&polls),+ }));+ let request = Request {+ llm_request: text_request(None, "hello"),+ ..Request::default()+ };++ let output = route+ .execute(request, None)+ .await+ .expect("stream handle should be returned");++ assert!(matches!(+ output.response.llm_response,+ LlmResponse::Stream(_)+ ));+ assert_eq!(polls.load(Ordering::SeqCst), 0);+}++#[test]+fn algorithm_build_reports_unknown_configured_target() {+ let spec = AlgorithmSpec::Random {+ targets: vec!["missing".to_string()],+ weights: None,+ seed: None,+ };++ let error = match spec.build("plugin", &BTreeMap::new()) {+ Ok(_) => panic!("unknown target should fail"),+ Err(error) => error,+ };++ assert_eq!(+ error.to_string(),+ "route plugin references unknown target missing"+ );+}diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.mdindex efd438109..ae40d38eb 100644--- a/crates/switchyard-server/CONFIGURATION.md+++ b/crates/switchyard-server/CONFIGURATION.md@@ -26,11 +26,11 @@ target only when the upstream supports it and would otherwise return the verdictoutside normal assistant `content`.To support another wire format, add its `ClientFormat` variant and explicit construction match in-`src/config.rs`. Add a client type only when a second implementation exists.+`../switchyard-runner/src/config.rs`. Add a client type only when a second implementation exists.## Add an algorithm1. Implement and export the algorithm from `libsy`.-2. Add its TOML fields as an `AlgorithmConfig` variant in `src/config.rs`.-3. Construct it in the `build_algorithm` match, resolving target names with `resolve_targets`.+2. Add its fields as an `AlgorithmSpec` variant in `../switchyard-runner/src/algorithm.rs`.+3. Construct it in that module's builder, resolving names through the caller-supplied target map.4. Add a parsing test and an end-to-end server test when the algorithm makes LLM calls.diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.tomlindex 5502a0e8c..0c4b47e24 100644--- a/crates/switchyard-server/Cargo.toml+++ b/crates/switchyard-server/Cargo.toml@@ -32,11 +32,10 @@ opentelemetry-prometheus = "0.32"opentelemetry_sdk = { version = "0.32", default-features = false, features = ["metrics", "trace"] }parking_lot.workspace = trueprometheus = "0.14"-reqwest.workspace = trueserde.workspace = true-toml = "1.1"switchyard-llm-client.workspace = trueswitchyard-protocol.workspace = true+switchyard-runner.workspace = trueserde_json.workspace = trueswitchyard-translation.workspace = truehumantime = "2.4"diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 77373959b..ba88b3445 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -1,2262 +1,15 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Typed TOML configuration and explicit construction for the Rust server.+//! Compatibility wrapper for loading shared runner configuration.-use std::collections::{BTreeMap, HashMap, HashSet};-use std::fs;use std::path::Path;-use std::sync::Arc;-use libsy::{- AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,- ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig,- GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop,- Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter,- SubagentRouterConfig, TargetPrompts, TaskClassifierConfig,-};-use serde::Deserialize;-use serde_json::Value;-use switchyard_llm_client::{- Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,- TranslatingLlmClient,-};-use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};+use switchyard_runner::Runner;-use crate::{- CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState,-};--const SUPPORTED_SCHEMA_VERSION: u32 = 1;-const MAX_CONFIGURED_RETRIES: u32 = 10;+use crate::{ServerError, ServerResult, ServerState};/// Loads a TOML deployment file and constructs the complete server state.pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {- let path = path.as_ref();- let toml = fs::read_to_string(path).map_err(|error| {- ServerError::new(format!(- "failed to read server config {}: {error}",- path.display()- ))- })?;- server_state_from_toml(&toml).map_err(|error| {- ServerError::new(format!("invalid server config {}: {error}", path.display()))- })-}--fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {- let config: Arc<ServerConfig> = Arc::new(- toml::from_str(toml)- .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?,- );- let state = config.build()?;- Ok(state.with_config(config))-}--#[derive(Debug, Deserialize)]-#[serde(deny_unknown_fields)]-pub(crate) struct ServerConfig {- schema_version: u32,- #[serde(default)]- pub(crate) llm_clients: BTreeMap<String, LlmClientConfig>,- pub(crate) targets: BTreeMap<String, TargetConfig>,- routes: BTreeMap<String, RouteConfig>,-}--impl ServerConfig {- pub(crate) fn routing_target_names(&self, route_name: &str) -> Option<Vec<&str>> {- self.routes- .get(route_name)- .map(RouteConfig::routing_target_names)- }-- fn build(&self) -> ServerResult<ServerState> {- if self.schema_version != SUPPORTED_SCHEMA_VERSION {- return Err(ServerError::new(format!(- "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}",- self.schema_version- )));- }-- // An llm client keys its models by id, so two targets that share an id on one- // client end up as one entry: the client keeps one target and drops the other. Warn- // so the drop is visible at startup instead of silent, and still build both routes.- // The set only tells us the pair was already seen, not whether the two targets- // differ, so it warns for a harmless duplicate and for one whose extra_body would be- // dropped alike. The same id on two different clients never collides: each client- // keeps its own model.- let mut seen_client_model_ids = HashSet::new();- for (target_name, target) in &self.targets {- validate_value("target name", target_name)?;- validate_value(&format!("target {target_name} id"), &target.id)?;- if !seen_client_model_ids.insert((target.llm_client.as_str(), target.id.as_str())) {- tracing::warn!(- "target {target_name} reuses model id {} on llm client {}; only one target per id is kept and the other is dropped. Give each target a unique model id, or point both routes at one target.",- target.id,- target.llm_client- );- }- }-- let clients = self.build_clients()?;- let targets = self.build_targets()?;- let mut routes = Vec::with_capacity(self.routes.len());- for (route_name, config) in &self.routes {- validate_value("route name", route_name)?;- validate_value(&format!("route {route_name} id"), config.id())?;- for target_name in config.callable_target_names() {- self.targets.get(target_name).ok_or_else(|| {- ServerError::new(format!("route references unknown target {target_name}"))- })?;- }- let capabilities = config.capabilities();- if capabilities.context_window == Some(0) {- return Err(ServerError::new(format!(- "route {route_name} context_window must be greater than zero"- )));- }- let algorithm = build_algorithm(route_name, config, &targets)?;- let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?;- let count_tokens_target = self.build_count_tokens_target(config, &clients);- routes.push((- config.id().clone(),- algorithm,- client,- caller_auth,- capabilities,- count_tokens_target,- Some(route_name.clone()),- ));- }- ServerState::new_with_capabilities(routes)- }-- fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<TranslatingLlmClient>>> {- let mut models_by_client = self- .llm_clients- .keys()- .map(|name| (name.clone(), Vec::new()))- .collect::<BTreeMap<String, Vec<ModelConfig>>>();-- // Validate every declared client even when no target currently references it.- for (name, client_config) in &self.llm_clients {- validate_value("llm client name", name)?;- build_backend(name, client_config, &BTreeMap::new())?;- }- for (target_name, target) in &self.targets {- let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {- ServerError::new(format!(- "target {target_name} references unknown llm client {}",- target.llm_client- ))- })?;- let model_configs = models_by_client- .get_mut(&target.llm_client)- .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;- model_configs.push(ModelConfig::new(- target.id.clone(),- build_backend(&target.llm_client, client_config, &target.extra_body)?,- None,- ));- }-- let mut clients = BTreeMap::new();- for (name, model_configs) in models_by_client {- let client = Arc::new(- TranslatingLlmClient::new(&model_configs)- .map_err(|error| ServerError::new(error.to_string()))?,- );- clients.insert(name, client);- }- Ok(clients)- }-- fn build_targets(&self) -> ServerResult<BTreeMap<String, ModelId>> {- Ok(self- .targets- .iter()- .map(|(name, config)| (name.clone(), config.id.clone()))- .collect())- }-- /// Maps every model this route can select to the client configured for that target.- ///- /// A route is a synthetic model (`switchyard/random`) with no upstream of its own — the- /// algorithm picks a real target, and the *target* names the `[llm_clients.*]` section- /// that serves it. Two targets in one route may therefore sit on different clients, and- /// one request can emit calls for both.- ///- /// The map is built per route because targets are only reachable through the routes that- /// name them, so the same model id may resolve to different clients in two routes without- /// colliding.- fn build_route_clients(- &self,- route_name: &str,- route: &RouteConfig,- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,- ) -> ServerResult<(ClientRouter, Option<CallerAuthKind>)> {- let mut by_model = HashMap::new();- let mut caller_auth = None;- for name in route.callable_target_names() {- let target = self.targets.get(name).ok_or_else(|| {- ServerError::new(format!("route references unknown target {name}"))- })?;- let client = clients.get(&target.llm_client).ok_or_else(|| {- ServerError::new(format!("target {name} has no constructed llm client"))- })?;- let client_config = self.llm_clients.get(&target.llm_client).ok_or_else(|| {- ServerError::new(format!(- "target {name} references unknown llm client {}",- target.llm_client- ))- })?;- if client_config.forward_auth {- let target_auth = client_config.format.caller_auth_kind();- if caller_auth.is_some_and(|kind| kind != target_auth) {- return Err(ServerError::new(format!(- "route {route_name} cannot forward both Anthropic and OpenAI caller credentials"- )));- }- caller_auth = Some(target_auth);- }- let client: Arc<dyn RoutedLlmClient> = client.clone();- by_model.insert(target.id.clone(), client);- }- Ok((ClientRouter::new(by_model), caller_auth))- }-- fn build_count_tokens_target(- &self,- route_config: &RouteConfig,- clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,- ) -> Option<CountTokensTarget> {- route_config- .routing_target_names()- .into_iter()- .enumerate()- .filter_map(|(index, name)| {- let target = self.targets.get(name)?;- let client = clients.get(&target.llm_client)?;- client.supports_count_tokens(&target.id).then_some((- count_tokens_priority(name, &target.id),- index,- target,- client,- ))- })- .min_by_key(|(priority, index, _, _)| (*priority, *index))- .map(|(_, _, target, client)| CountTokensTarget {- model: target.id.clone(),- client: client.clone(),- })- }-}--// Prefer known Claude families, then preserve the route's target order.-fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {- let target_name = target_name.to_ascii_lowercase();- let model_id = model_id.to_ascii_lowercase();- ["opus", "sonnet", "haiku"]- .iter()- .position(|hint| target_name.contains(hint) || model_id.contains(hint))- .unwrap_or(3)-}--/// A client endpoint, parsed when the config loads rather than checked afterwards.-///-/// Holding a `HttpBaseUrl` is proof the value is an absolute HTTP(S) URL, so no-/// later stage has to re-check it or can forget to.-#[derive(Clone, Debug)]-pub(crate) struct HttpBaseUrl(reqwest::Url);--impl HttpBaseUrl {- pub(crate) fn as_str(&self) -> &str {- self.0.as_str()- }-}--impl<'de> Deserialize<'de> for HttpBaseUrl {- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where- D: serde::Deserializer<'de>,- {- let raw = String::deserialize(deserializer)?;- let url = reqwest::Url::parse(raw.trim()).map_err(|error| {- serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}"))- })?;- if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() {- return Err(serde::de::Error::custom(- "base_url must be an absolute HTTP(S) URL",- ));- }- Ok(Self(url))- }-}--#[derive(Debug, Deserialize)]-#[serde(deny_unknown_fields)]-pub(crate) struct LlmClientConfig {- pub(crate) format: ClientFormat,- pub(crate) base_url: HttpBaseUrl,- api_key_env: Option<String>,- #[serde(default)]- forward_auth: bool,- #[serde(default)]- extra_headers: BTreeMap<String, String>,- #[serde(default = "default_max_retries")]- max_retries: u32,-}--#[derive(Debug, Deserialize)]-#[serde(deny_unknown_fields)]-pub(crate) struct TargetConfig {- pub(crate) id: ModelId,- pub(crate) llm_client: String,- #[serde(default)]- pub(crate) extra_body: BTreeMap<String, Value>,-}--#[derive(Clone, Copy, Debug, Deserialize)]-pub(crate) enum ClientFormat {- #[serde(rename = "openai_chat")]- OpenAiChat,- #[serde(rename = "openai_responses")]- OpenAiResponses,- #[serde(rename = "anthropic_messages")]- AnthropicMessages,-}--impl ClientFormat {- pub(crate) const fn wire_format(self) -> WireFormat {- match self {- Self::OpenAiChat => WireFormat::OpenAiChat,- Self::OpenAiResponses => WireFormat::OpenAiResponses,- Self::AnthropicMessages => WireFormat::AnthropicMessages,- }- }-- const fn caller_auth_kind(self) -> CallerAuthKind {- match self {- Self::AnthropicMessages => CallerAuthKind::Anthropic,- Self::OpenAiChat | Self::OpenAiResponses => CallerAuthKind::OpenAi,- }- }-}--#[derive(Clone, Debug, Deserialize)]-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]-enum ClassifierPolicyConfig {- TargetSelector { selector: String },-}--#[derive(Clone, Copy, Debug, Deserialize)]-#[serde(rename_all = "snake_case")]-enum ClassifierMode {- Capability,- Escalation,- Custom,-}--impl ClassifierPolicyConfig {- fn into_libsy(self) -> CustomClassifierPolicy {- match self {- Self::TargetSelector { selector } => CustomClassifierPolicy::target_selector(selector),- }- }-}--#[derive(Debug)]-enum LlmClassifierModeConfig {- Capability(CapabilityClassifierRouteConfig),- Escalation(EscalationClassifierRouteConfig),- Custom(CustomClassifierRouteConfig),-}--#[derive(Debug)]-struct CapabilityClassifierRouteConfig {- strong_target: String,- weak_target: String,- base_threshold: f64,- threshold_step: f64,- classify_trigger: ClassifyTrigger,- message_hash_fallback: bool,- recent_turn_window: Option<usize>,- prompt: Option<String>,- response_format_type: ClassifierResponseFormat,- max_output_tokens: u64,-}--#[derive(Debug)]-struct EscalationClassifierRouteConfig {- strong_target: String,- weak_target: String,- prompt: Option<String>,- response_format_type: ClassifierResponseFormat,- max_output_tokens: u64,- judge: EscalationJudgeConfig,-}--#[derive(Debug)]-struct CustomClassifierRouteConfig {- classifier_target: String,- targets: Vec<String>,- default_target: String,- prompt: String,- response_schema: String,- policy: ClassifierPolicyConfig,- classify_trigger: ClassifyTrigger,- message_hash_fallback: bool,- recent_turn_window: Option<usize>,- max_output_tokens: u64,-}--#[derive(Debug, Default, Deserialize)]-#[serde(default, deny_unknown_fields)]-struct LlmClassifierRouteConfig {- classifier_target: String,- mode: Option<ClassifierMode>,- strong_target: Option<String>,- weak_target: Option<String>,- base_threshold: Option<f64>,- threshold_step: Option<f64>,- classify_trigger: ClassifyTrigger,- message_hash_fallback: bool,- recent_turn_window: Option<usize>,- prompt: Option<String>,- response_format_type: ClassifierResponseFormat,- #[serde(default = "default_classifier_max_output_tokens")]- max_output_tokens: u64,- escalation: Option<EscalationJudgeConfig>,- targets: Option<Vec<String>>,- default_target: Option<String>,- response_schema: Option<String>,- policy: Option<ClassifierPolicyConfig>,-}--#[derive(Debug, Deserialize)]-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]-enum SubagentRouteConfig {- Passthrough { target: String },- LlmClassifier(Box<LlmClassifierRouteConfig>),-}--impl SubagentRouteConfig {- fn routing_target_names(&self) -> Vec<&str> {- match self {- Self::Passthrough { target } => vec![target],- Self::LlmClassifier(classifier) => classifier- .targets- .iter()- .flatten()- .map(String::as_str)- .collect(),- }- }-- fn classifier_target_name(&self) -> Option<&str> {- match self {- Self::LlmClassifier(classifier) => Some(&classifier.classifier_target),- Self::Passthrough { .. } => None,- }- }-}--#[derive(Debug, Deserialize)]-#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]-enum RouteConfig {- Noop {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- },- Random {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- targets: Vec<String>,- weights: Option<Vec<f64>>,- seed: Option<u64>,- },- Passthrough {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- target: String,- #[serde(default)]- subagents: Option<SubagentRouteConfig>,- },- LlmClassifier {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- #[serde(flatten)]- config: LlmClassifierRouteConfig,- },- StageRouter {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- capable_target: String,- efficient_target: String,- /// Tier a turn falls back to when the signals are not confident.- picker: PickerMode,- confidence_threshold: f64,- /// Trailing tool results the signals are computed over.- #[serde(default)]- recent_turn_window: Option<usize>,- /// Note handed to the model a signal-driven switch routes to.- #[serde(default)]- handoff_notes: Option<HandoffNoteConfig>,- /// System prompt handed to each tier on every turn it serves.- #[serde(default)]- capable_system_prompt: Option<String>,- #[serde(default)]- efficient_system_prompt: Option<String>,- /// Capability judge consulted on turns the signals leave undecided.- #[serde(default)]- classifier: Option<StageClassifierConfig>,- /// Optional routing applied only to delegated sub-agent work.- #[serde(default)]- subagents: Option<SubagentRouteConfig>,- },- Advisor {- id: ModelId,- #[serde(default)]- context_window: Option<u32>,- #[serde(default)]- tool_calling: Option<bool>,- #[serde(default)]- reasoning: Option<bool>,- /// Serves every client-visible turn; also the count_tokens target.- executor_target: String,- /// Reviews the executor's first terminal turn. Judge-only, never a- /// routing destination.- advisor_target: String,- #[serde(default)]- reviewer_system_prompt: Option<String>,- #[serde(default)]- redo_feedback_prefix: Option<String>,- #[serde(default)]- gate_trigger: AdvisorTriggerConfig,- #[serde(default)]- gate_trigger_pattern: Option<String>,- #[serde(default = "default_max_reviews")]- max_reviews: u32,- #[serde(default)]- gate_stall_turns: u32,- #[serde(default)]- gate_min_tool_results: u32,- #[serde(default = "default_advisor_max_tokens")]- advisor_max_tokens: u64,- #[serde(default)]- advisor_temperature: Option<f64>,- #[serde(default = "default_transcript_max_chars")]- transcript_max_chars: usize,- #[serde(default = "default_fail_open")]- fail_open: bool,- },-}--/// What fires an advisor route's review.-#[derive(Debug, Default, Deserialize, PartialEq)]-#[serde(rename_all = "snake_case")]-enum AdvisorTriggerConfig {- /// The executor's first turn without tool calls.- #[default]- NoToolCall,- /// The first turn whose text matches `gate_trigger_pattern`.- Pattern,-}--/// The judge a `stage_router` route falls through to, and how it routes.-#[derive(Debug, Deserialize)]-#[serde(deny_unknown_fields)]-struct StageClassifierConfig {- /// Target the judge is called through. Not a routing destination.- target: String,- base_threshold: f64,- #[serde(default)]- threshold_step: f64,- #[serde(default)]- classify_trigger: ClassifyTrigger,- #[serde(default)]- message_hash_fallback: bool,- #[serde(default)]- recent_turn_window: Option<usize>,- #[serde(default)]- prompt: Option<String>,- #[serde(default)]- response_format_type: ClassifierResponseFormat,- #[serde(default = "default_classifier_max_output_tokens")]- max_output_tokens: u64,-}--impl StageClassifierConfig {- fn task_classifier_config(&self) -> TaskClassifierConfig {- TaskClassifierConfig {- base_threshold: self.base_threshold,- threshold_step: self.threshold_step,- classify_trigger: self.classify_trigger,- message_hash_fallback: self.message_hash_fallback,- recent_turn_window: self.recent_turn_window,- contract: classifier_contract(self.prompt.as_deref())- .with_response_format_type(self.response_format_type),- max_output_tokens: self.max_output_tokens,- }- }-}--impl RouteConfig {- fn id(&self) -> &ModelId {- use RouteConfig::*;- match self {- Noop { id, .. }- | Random { id, .. }- | LlmClassifier { id, .. }- | Passthrough { id, .. }- | StageRouter { id, .. }- | Advisor { id, .. } => id,- }- }-- // Completion targets in algorithm order; judge-only targets are excluded.- fn routing_target_names(&self) -> Vec<&str> {- match self {- Self::Noop { .. } => Vec::new(),- Self::Random { targets, .. } => targets.iter().map(String::as_str).collect(),- Self::Passthrough {- target, subagents, ..- } => {- let mut names = vec![target.as_str()];- if let Some(subagents) = subagents {- names.extend(subagents.routing_target_names());- }- names- }- Self::LlmClassifier { config, .. } => {- match config.mode.unwrap_or(if config.escalation.is_some() {- ClassifierMode::Escalation- } else {- ClassifierMode::Capability- }) {- ClassifierMode::Capability => config- .weak_target- .iter()- .chain(&config.strong_target)- .map(String::as_str)- .collect(),- ClassifierMode::Escalation => config- .strong_target- .iter()- .chain(&config.weak_target)- .map(String::as_str)- .collect(),- ClassifierMode::Custom => config- .targets- .iter()- .flatten()- .map(String::as_str)- .collect(),- }- }- Self::StageRouter {- capable_target,- efficient_target,- subagents,- ..- } => {- let mut names = vec![capable_target.as_str(), efficient_target.as_str()];- if let Some(subagents) = subagents {- names.extend(subagents.routing_target_names());- }- names- }- // The advisor is judge-only: reviews go through its own client,- // so it is not a completion (or count_tokens) destination.- Self::Advisor {- executor_target, ..- } => vec![executor_target],- }- }-- /// Every target the algorithm may call, including judge-only targets.- ///- /// [`routing_target_names`](Self::routing_target_names) covers completion destinations;- /// a classifier also calls its judge, and that call needs a client too.- fn callable_target_names(&self) -> Vec<&str> {- let mut names = self.routing_target_names();- match self {- Self::LlmClassifier { config, .. } => names.push(&config.classifier_target),- Self::Passthrough {- subagents: Some(subagents),- ..- } => names.extend(subagents.classifier_target_name()),- Self::StageRouter {- classifier,- subagents,- ..- } => {- if let Some(classifier) = classifier {- names.push(&classifier.target);- }- if let Some(subagents) = subagents {- names.extend(subagents.classifier_target_name());- }- }- Self::Advisor { advisor_target, .. } => names.push(advisor_target),- _ => {}- }- names- }-- fn capabilities(&self) -> ModelCapabilities {- use RouteConfig::*;- match self {- Noop {- context_window,- tool_calling,- reasoning,- ..- }- | Random {- context_window,- tool_calling,- reasoning,- ..- }- | Passthrough {- context_window,- tool_calling,- reasoning,- ..- }- | LlmClassifier {- context_window,- tool_calling,- reasoning,- ..- }- | StageRouter {- context_window,- tool_calling,- reasoning,- ..- }- | Advisor {- context_window,- tool_calling,- reasoning,- ..- } => ModelCapabilities {- context_window: *context_window,- tool_calling: *tool_calling,- reasoning: *reasoning,- },- }- }-}--impl LlmClassifierRouteConfig {- fn classifier_mode(&self, route_name: &str) -> ServerResult<LlmClassifierModeConfig> {- let Self {- classifier_target,- mode,- strong_target,- weak_target,- base_threshold,- threshold_step,- classify_trigger,- message_hash_fallback,- recent_turn_window,- prompt,- response_format_type,- max_output_tokens,- escalation,- targets,- default_target,- response_schema,- policy,- } = self;-- let selected_mode = match (mode, escalation.is_some()) {- (Some(mode), _) => *mode,- (None, true) => ClassifierMode::Escalation,- (None, false) => ClassifierMode::Capability,- };-- match selected_mode {- ClassifierMode::Capability => {- if escalation.is_some() {- return Err(classifier_field_error(- route_name,- "escalation",- "capability",- ));- }- reject_custom_fields(- route_name,- "capability",- targets,- default_target,- response_schema,- policy,- )?;- Ok(LlmClassifierModeConfig::Capability(- CapabilityClassifierRouteConfig {- strong_target: required_classifier_field(- route_name,- "strong_target",- strong_target,- )?,- weak_target: required_classifier_field(- route_name,- "weak_target",- weak_target,- )?,- base_threshold: required_classifier_field(- route_name,- "base_threshold",- base_threshold,- )?,- threshold_step: threshold_step.unwrap_or_default(),- classify_trigger: *classify_trigger,- message_hash_fallback: *message_hash_fallback,- recent_turn_window: *recent_turn_window,- prompt: prompt.clone(),- response_format_type: *response_format_type,- max_output_tokens: *max_output_tokens,- },- ))- }- ClassifierMode::Escalation => {- reject_custom_fields(- route_name,- "escalation",- targets,- default_target,- response_schema,- policy,- )?;- if *classify_trigger != ClassifyTrigger::EveryRequest {- return Err(ServerError::new(format!(- "llm_classifier route {route_name} mode escalation cannot use classify_trigger"- )));- }- if mode.is_some()- && (base_threshold.is_some()- || threshold_step.is_some()- || *message_hash_fallback- || recent_turn_window.is_some())- {- return Err(ServerError::new(format!(- "llm_classifier route {route_name} mode escalation cannot use capability routing settings"- )));- }- Ok(LlmClassifierModeConfig::Escalation(- EscalationClassifierRouteConfig {- strong_target: required_classifier_field(- route_name,- "strong_target",- strong_target,- )?,- weak_target: required_classifier_field(- route_name,- "weak_target",- weak_target,- )?,- prompt: prompt.clone(),- response_format_type: *response_format_type,- max_output_tokens: *max_output_tokens,- judge: required_classifier_field(route_name, "escalation", escalation)?,- },- ))- }- ClassifierMode::Custom => {- if strong_target.is_some()- || weak_target.is_some()- || base_threshold.is_some()- || threshold_step.is_some()- || escalation.is_some()- || *response_format_type != ClassifierResponseFormat::JsonSchema- {- return Err(ServerError::new(format!(- "llm_classifier route {route_name} mode custom cannot use capability or escalation fields"- )));- }- Ok(LlmClassifierModeConfig::Custom(- CustomClassifierRouteConfig {- classifier_target: classifier_target.clone(),- targets: required_classifier_field(route_name, "targets", targets)?,- default_target: required_classifier_field(- route_name,- "default_target",- default_target,- )?,- prompt: required_classifier_field(route_name, "prompt", prompt)?,- response_schema: required_classifier_field(- route_name,- "response_schema",- response_schema,- )?,- policy: required_classifier_field(route_name, "policy", policy)?,- classify_trigger: *classify_trigger,- message_hash_fallback: *message_hash_fallback,- recent_turn_window: *recent_turn_window,- max_output_tokens: *max_output_tokens,- },- ))- }- }- }-}--fn reject_custom_fields(- route_name: &str,- mode: &str,- targets: &Option<Vec<String>>,- default_target: &Option<String>,- response_schema: &Option<String>,- policy: &Option<ClassifierPolicyConfig>,-) -> ServerResult<()> {- if targets.is_some()- || default_target.is_some()- || response_schema.is_some()- || policy.is_some()- {- return Err(ServerError::new(format!(- "llm_classifier route {route_name} mode {mode} cannot use custom classifier fields"- )));- }- Ok(())-}--fn classifier_field_error(route_name: &str, field: &str, mode: &str) -> ServerError {- ServerError::new(format!(- "llm_classifier route {route_name} mode {mode} cannot use {field}"- ))-}--fn required_classifier_field<T: Clone>(- route_name: &str,- field: &str,- value: &Option<T>,-) -> ServerResult<T> {- value.clone().ok_or_else(|| {- ServerError::new(format!(- "llm_classifier route {route_name} requires {field}"- ))- })-}--fn build_backend(- client_name: &str,- config: &LlmClientConfig,- extra_body: &BTreeMap<String, Value>,-) -> ServerResult<Backend> {- if config.max_retries > MAX_CONFIGURED_RETRIES {- return Err(ServerError::new(format!(- "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"- )));- }- if config.forward_auth && config.api_key_env.is_some() {- return Err(ServerError::new(format!(- "llm client {client_name} cannot set both forward_auth and api_key_env"- )));- }- let api_key = config- .api_key_env- .as_deref()- .map(|variable| {- if variable.trim().is_empty() {- return Err(ServerError::new(format!(- "llm client {client_name} api_key_env must not be empty"- )));- }- let api_key = std::env::var(variable).map_err(|error| {- ServerError::new(format!(- "llm client {client_name} could not read api_key_env {variable}: {error}"- ))- })?;- if api_key.trim().is_empty() {- return Err(ServerError::new(format!(- "llm client {client_name} api_key_env {variable} is empty"- )));- }- Ok(api_key)- })- .transpose()?;- let http = HttpBackendConfig {- base_url: config.base_url.as_str().to_string(),- api_key,- forward_auth: config.forward_auth,- extra_headers: config.extra_headers.clone(),- extra_body: extra_body.clone(),- max_retries: config.max_retries,- };- let backend = match config.format {- ClientFormat::OpenAiChat => Backend::OpenAiChat(http),- ClientFormat::OpenAiResponses => Backend::OpenAiResponses(http),- ClientFormat::AnthropicMessages => Backend::Anthropic(http),- };- Ok(backend)-}--const fn default_max_retries() -> u32 {- DEFAULT_MAX_RETRIES-}--// Resolve nested policy once so every supported parent builds the same child route.-fn build_subagent_router_config(- route_name: &str,- config: &SubagentRouteConfig,- targets: &BTreeMap<String, ModelId>,-) -> ServerResult<SubagentRouterConfig> {- match config {- SubagentRouteConfig::Passthrough { target } => Ok(SubagentRouterConfig::fixed_target(- resolve_target_model_id(route_name, target, targets)?,- )),- SubagentRouteConfig::LlmClassifier(config) => {- let LlmClassifierModeConfig::Custom(config) = config.classifier_mode(route_name)?- else {- return Err(ServerError::new(format!(- "route {route_name}: subagents llm_classifier only supports mode custom"- )));- };- let judge_target =- resolve_target_model_id(route_name, &config.classifier_target, targets)?;- let resolved_targets = config- .targets- .iter()- .map(|name| {- resolve_target_model_id(route_name, name, targets)- .map(|target| (name.clone(), target))- })- .collect::<ServerResult<Vec<_>>>()?;- let default_target = resolved_targets- .iter()- .find(|(name, _)| *name == config.default_target)- .map(|(_, target)| target.clone())- .ok_or_else(|| {- ServerError::new(format!(- "route {route_name}: subagents llm_classifier default_target {:?} must be one of its configured targets",- config.default_target- ))- })?;- let response_schema =- serde_json::from_str(&config.response_schema).map_err(|error| {- ServerError::new(format!(- "route {route_name}: subagents llm_classifier response_schema is invalid JSON: {error}"- ))- })?;- let mut classifier_config = CustomClassifierConfig::new(- config.prompt,- response_schema,- config.policy.into_libsy(),- );- classifier_config.recent_turn_window = config.recent_turn_window;- classifier_config.max_output_tokens = config.max_output_tokens;- let subagent_targets = resolved_targets- .iter()- .map(|(_, target)| target.clone())- .collect();- let classifier = Arc::new(- LlmTaskClassifier::new(LlmClassifierConfig::Custom {- judge_target,- targets: resolved_targets,- default_target: config.default_target,- config: classifier_config,- })- .map_err(|error| {- ServerError::new(format!(- "route {route_name}: subagents llm_classifier: {error}"- ))- })?,- );- Ok(SubagentRouterConfig {- targets: subagent_targets,- classifier,- default_target,- classify_trigger: config.classify_trigger,- message_hash_fallback: config.message_hash_fallback,- })- }- }-}--fn attach_subagent_router(- route_name: &str,- parent: Arc<dyn Algorithm>,- config: Option<&SubagentRouteConfig>,- targets: &BTreeMap<String, ModelId>,-) -> ServerResult<Arc<dyn Algorithm>> {- let Some(config) = config else {- return Ok(parent);- };- let config = build_subagent_router_config(route_name, config, targets)?;- let algorithm = SubagentRouter::new(parent, config).map_err(|error| {- ServerError::new(format!("route {route_name}: subagent routing: {error}"))- })?;- Ok(Arc::new(algorithm))-}--fn build_algorithm(- route_name: &str,- config: &RouteConfig,- targets: &BTreeMap<String, ModelId>,-) -> ServerResult<Arc<dyn Algorithm>> {- match config {- RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),- RouteConfig::Random {- targets: names,- weights,- seed,- ..- } => {- let target_set =- resolve_targets(route_name, names.iter().map(String::as_str), targets)?;- let algorithm = Random::new(target_set, weights.clone(), *seed)- .map_err(|error| ServerError::new(format!("random route {route_name}: {error}")))?;- Ok(Arc::new(algorithm))- }- RouteConfig::Passthrough {- target, subagents, ..- } => {- let parent_target = resolve_target_model_id(route_name, target, targets)?;- let algorithm = Passthrough::new(parent_target);- let parent: Arc<dyn Algorithm> = Arc::new(algorithm);- attach_subagent_router(route_name, parent, subagents.as_ref(), targets)- }- RouteConfig::LlmClassifier {- config: classifier_config,- ..- } => {- let classifier =- resolve_target_model_id(route_name, &classifier_config.classifier_target, targets)?;- let mode = classifier_config.classifier_mode(route_name)?;- let algorithm = match mode {- LlmClassifierModeConfig::Capability(config) => {- let strong =- resolve_target_model_id(route_name, &config.strong_target, targets)?;- let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;- let classifier_config = TaskClassifierConfig {- base_threshold: config.base_threshold,- threshold_step: config.threshold_step,- classify_trigger: config.classify_trigger,- message_hash_fallback: config.message_hash_fallback,- recent_turn_window: config.recent_turn_window,- contract: classifier_contract(config.prompt.as_deref())- .with_response_format_type(config.response_format_type),- max_output_tokens: config.max_output_tokens,- };- LlmTaskClassifier::new(LlmClassifierConfig::Capability {- judge_target: classifier,- efficient_target: weak,- capable_target: strong,- config: classifier_config,- })- }- LlmClassifierModeConfig::Escalation(config) => {- let strong =- resolve_target_model_id(route_name, &config.strong_target, targets)?;- let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?;- LlmTaskClassifier::new(LlmClassifierConfig::Escalation {- judge_target: classifier,- efficient_target: weak,- capable_target: strong,- contract: classifier_contract(config.prompt.as_deref())- .with_response_format_type(config.response_format_type),- config: config.judge,- max_output_tokens: config.max_output_tokens,- })- }- LlmClassifierModeConfig::Custom(config) => {- let resolved_targets = config- .targets- .iter()- .map(|name| {- resolve_target_model_id(route_name, name, targets)- .map(|target| (name.clone(), target))- })- .collect::<ServerResult<Vec<_>>>()?;- let response_schema = serde_json::from_str(&config.response_schema).map_err(- |error| {- ServerError::new(format!(- "llm_classifier route {route_name}: response_schema is invalid JSON: {error}"- ))- },- )?;- let mut classifier_config = CustomClassifierConfig::new(- config.prompt,- response_schema,- config.policy.into_libsy(),- );- classifier_config.classify_trigger = config.classify_trigger;- classifier_config.message_hash_fallback = config.message_hash_fallback;- classifier_config.recent_turn_window = config.recent_turn_window;- classifier_config.max_output_tokens = config.max_output_tokens;- LlmTaskClassifier::new(LlmClassifierConfig::Custom {- judge_target: classifier,- targets: resolved_targets,- default_target: config.default_target,- config: classifier_config,- })- }- }- .map_err(|error| {- ServerError::new(format!("llm_classifier route {route_name}: {error}"))- })?;- Ok(Arc::new(algorithm))- }- RouteConfig::StageRouter {- capable_target,- efficient_target,- picker,- confidence_threshold,- recent_turn_window,- handoff_notes,- capable_system_prompt,- efficient_system_prompt,- classifier,- subagents,- ..- } => {- if matches!(picker, PickerMode::CapableFirst) {- tracing::warn!(- "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration."- );- }- let capable = resolve_target_model_id(route_name, capable_target, targets)?;- let efficient = resolve_target_model_id(route_name, efficient_target, targets)?;- let mut config = StageRouterConfig::new(*picker, *confidence_threshold);- config.recent_window = *recent_turn_window;- config.handoff_notes = handoff_notes.clone();- config.tier_prompts = tier_prompts(- &capable,- capable_system_prompt.as_deref(),- &efficient,- efficient_system_prompt.as_deref(),- );- // The judge is called through its own target, so it is not a routing- // destination and stays out of the tier pair.- config.llm_fallback = classifier- .as_ref()- .map(|classifier| {- resolve_target_model_id(route_name, &classifier.target, targets).map(- |judge_target| LlmFallback {- judge_target,- config: classifier.task_classifier_config(),- },- )- })- .transpose()?;- let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| {- ServerError::new(format!("stage_router route {route_name}: {error}"))- })?;- let parent: Arc<dyn Algorithm> = Arc::new(algorithm);- attach_subagent_router(route_name, parent, subagents.as_ref(), targets)- }- RouteConfig::Advisor {- executor_target,- advisor_target,- reviewer_system_prompt,- redo_feedback_prefix,- gate_trigger,- gate_trigger_pattern,- max_reviews,- gate_stall_turns,- gate_min_tool_results,- advisor_max_tokens,- advisor_temperature,- transcript_max_chars,- fail_open,- ..- } => {- let executor = resolve_target_model_id(route_name, executor_target, targets)?;- let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;- // A pattern set under the default trigger would be silently- // ignored; reject the misconfiguration instead.- if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {- return Err(ServerError::new(format!(- "advisor route {route_name}: gate_trigger_pattern requires \- gate_trigger = \"pattern\""- )));- }- let mut config = AdvisorGateConfig::default();- if let Some(prompt) = reviewer_system_prompt {- config.reviewer_system_prompt = prompt.clone();- }- if let Some(prefix) = redo_feedback_prefix {- config.redo_feedback_prefix = prefix.clone();- }- config.gate_trigger = match gate_trigger {- AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,- AdvisorTriggerConfig::Pattern => {- GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())- }- };- config.max_reviews = *max_reviews;- config.gate_stall_turns = *gate_stall_turns;- config.gate_min_tool_results = *gate_min_tool_results;- config.advisor_max_tokens = *advisor_max_tokens;- config.advisor_temperature = *advisor_temperature;- config.transcript_max_chars = *transcript_max_chars;- config.fail_open = *fail_open;- let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {- ServerError::new(format!("advisor route {route_name}: {error}"))- })?;- Ok(Arc::new(algorithm))- }- }-}--const fn default_max_reviews() -> u32 {- 1-}--const fn default_advisor_max_tokens() -> u64 {- 2048-}--const fn default_transcript_max_chars() -> usize {- 200_000-}--const fn default_fail_open() -> bool {- true-}--fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {- prompt.map_or_else(ClassifierContractConfig::default, |prompt| {- ClassifierContractConfig::default().with_prompt(prompt)- })-}--fn default_classifier_max_output_tokens() -> u64 {- TaskClassifierConfig::default().max_output_tokens-}--/// Keys each configured system prompt by the target it belongs to.-fn tier_prompts(- capable: &str,- capable_prompt: Option<&str>,- efficient: &str,- efficient_prompt: Option<&str>,-) -> TargetPrompts {- let mut prompts = TargetPrompts::default();- if let Some(prompt) = capable_prompt {- prompts = prompts.with(capable, prompt);- }- if let Some(prompt) = efficient_prompt {- prompts = prompts.with(efficient, prompt);- }- prompts-}--fn resolve_targets<'a>(- route_name: &str,- names: impl IntoIterator<Item = &'a str>,- targets: &BTreeMap<String, ModelId>,-) -> ServerResult<Vec<ModelId>> {- names- .into_iter()- .map(|name| resolve_target_model_id(route_name, name, targets))- .collect()-}--fn resolve_target_model_id(- route_name: &str,- name: &str,- targets: &BTreeMap<String, ModelId>,-) -> ServerResult<ModelId> {- targets.get(name).cloned().ok_or_else(|| {- ServerError::new(format!(- "route {route_name} references unknown target {name}"- ))- })-}--fn validate_value(label: &str, value: &str) -> ServerResult<()> {- if value.trim().is_empty() || value.trim() != value {- return Err(ServerError::new(format!(- "{label} must be non-empty and have no surrounding whitespace"- )));- }- Ok(())-}--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- const VALID_CONFIG: &str = r#"-schema_version = 1--[llm_clients.primary]-format = "openai_chat"-base_url = "https://example.test/v1"--[llm_clients.responses]-format = "openai_responses"-base_url = "https://example.test/v1"--[llm_clients.anthropic]-format = "anthropic_messages"-base_url = "https://example.test"--[targets.classifier]-id = "classifier/model"-llm_client = "primary"--[targets.strong]-id = "strong/model"-llm_client = "responses"--[targets.weak]-id = "weak/model"-llm_client = "anthropic"--[routes.noop]-id = "switchyard/noop"-type = "noop"--[routes.random]-id = "switchyard/random"-type = "random"-targets = ["strong", "weak"]--[routes.classifier]-id = "switchyard/classifier"-type = "llm_classifier"-classifier_target = "classifier"-strong_target = "strong"-weak_target = "weak"-base_threshold = 0.5--[routes.passthrough]-id = "switchyard/passthrough"-type = "passthrough"-target = "weak"-"#;-- fn error_message(toml: &str) -> String {- match server_state_from_toml(toml) {- Ok(_) => "configuration unexpectedly succeeded".to_string(),- Err(error) => error.to_string(),- }- }-- fn with_subagent_llm_classifier(config: &str, route: &str, extra: &str) -> String {- let mut configured = config.to_string();- configured.push_str(&format!("\n[routes.{route}.subagents]\n"));- configured.push_str(- r#"type = "llm_classifier"-mode = "custom"-classifier_target = "classifier"-targets = ["strong", "weak"]-default_target = "weak"-prompt = "Select a target for this delegated task."-response_schema = '{"type":"object","properties":{"target":{"type":"string","enum":["strong","weak"]}},"required":["target"],"additionalProperties":false}'-policy = { type = "target_selector", selector = "/target" }-classify_trigger = "new_session""#,- );- configured.push_str(extra);- configured- }-- fn with_subagent_passthrough(config: &str, route: &str) -> String {- format!("{config}\n[routes.{route}.subagents]\ntype = \"passthrough\"\ntarget = \"strong\"")- }-- fn stage_config() -> String {- format!(- r#"{VALID_CONFIG}-[targets.stage_judge]-id = "stage-judge/model"-llm_client = "primary"--[routes.stage]-id = "switchyard/stage"-type = "stage_router"-capable_target = "strong"-efficient_target = "weak"-picker = "efficient_first"-confidence_threshold = 1.0--[routes.stage.classifier]-target = "stage_judge"-base_threshold = 0.5-"#- )- }-- #[test]- fn builds_all_supported_algorithm_types() -> ServerResult<()> {- let state = server_state_from_toml(VALID_CONFIG)?;- // The model id array is sorted alphabetically- assert_eq!(- state.models().collect::<Vec<_>>(),- [- "switchyard/classifier",- "switchyard/noop",- "switchyard/passthrough",- "switchyard/random",- ]- );- Ok(())- }-- #[test]- fn passthrough_and_stage_accept_subagent_routing() -> ServerResult<()> {- let stage = stage_config();- let stage_with_classifier = with_subagent_llm_classifier(&stage, "stage", "");- let parsed: ServerConfig = toml::from_str(&stage_with_classifier)- .map_err(|error| ServerError::new(format!("failed to parse stage config: {error}")))?;- let Some(stage_route) = parsed.routes.get("stage") else {- return Err(ServerError::new("stage route is missing"));- };- let callable_targets = stage_route.callable_target_names();- for expected in ["strong", "weak", "stage_judge", "classifier"] {- assert!(callable_targets.contains(&expected));- }-- for configured in [- with_subagent_llm_classifier(VALID_CONFIG, "passthrough", ""),- with_subagent_passthrough(VALID_CONFIG, "passthrough"),- stage_with_classifier,- with_subagent_passthrough(&stage, "stage"),- ] {- server_state_from_toml(&configured)?;- }- Ok(())- }-- #[test]- fn rejects_invalid_unreferenced_llm_client() {- let invalid = format!(- "{VALID_CONFIG}\n\- [llm_clients.unused]\n\- format = \"openai_chat\"\n\- base_url = \"not a url\"\n"- );- let message = error_message(&invalid);- assert!(- message.contains("base_url must be an absolute HTTP(S) URL"),- "unexpected error: {message}"- );- }-- #[test]- fn an_escalation_table_switches_the_classifier_route_to_escalation() -> ServerResult<()> {- // Present: the classifier target judges the weak tier's reply each turn instead of- // picking a tier ahead of it. The route builds either way, so the assertion is that- // the knob parses and its settings reach the algorithm's validation.- let escalating = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nescalation = { confirmations = 2 }",- );- server_state_from_toml(&escalating)?;-- // A setting that would starve the judge is rejected here rather than on the first- // request, the same as any other unusable route configuration.- let starved = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nescalation = { confirmations = 0 }",- );- assert!(error_message(&starved).contains("confirmations must be at least 1"));- Ok(())- }-- #[test]- fn classifier_judge_completion_caps_are_configurable() -> ServerResult<()> {- let capability = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nmax_output_tokens = 512",- );- server_state_from_toml(&capability)?;-- let escalation = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nmax_output_tokens = 256\nescalation = { confirmations = 2 }",- );- server_state_from_toml(&escalation)?;- Ok(())- }-- #[test]- fn classifier_prompts_are_configurable_in_both_modes() -> ServerResult<()> {- let capability = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nprompt = \"custom capability rubric\"",- );- server_state_from_toml(&capability)?;-- let escalation = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nprompt = \"custom trajectory rubric\"\nescalation = { confirmations = 2 }",- );- server_state_from_toml(&escalation)?;-- let empty = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nprompt = \" \"",- );- assert!(error_message(&empty).contains("classifier prompt must not be empty"));-- let schema_placeholder = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",- );- assert!(- error_message(&schema_placeholder)- .contains("Switchyard supplies the schema automatically")- );- Ok(())- }-- #[test]- fn mode_custom_rejects_capability_fields() {- let mixed = VALID_CONFIG.replace(- "base_threshold = 0.5",- "mode = \"custom\"\nbase_threshold = 0.5",- );-- assert!(- error_message(&mixed)- .contains("mode custom cannot use capability or escalation fields")- );- }-- #[test]- fn rejects_unknown_fields_and_algorithm_types() {- let unknown_field =- VALID_CONFIG.replace("schema_version = 1", "schema_version = 1\nmagic = true");- assert!(error_message(&unknown_field).contains("unknown field"));-- let nested_completion_cap = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nescalation = { max_output_tokens = 256 }",- );- assert!(error_message(&nested_completion_cap).contains("unknown field"));-- let unknown_classifier_field = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nclassifier_magic = true",- );- assert!(error_message(&unknown_classifier_field).contains("unknown field"));-- let target_capability = VALID_CONFIG.replace(- "llm_client = \"responses\"",- "llm_client = \"responses\"\ncontext_window = 1000000",- );- assert!(error_message(&target_capability).contains("unknown field `context_window`"));-- let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");- assert!(error_message(&unknown_algorithm).contains("unknown variant"));- }-- #[test]- fn rejects_unknown_stage_classifier_fields() {- // Nested classifier typos must fail instead of silently using a default.- let config = format!(- r#"{VALID_CONFIG}--[routes.stage]-id = "switchyard/stage"-type = "stage_router"-capable_target = "strong"-efficient_target = "weak"-picker = "efficient_first"-confidence_threshold = 1.0--[routes.stage.classifier]-target = "classifier"-base_threshold = 0.5-classifier_magic = true-"#- );-- let error = error_message(&config);- assert!(- error.contains("unknown field `classifier_magic`"),- "{error}"- );- }-- #[test]- fn rejects_invalid_references_and_parameters() {- let cases = [- (- VALID_CONFIG.replace("llm_client = \"primary\"", "llm_client = \"missing\""),- "unknown llm client missing",- ),- (- VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"missing\"]",- ),- "unknown target missing",- ),- (- VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"strong\"]",- ),- "random targets must be unique",- ),- (- VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"weak\"]\nweights = [1]",- ),- "expected 2 weights, got 1",- ),- (- VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"weak\"]\nweights = [0, 0]",- ),- "at least one weight must be positive",- ),- (- VALID_CONFIG.replace("base_threshold = 0.5", "base_threshold = 1.5"),- "base_threshold must be between 0 and 1",- ),- (- VALID_CONFIG.replace("classifier_target = \"classifier\"\n", ""),- "route references unknown target",- ),- (- VALID_CONFIG.replace(- "classifier_target = \"classifier\"",- "classifier_target = \"\"",- ),- "route references unknown target",- ),- (- VALID_CONFIG.replace(- "classifier_target = \"classifier\"",- "classifier_target = \" \"",- ),- "route references unknown target",- ),- (- VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nthreshold_step = -0.1",- ),- "threshold_step must be finite and greater than or equal to 0",- ),- (- VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.8\nthreshold_step = 0.11",- ),- "base_threshold + 2 * threshold_step must be at most 1",- ),- (- VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nmax_output_tokens = 0\nescalation = { confirmations = 2 }",- ),- "max_output_tokens must be at least 1",- ),- (- VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.5\nmessage_hash_fallback = true",- ),- "message_hash_fallback requires classify_trigger = new_session",- ),- (- with_subagent_llm_classifier(- VALID_CONFIG,- "passthrough",- "\nmessage_hash_fallback = true",- ),- "cannot use message_hash_fallback",- ),- (- with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "")- .replace("mode = \"custom\"", "mode = \"capability\""),- "mode capability cannot use custom classifier fields",- ),- (- VALID_CONFIG.replace(- "base_threshold = 0.5",- "escalation = { confirmations = 2 }\nclassify_trigger = \"user_turn\"",- ),- "mode escalation cannot use classify_trigger",- ),- (- VALID_CONFIG.replace("schema_version = 1", "schema_version = 2"),- "unsupported schema_version 2",- ),- (- VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),- "target name must be non-empty and have no surrounding whitespace",- ),- (- VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"weak\"]\ncontext_window = 0",- ),- "route random context_window must be greater than zero",- ),- ];-- for (toml, expected) in cases {- let error = error_message(&toml);- assert!(- error.contains(expected),- "expected error containing {expected}, got {error}"- );- }- }-- #[test]- fn accepts_duplicate_target_model_ids_on_one_client() -> ServerResult<()> {- // Two targets share one model id on one client. The client keeps one and drops the- // other, so the build warns and still succeeds, and both routes resolve. Serving one- // model under two route names this way is allowed; pointing both routes at one target- // is the tidier form.- const SAME_MODEL_TWO_ROUTES: &str = r#"-schema_version = 1--[llm_clients.primary]-format = "openai_chat"-base_url = "https://example.test/v1"--[targets.fast]-id = "gpt-4o"-llm_client = "primary"--[targets.smart]-id = "gpt-4o"-llm_client = "primary"--[routes.fast]-id = "switchyard/fast"-type = "passthrough"-target = "fast"--[routes.smart]-id = "switchyard/smart"-type = "passthrough"-target = "smart"-"#;- let state = server_state_from_toml(SAME_MODEL_TWO_ROUTES)?;- assert_eq!(- state.models().collect::<Vec<_>>(),- ["switchyard/fast", "switchyard/smart"]- );- Ok(())- }-- #[test]- fn accepts_same_model_id_on_different_llm_clients() -> ServerResult<()> {- // The same model id served by two llm clients never collides (each client keys its own- // models), so cross-provider A/B builds with no warning; only a repeat within one client- // warns.- const CROSS_PROVIDER: &str = r#"-schema_version = 1--[llm_clients.openai]-format = "openai_chat"-base_url = "https://example.test/v1"--[llm_clients.azure]-format = "openai_chat"-base_url = "https://azure.test/v1"--[targets.openai]-id = "gpt-4o"-llm_client = "openai"--[targets.azure]-id = "gpt-4o"-llm_client = "azure"--[routes.openai]-id = "switchyard/openai-gpt4o"-type = "passthrough"-target = "openai"--[routes.azure]-id = "switchyard/azure-gpt4o"-type = "passthrough"-target = "azure"-"#;- server_state_from_toml(CROSS_PROVIDER)?;- Ok(())- }-- #[test]- fn accepts_relative_weights_and_seed() -> ServerResult<()> {- let weighted = VALID_CONFIG.replace(- "targets = [\"strong\", \"weak\"]",- "targets = [\"strong\", \"weak\"]\nweights = [1, 3]\nseed = 42",- );- server_state_from_toml(&weighted)?;- Ok(())- }-- #[test]- fn accepts_new_session_trigger_with_message_hash_fallback() -> ServerResult<()> {- let configured = VALID_CONFIG.replace(- "base_threshold = 0.5",- "base_threshold = 0.25\nthreshold_step = 0.1\nclassify_trigger = \"new_session\"\nmessage_hash_fallback = true",- );- server_state_from_toml(&configured)?;- Ok(())- }-- #[test]- fn target_extra_body_is_parsed_and_applied_to_its_backend() -> ServerResult<()> {- let configured = VALID_CONFIG.replacen(- "llm_client = \"primary\"",- "llm_client = \"primary\"\n\- extra_body = { service_tier = \"priority\", \- chat_template_kwargs = { enable_thinking = false } }",- 1,- );- let config: ServerConfig = toml::from_str(&configured)- .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?;- let Some(target) = config.targets.get("classifier") else {- return Err(ServerError::new("classifier target is missing"));- };- let Some(client) = config.llm_clients.get("primary") else {- return Err(ServerError::new("primary llm client is missing"));- };- let backend = build_backend("primary", client, &target.extra_body)?;-- assert_eq!(- backend.extra_body().get("service_tier"),- Some(&json!("priority"))- );- assert_eq!(- backend- .extra_body()- .get("chat_template_kwargs")- .and_then(|value| value.get("enable_thinking")),- Some(&json!(false))- );- Ok(())- }-- #[test]- fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> {- let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| {- ServerError::new(format!("failed to parse default config: {error}"))- })?;- let Some(primary) = default.llm_clients.get("primary") else {- return Err(ServerError::new("primary llm client is missing"));- };- assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);-- let explicit = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\nmax_retries = 0",- 1,- );- let config: ServerConfig = toml::from_str(&explicit).map_err(|error| {- ServerError::new(format!("failed to parse explicit retry config: {error}"))- })?;- let Some(primary) = config.llm_clients.get("primary") else {- return Err(ServerError::new("primary llm client is missing"));- };- assert_eq!(primary.max_retries, 0);-- let maximum = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- &format!(- "base_url = \"https://example.test/v1\"\nmax_retries = {MAX_CONFIGURED_RETRIES}"- ),- 1,- );- let config: ServerConfig = toml::from_str(&maximum).map_err(|error| {- ServerError::new(format!("failed to parse maximum retry config: {error}"))- })?;- let Some(primary) = config.llm_clients.get("primary") else {- return Err(ServerError::new("primary llm client is missing"));- };- assert_eq!(primary.max_retries, MAX_CONFIGURED_RETRIES);- Ok(())- }-- #[test]- fn rejects_headers_that_switchyard_sets() {- let cases = [- (- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\n\- extra_headers = { AUTHORIZATION = \"Bearer custom-key\" }",- "AUTHORIZATION",- ),- (- "base_url = \"https://example.test\"",- "base_url = \"https://example.test\"\n\- extra_headers = { \"X-Api-Key\" = \"custom-key\" }",- "X-Api-Key",- ),- (- "base_url = \"https://example.test\"",- "base_url = \"https://example.test\"\n\- extra_headers = { \"ANTHROPIC-VERSION\" = \"custom-version\" }",- "ANTHROPIC-VERSION",- ),- ];-- for (original, replacement, header) in cases {- let configured = VALID_CONFIG.replacen(original, replacement, 1);- let error = error_message(&configured);- assert!(- error.contains(&format!("extra_headers cannot set {header:?}")),- "expected {header} to be rejected, got: {error}"- );- }- }-- #[test]- fn accepts_additional_headers() -> ServerResult<()> {- let configured = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\n\- extra_headers = { X-Inference-Priority = \"batch\" }",- 1,- );-- server_state_from_toml(&configured)?;- Ok(())- }-- #[test]- fn retry_budget_rejects_negative_values() {- let invalid = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\nmax_retries = -1",- 1,- );- assert!(error_message(&invalid).contains("max_retries"));- }-- #[test]- fn retry_budget_rejects_excessive_values() {- let invalid = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\nmax_retries = 11",- 1,- );- assert!(- error_message(&invalid).contains("llm client primary max_retries must be at most 10")- );- }-- #[test]- fn api_key_environment_reference_is_validated() {- let missing = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\napi_key_env = \"SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET\"",- 1,- );- assert!(error_message(&missing).contains("SWITCHYARD_CONFIG_TEST_KEY_THAT_IS_NOT_SET"));-- const EMPTY_KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_EMPTY_KEY";- unsafe {- // "unsafe" is for concurrent reads and writes, very rare- std::env::set_var(EMPTY_KEY_ENV, "");- }- let empty = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- &format!("base_url = \"https://example.test/v1\"\napi_key_env = \"{EMPTY_KEY_ENV}\""),- 1,- );- let message = error_message(&empty);- unsafe {- std::env::remove_var(EMPTY_KEY_ENV);- }- assert!(message.contains("is empty"));- }-- #[test]- fn forward_auth_rejects_conflicting_credentials() {- let competing_auth = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- "base_url = \"https://example.test/v1\"\n\- forward_auth = true\n\- api_key_env = \"UNUSED_TEST_KEY\"",- 1,- );- assert!(- error_message(&competing_auth).contains("cannot set both forward_auth and api_key_env")- );-- let static_auth = VALID_CONFIG.replacen(- "base_url = \"https://example.test\"",- "base_url = \"https://example.test\"\n\- forward_auth = true\n\- extra_headers = { Authorization = \"static-value\" }",- 1,- );- assert!(error_message(&static_auth).contains("extra_headers cannot set \"Authorization\""));-- let static_beta = static_auth.replace("Authorization", "anthropic-beta");- assert!(- error_message(&static_beta).contains("extra_headers cannot set \"anthropic-beta\"")- );-- for header in ["chatgpt-account-id", "x-openai-fedramp"] {- let static_context = VALID_CONFIG.replacen(- "base_url = \"https://example.test/v1\"",- &format!(- "base_url = \"https://example.test/v1\"\n\- forward_auth = true\n\- extra_headers = {{ \"{header}\" = \"static-value\" }}"- ),- 1,- );- assert!(- error_message(&static_context)- .contains(&format!("extra_headers cannot set \"{header}\""))- );- }- }-- const ADVISOR_CONFIG: &str = r#"-schema_version = 1--[llm_clients.anthropic]-format = "anthropic_messages"-base_url = "https://example.test"--[targets.executor]-id = "executor/model"-llm_client = "anthropic"--[targets.advisor]-id = "advisor/model"-llm_client = "anthropic"--[routes.gated]-id = "switchyard/advisor"-type = "advisor"-executor_target = "executor"-advisor_target = "advisor"-"#;-- #[test]- fn advisor_route_parses_with_defaults_and_builds() -> ServerResult<()> {- let state = server_state_from_toml(ADVISOR_CONFIG)?;- assert_eq!(state.models().collect::<Vec<_>>(), ["switchyard/advisor"]);- Ok(())- }-- #[test]- fn advisor_route_accepts_every_gate_knob() -> ServerResult<()> {- let tuned = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- concat!(- "advisor_target = \"advisor\"\n",- "reviewer_system_prompt = \"review it\"\n",- "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",- "gate_trigger = \"pattern\"\n",- "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",- "max_reviews = 2\n",- "gate_stall_turns = 40\n",- "gate_min_tool_results = 1\n",- "advisor_max_tokens = 1024\n",- "advisor_temperature = 0.0\n",- "transcript_max_chars = 100000\n",- "fail_open = false\n",- "context_window = 200000\n",- "tool_calling = true\n",- "reasoning = true",- ),- );- server_state_from_toml(&tuned)?;- Ok(())- }-- #[test]- fn advisor_route_rejects_unknown_keys() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"advisor\"\nbogus_field = 1",- );- assert!(error_message(&invalid).contains("bogus_field"));- }-- #[test]- fn advisor_route_requires_both_targets() {- let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");- assert!(error_message(&missing).contains("advisor_target"));- }-- #[test]- fn advisor_route_rejects_unknown_target() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"missing\"",- );- assert!(error_message(&invalid).contains("missing"));- }-- #[test]- fn advisor_route_rejects_invalid_pattern() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",- );- assert!(error_message(&invalid).contains("not a valid regex"));- }-- #[test]- fn advisor_route_rejects_pattern_without_pattern_trigger() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",- );- assert!(- error_message(&invalid)- .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")- );- }-- #[test]- fn advisor_route_pattern_trigger_requires_pattern() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",- );- assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));- }-- #[test]- fn advisor_route_rejects_zero_max_reviews() {- let invalid = ADVISOR_CONFIG.replace(- "advisor_target = \"advisor\"",- "advisor_target = \"advisor\"\nmax_reviews = 0",- );- assert!(error_message(&invalid).contains("max_reviews must be at least 1"));- }+ ServerState::from_runner(Runner::load(path).map_err(ServerError::from)?)}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 80f75b7fe..914fb6c8a 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -32,19 +32,21 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Extension, Json, Router};use axum_server::tls_rustls::RustlsConfig;-use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};+use libsy::{Algorithm, LibsyError, RoutingOutcome};use parking_lot::Mutex;use serde::{Deserialize, Serialize};use serde_json::{Value, json};-use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};+use switchyard_runner::{+ CallerAuthKind, DecisionTarget, ModelCapabilities, Route, RunOutput, Runner, RunnerError,+};use tokio::net::{TcpListener, TcpSocket};use tokio::task;use tracing::{Instrument, Level};use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};-use crate::config::ServerConfig;use crate::response::into_http_response;use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};@@ -86,106 +88,51 @@ impl Display for ServerError {impl Error for ServerError {}+impl From<RunnerError> for ServerError {+ fn from(error: RunnerError) -> Self {+ let mut message = error.to_string();+ let mut source = error.source();+ while let Some(error) = source {+ message.push_str(": ");+ message.push_str(&error.to_string());+ source = error.source();+ }+ Self::new(message)+ }+}+/// Result returned by server setup and lifecycle operations.pub type ServerResult<T> = std::result::Result<T, ServerError>;-/// Capabilities that one route advertises on `GET /v1/models`.-///-/// An unset capability is undeclared: it serializes as `null` in the OpenAI-/// `data` entry, and the Codex entry falls back to a safe default for it.-#[derive(Clone, Copy, Default)]-struct ModelCapabilities {- context_window: Option<u32>,- tool_calling: Option<bool>,- // Whether the routed model takes reasoning controls. The server cannot probe- // this, so a route opts in via config; undeclared routes advertise as- // non-reasoning to Codex (fail closed).- reasoning: Option<bool>,-}--/// A registered algorithm route and its server-owned endpoint metadata.-struct RouteEntry {- algorithm: Arc<dyn Algorithm>,- /// Resolves each offloaded call to the client configured for the target the algorithm- /// selected. A route is a synthetic model with no upstream of its own, so this is a- /// per-target lookup, never one client serving the whole route.- target_clients: ClientRouter,- caller_auth: Option<CallerAuthKind>,- capabilities: ModelCapabilities,- count_tokens_target: Option<CountTokensTarget>,- config_name: Option<String>,-}-/// Decision result using the same selected/fallback order as libsy.#[derive(Serialize)]-struct DecisionResponse<'a> {- selected: DecisionTargetResponse<'a>,- fallbacks: Vec<DecisionTargetResponse<'a>>,+struct DecisionResponse {+ selected: DecisionTargetResponse,+ fallbacks: Vec<DecisionTargetResponse>,#[serde(skip_serializing_if = "Option::is_none")]response: Option<Value>,}-/// Borrowed response view over one target's existing configuration.+/// Response view over one target's existing configuration.#[derive(Serialize)]-struct DecisionTargetResponse<'a> {- target: &'a str,- model: &'a ModelId,- llm_client: DecisionLlmClientResponse<'a>,- extra_body: &'a BTreeMap<String, Value>,+struct DecisionTargetResponse {+ target: String,+ model: ModelId,+ llm_client: DecisionLlmClientResponse,+ extra_body: BTreeMap<String, Value>,}/// Non-secret client settings needed to call a selected model.#[derive(Serialize)]-struct DecisionLlmClientResponse<'a> {+struct DecisionLlmClientResponse {format: WireFormat,- base_url: &'a str,-}--/// Caller credential family required by forwarded-auth backends.-#[derive(Clone, Copy, Debug, Eq, PartialEq)]-pub(crate) enum CallerAuthKind {- Anthropic,- OpenAi,-}--impl CallerAuthKind {- const fn as_str(self) -> &'static str {- match self {- Self::Anthropic => "anthropic",- Self::OpenAi => "openai",- }- }-- const fn accepts(self, wire_format: WireFormat) -> bool {- matches!(- (self, wire_format),- (Self::Anthropic, WireFormat::AnthropicMessages)- | (- Self::OpenAi,- WireFormat::OpenAiChat | WireFormat::OpenAiResponses- )- )- }-}--/// Exact upstream model used by the server's Anthropic token-count endpoint.-#[derive(Clone)]-struct CountTokensTarget {- model: ModelId,- client: Arc<TranslatingLlmClient>,-}--impl CountTokensTarget {- async fn count_tokens(&self, request: Request) -> Result<Value, LlmClientError> {- self.client.count_tokens(&self.model, request).await- }+ base_url: String,}/// Shared server state used by all endpoint handlers.#[derive(Clone)]pub struct ServerState {- routes: Arc<BTreeMap<ModelId, RouteEntry>>,- config: Option<Arc<ServerConfig>>,+ runner: Arc<Runner>,metrics: prometheus::Registry,stats: StatsAccumulator,routing_log: Option<SharedRoutingLog>,@@ -227,75 +174,37 @@ impl SharedRoutingLog {}impl ServerState {- /// Creates server state from route model IDs, their libsy algorithms, and the- /// per-target client routing each route's calls are resolved through.- pub fn new(- routes: impl IntoIterator<Item = (ModelId, Arc<dyn Algorithm>, ClientRouter)>,- ) -> ServerResult<Self> {- Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| {- (- model,- algorithm,- clients,- None,- ModelCapabilities::default(),- None,- None,- )- }))+ /// Creates server state from route model IDs, algorithms, and per-target clients.+ pub fn new(routes: Vec<(ModelId, Arc<dyn Algorithm>, ClientRouter)>) -> ServerResult<Self> {+ let routes = routes+ .into_iter()+ .map(|(model, algorithm, clients)| {+ (+ model,+ Route::new(+ algorithm,+ clients,+ None,+ ModelCapabilities::default(),+ None,+ Vec::new(),+ ),+ )+ })+ .collect();+ let runner = Runner::new(routes);+ Self::from_runner(runner)}- fn new_with_capabilities(- routes: impl IntoIterator<- Item = (- ModelId,- Arc<dyn Algorithm>,- ClientRouter,- Option<CallerAuthKind>,- ModelCapabilities,- Option<CountTokensTarget>,- Option<String>,- ),- >,- ) -> ServerResult<Self> {- let mut entries = BTreeMap::new();- for (- model,- algorithm,- target_clients,- caller_auth,- capabilities,- count_tokens_target,- config_name,- ) in routes- {- let model = ModelId::from(model.trim());- if model.is_empty() {- return Err(ServerError::new("route model must not be empty"));- }- let entry = RouteEntry {- algorithm,- target_clients,- caller_auth,- capabilities,- count_tokens_target,- config_name,- };- if entries.insert(model.clone(), entry).is_some() {- return Err(ServerError::new(format!("duplicate route model {model}")));- }- }- if entries.is_empty() {- return Err(ServerError::new("at least one algorithm route is required"));- }+ /// Creates HTTP-server state around an already configured runner.+ pub fn from_runner(runner: Runner) -> ServerResult<Self> {let metrics = metrics::registry().map_err(ServerError::new)?;let stats = StatsAccumulator::new(metrics.clone(),- entries.values().map(|entry| entry.algorithm.name()),+ runner.models().map(|model| model.algorithm),);Ok(Self {- routes: Arc::new(entries),- config: None,+ runner: Arc::new(runner),metrics,stats,routing_log: None,@@ -303,12 +212,6 @@ impl ServerState {})}- /// Retains the validated configuration used to describe decision results.- fn with_config(mut self, config: Arc<ServerConfig>) -> Self {- self.config = Some(config);- self- }-/// Enables durable per-request routing records at `path`.pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {self.routing_log = Some(SharedRoutingLog::new(path.into())?);@@ -317,57 +220,44 @@ impl ServerState {/// Returns the route model IDs served by the configured algorithms.pub fn models(&self) -> impl Iterator<Item = &str> {- self.routes.keys().map(ModelId::as_str)+ self.runner.models().map(|model| model.id.as_str())}/// Returns the caller credential family used by `model`, if any.- pub fn caller_auth_kind(&self, model: &str) -> ServerResult<Option<&'static str>> {- self.route_for_model(model)- .map(|entry| entry.caller_auth.map(CallerAuthKind::as_str))- .ok_or_else(|| ServerError::new(format!("unknown route model {model:?}")))+ pub fn caller_auth_kind(&self, model: &str) -> Option<&'static str> {+ self.runner+ .route(model)+ .and_then(|route| route.caller_auth())+ .map(|kind| kind.as_str())}- fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {- self.routes.get(model)+ fn route_for_model(&self, model: &str) -> Option<&Route> {+ self.runner.route(model)}- /// Resolves the routing outcome through the target names configured for its route.- fn decision_response<'a>(- &'a self,- route: &'a RouteEntry,+ fn decision_response(+ &self,+ route_model: &ModelId,outcome: &RoutingOutcome,response: Option<Value>,- ) -> Option<DecisionResponse<'a>> {- let config = self.config.as_ref()?;- let target_names = config.routing_target_names(route.config_name.as_ref()?)?;- let resolve = |model: &ModelId| {- let (target_name, target) = target_names- .iter()- .filter_map(|name| config.targets.get_key_value(*name))- .find(|(_, target)| target.id == *model)?;- let client = config.llm_clients.get(&target.llm_client)?;- Some(DecisionTargetResponse {- target: target_name,- model: &target.id,- llm_client: DecisionLlmClientResponse {- format: client.format.wire_format(),- base_url: client.base_url.as_str(),- },- extra_body: &target.extra_body,- })+ ) -> Option<DecisionResponse> {+ let description = self.runner.describe_decision(route_model, outcome)?;+ let convert = |target: DecisionTarget| DecisionTargetResponse {+ target: target.target,+ model: target.model,+ llm_client: DecisionLlmClientResponse {+ format: target.format,+ base_url: target.base_url,+ },+ extra_body: target.extra_body,};Some(DecisionResponse {- selected: resolve(&outcome.selected_model_id)?,- fallbacks: outcome- .fallback_models- .iter()- .map(resolve)- .collect::<Option<Vec<_>>>()?,+ selected: convert(description.selected),+ fallbacks: description.fallbacks.into_iter().map(convert).collect(),response,})}}-/// Runtime options shared by server entry points.#[derive(Clone, Debug)]pub struct ServerRunOptions {@@ -682,9 +572,15 @@ async fn decision(Ok(resolved) => resolved,Err(response) => return response,};- let mut outcome = match run_decision_only(route, request).await {+ let route_model = request+ .llm_request+ .model+ .as_deref()+ .map(ModelId::from)+ .unwrap_or_default();+ let mut outcome = match route.decide(request).await {Ok(outcome) => outcome,- Err(error) => return algorithm_error(error),+ Err(error) => return runner_error(error),};let response = match outcome.response.take() {Some(response) => {@@ -706,7 +602,7 @@ async fn decision(}None => None,};- match state.decision_response(route, &outcome, response) {+ match state.decision_response(&route_model, &outcome, response) {Some(response) => Json(response).into_response(),None => {server_error("routing outcome contains a model with no callable target configuration")@@ -714,27 +610,6 @@ async fn decision(}}-/// Completes routing-time calls and returns the outcome without serving its answer target.-async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result<RoutingOutcome> {- drive(Arc::clone(&route.algorithm), request, |call| {- serve_decision_dependency(route, call)- })- .await-}--/// Serves a classifier or judge call that the decision depends on.-async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> {- let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;- let result = match route.target_clients.route(&model) {- Ok(client) => client- .call(call.request.clone())- .await- .map_err(|source| LibsyError::client_call(model, source)),- Err(source) => Err(LibsyError::client_call(model, source)),- };- call.respond(result)-}-/// Anthropic token counting against the route's explicitly configured target.async fn anthropic_count_tokens(State(state): State<ServerState>,@@ -756,17 +631,16 @@ async fn anthropic_count_tokens(Ok(resolved) => resolved,Err(response) => return anthropic_error_response(response),};- let Some(target) = route.count_tokens_target.as_ref() else {- return anthropic_error_response(error_response(+ anthropic_error_response(match route.count_tokens(request).await {+ Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),+ Err(RunnerError::CountTokensUnsupported) => error_response(StatusCode::BAD_REQUEST,"route has no Anthropic target for token counting","invalid_request_error","count_tokens_unsupported",- ));- };- anthropic_error_response(match target.count_tokens(request).await {- Ok(payload) => (StatusCode::OK, Json(payload)).into_response(),- Err(error) => count_tokens_error(error),+ ),+ Err(RunnerError::Client(error)) => count_tokens_error(error),+ Err(error) => server_error(error.to_string()),})}@@ -867,7 +741,7 @@ fn resolve_route(metadata: Metadata,body: Value,wire_format: WireFormat,-) -> std::result::Result<(&RouteEntry, Request), Response> {+) -> std::result::Result<(&Route, Request), Response> {let llm_request = decode_request(wire_format, &body).map_err(|error| invalid_body_error(StatusCode::BAD_REQUEST, error.to_string()))?;let requested_model = llm_request@@ -890,8 +764,8 @@ fn resolve_route("model_not_found",)})?;- if let Some(caller_auth) = route.caller_auth- && !caller_auth.accepts(wire_format)+ if let Err(RunnerError::IncompatibleCallerFormat(caller_auth)) =+ route.check_caller_format(wire_format){let (provider, expected_endpoint) = match caller_auth {CallerAuthKind::Anthropic => ("Anthropic", "/v1/messages"),@@ -929,17 +803,18 @@ async fn handle_llm_request(};// Only the Codex namespace mapping is needed downstream, not the whole request.let request_extensions = request.llm_request.extensions.clone();- let algorithm = Arc::clone(&route.algorithm);- let client_router = route.target_clients.clone();let observer = stats_observer(state.stats.clone(),state.routing_log.clone().zip(routing_log_context.clone()),);- let (selected_model, response) =- match switchyard_llm_client::run(algorithm, client_router, request, Some(observer)).await {- Ok(result) => result,- Err(error) => return algorithm_error(error),- };+ let output = match route.execute(request, Some(observer)).await {+ Ok(output) => output,+ Err(error) => return runner_error(error),+ };+ let RunOutput {+ selected_model,+ response,+ } = output;// The response carries the candidate that actually served it. Fall back to the routing// selection for algorithms that return a response without an offloaded model call.let served_model = response.served_model().cloned().or(Some(selected_model));@@ -1071,6 +946,14 @@ fn algorithm_error(error: LibsyError) -> Response {client_error(source)}+fn runner_error(error: RunnerError) -> Response {+ match error {+ RunnerError::Algorithm(error) => algorithm_error(error),+ RunnerError::Client(error) => client_error(&error),+ error => server_error(error.to_string()),+ }+}+fn client_error(error: &LlmClientError) -> Response {match error {LlmClientError::InvalidRequest { message }@@ -1235,9 +1118,9 @@ fn error_response(async fn models(State(state): State<ServerState>) -> Json<Value> {Json(model_list_payload(state- .routes- .iter()- .map(|(model, entry)| (model.as_str(), entry.capabilities)),+ .runner+ .models()+ .map(|model| (model.id.as_str(), model.capabilities)),))}@@ -1321,7 +1204,8 @@ async fn not_found() -> Response {fn model_list_payload<'a>(entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,) -> Value {- let entries = entries.into_iter().collect::<Vec<_>>();+ let mut entries = entries.into_iter().collect::<Vec<_>>();+ entries.sort_unstable_by_key(|(model_id, _)| *model_id);let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();let first_id = model_ids.first().copied();let last_id = model_ids.last().copied();