Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(runner): configure Jev decision judges (#916)
nachiketb-nvidiafdf6454a0fa3
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex 5cc838209..a7e40e751 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -12,12 +12,14 @@ use std::sync::Arc;use libsy::{AdvisorGate, AdvisorGateConfig, Algorithm, CapabilityJudgeConfig, ClassifierContractConfig,ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig,- CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger,- HandoffNoteConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier,- Noop, Passthrough, PickerMode, PlanExecute, PlanExecuteConfig, Random, StageRouter,- StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics,+ CustomClassifierConfig, CustomClassifierPolicy, DecisionJudgeConfig, EscalationJudgeConfig,+ GateTrigger, HandoffNoteConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback,+ LlmTaskClassifier, Noop, Passthrough, PickerMode, PlanExecute, PlanExecuteConfig, Random,+ StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig,+ ToolSemantics,};use serde::Deserialize;+use serde_json::Value;use switchyard_protocol::{Category, ModelId};/// Error returned when an algorithm description cannot be constructed.@@ -102,14 +104,16 @@ struct CapabilityClassifierRouteConfig {fail_open: bool,strong_target: String,weak_target: String,- base_threshold: f64,- threshold_step: f64,+ judge: CapabilityJudgeRouteConfig,classify_trigger: ClassifyTrigger,message_hash_fallback: bool,recent_turn_window: Option<usize>,- prompt: Option<String>,- response_format_type: ClassifierResponseFormat,- max_output_tokens: u64,+}++#[derive(Clone, Debug)]+enum CapabilityJudgeRouteConfig {+ Llm(LlmCapabilityConfig),+ Decision(DecisionJudgeRouteConfig),}#[derive(Clone, Debug)]@@ -219,6 +223,20 @@ impl CategoryModelConfig {}}+/// Relative-advantage judgment using a decision model instead of an LLM prompt.+#[derive(Clone, Debug, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct DecisionJudgeRouteConfig {+ /// Route capable only when its advantage score is strictly above this cutoff.+ pub cutoff: f64,+ /// Replaces the default relative-advantage instructions.+ pub instructions: Option<Value>,+ /// Anonymous evidence labels mapped to configured LLM target names.+ pub candidates: BTreeMap<String, String>,+ /// Candidate descriptions and reference outcomes, passed unchanged to the judge.+ pub evidence: Value,+}+/// Settings for an `llm_classifier` route. Which fields are required depends on/// the [`ClassifierMode`]; using a field from the wrong mode is an error.#[derive(Clone, Debug, Default, Deserialize)]@@ -260,6 +278,8 @@ pub struct LlmClassifierRouteConfig {/// Escalation mode: how many escalate verdicts latch the session, and how/// much of the transcript the judge sees.pub escalation: Option<EscalationJudgeConfig>,+ /// Capability mode: use a decision target as the judge, with relative-advantage scoring.+ pub decision: Option<DecisionJudgeRouteConfig>,/// Custom mode: runtime model groups.pub models: Option<CategoryModelConfig>,/// Custom mode: category used when the judge fails or its verdict cannot be routed.@@ -542,6 +562,16 @@ impl StageClassifierConfig {}impl AlgorithmSpec {+ pub(crate) fn decision_judge(&self) -> Option<(&str, &DecisionJudgeRouteConfig)> {+ match self {+ Self::LlmClassifier { config, .. } => config+ .decision+ .as_ref()+ .map(|decision| (config.classifier_target.as_str(), decision)),+ _ => None,+ }+ }+/// Completion targets in algorithm order; judge-only targets are excluded.pub fn routing_target_names(&self) -> Vec<&str> {match self {@@ -893,6 +923,7 @@ impl LlmClassifierRouteConfig {response_format_type,max_output_tokens,escalation,+ decision,models,default_target,response_schema,@@ -905,6 +936,12 @@ impl LlmClassifierRouteConfig {(None, false) => ClassifierMode::Capability,};+ if decision.is_some() && !matches!(selected_mode, ClassifierMode::Capability) {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name}: decision requires capability mode"+ )));+ }+if !matches!(selected_mode, ClassifierMode::Capability) && *fail_open == Some(true) {return Err(AlgorithmConfigError::new(format!("llm_classifier route {route_name}: fail_open = true requires capability mode"@@ -928,6 +965,31 @@ impl LlmClassifierRouteConfig {response_schema,policy,)?;+ let judge = if let Some(decision) = decision {+ if base_threshold.is_some()+ || threshold_step.is_some()+ || prompt.is_some()+ || *response_format_type != ClassifierResponseFormat::JsonSchema+ || *max_output_tokens != default_classifier_max_output_tokens()+ {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name}: decision cannot use LLM judge settings; use decision.cutoff and decision.instructions"+ )));+ }+ CapabilityJudgeRouteConfig::Decision(decision.clone())+ } else {+ CapabilityJudgeRouteConfig::Llm(LlmCapabilityConfig {+ base_threshold: required_classifier_field(+ route_name,+ "base_threshold",+ base_threshold,+ )?,+ threshold_step: threshold_step.unwrap_or_default(),+ contract: classifier_contract(prompt.as_deref())+ .with_response_format_type(*response_format_type),+ max_output_tokens: *max_output_tokens,+ })+ };Ok(LlmClassifierModeConfig::Capability(CapabilityClassifierRouteConfig {classifier_target: classifier_target.clone(),@@ -942,18 +1004,10 @@ impl LlmClassifierRouteConfig {"weak_target",weak_target,)?,- base_threshold: required_classifier_field(- route_name,- "base_threshold",- base_threshold,- )?,- threshold_step: threshold_step.unwrap_or_default(),+ judge,classify_trigger: *classify_trigger,message_hash_fallback: *message_hash_fallback,recent_turn_window: *recent_turn_window,- prompt: prompt.clone(),- response_format_type: *response_format_type,- max_output_tokens: *max_output_tokens,},))}@@ -1243,14 +1297,34 @@ fn build_algorithm(let mode = classifier_config.validated_classifier_mode(route_name)?;let algorithm = match mode {LlmClassifierModeConfig::Capability(config) => {+ let judge = match config.judge {+ CapabilityJudgeRouteConfig::Llm(judge) => CapabilityJudgeConfig::Llm(judge),+ CapabilityJudgeRouteConfig::Decision(judge) => {+ for target in [&config.strong_target, &config.weak_target] {+ if !judge.candidates.values().any(|name| name == target) {+ return Err(AlgorithmConfigError::new(format!(+ "llm_classifier route {route_name}: decision.candidates must include target {target}"+ )));+ }+ }+ let candidates = judge+ .candidates+ .into_iter()+ .map(|(label, name)| {+ resolve_target_model_id(route_name, &name, targets)+ .map(|model| (label, model))+ })+ .collect::<AlgorithmResult<_>>()?;+ CapabilityJudgeConfig::Decision(DecisionJudgeConfig {+ cutoff: judge.cutoff,+ instructions: judge.instructions,+ candidates,+ evidence: judge.evidence,+ })+ }+ };let classifier_config = TaskClassifierConfig {- judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {- base_threshold: config.base_threshold,- threshold_step: config.threshold_step,- contract: classifier_contract(config.prompt.as_deref())- .with_response_format_type(config.response_format_type),- max_output_tokens: config.max_output_tokens,- }),+ judge,fail_open: config.fail_open,classify_trigger: config.classify_trigger,message_hash_fallback: config.message_hash_fallback,diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 51105bd42..2c85a97ca 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -15,9 +15,9 @@ use serde::{Deserialize, Deserializer};use serde_json::Value;use switchyard_llm_client::{AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,- TranslatingLlmClient,+ SystemOneClient, TranslatingLlmClient,};-use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat};+use switchyard_protocol::{Category, ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat};use crate::{AlgorithmSpec, AuxiliaryTarget, CallerAuthKind, DecisionTarget, ModelCapabilities, Route,@@ -60,6 +60,10 @@ pub(crate) struct DeploymentConfig {#[serde(default)]llm_clients: BTreeMap<String, LlmClientConfig>,targets: BTreeMap<String, TargetConfig>,+ #[serde(default)]+ decision_clients: BTreeMap<String, DecisionClientConfig>,+ #[serde(default)]+ decision_targets: BTreeMap<String, DecisionTargetConfig>,routes: BTreeMap<String, RouteConfig>,}@@ -211,16 +215,49 @@ impl DeploymentConfig {let mut provider_api_keys = Vec::new();let clients = self.build_clients(&mut provider_api_keys)?;+ let decision_clients = self.build_decision_clients(&mut provider_api_keys)?;+ for (name, target) in &self.decision_targets {+ validate_value("decision target name", name)?;+ validate_value(&format!("decision target {name} id"), &target.id)?;+ if self.targets.contains_key(name) {+ return Err(RunnerError::configuration(format!(+ "target {name} is defined as both an LLM and decision target"+ )));+ }+ if !decision_clients.contains_key(&target.decision_client) {+ return Err(RunnerError::configuration(format!(+ "decision target {name} references unknown decision client {}",+ target.decision_client+ )));+ }+ }let targets = self.build_targets();let fallback_base_url = self.fallback_base_url()?;let mut routes = Vec::with_capacity(self.routes.len());for (route_name, config) in &self.routes {- for target_name in config.callable_target_names() {- self.targets.get(target_name).ok_or_else(|| {- RunnerError::configuration(format!(- "route references unknown target {target_name}"- ))- })?;+ let decision_judge = config.algorithm.decision_judge();+ for name in config.callable_target_names() {+ let (exists, kind) = if decision_judge.is_some_and(|(judge, _)| judge == name) {+ (self.decision_targets.contains_key(name), "decision")+ } else {+ (self.targets.contains_key(name), "LLM")+ };+ if !exists {+ return Err(RunnerError::configuration(format!(+ "route references unknown target {name}; route {route_name} requires target kind {kind}"+ )));+ }+ }+ for name in config.routing_target_names().into_iter().chain(+ decision_judge+ .into_iter()+ .flat_map(|(_, judge)| judge.candidates.values().map(String::as_str)),+ ) {+ if !self.targets.contains_key(name) {+ return Err(RunnerError::configuration(format!(+ "route {route_name} completion and candidate target {name} must be an LLM target"+ )));+ }}let capabilities = config.capabilities();if capabilities.context_window == Some(0) {@@ -233,7 +270,7 @@ impl DeploymentConfig {.build(route_name, &targets).map_err(|error| RunnerError::configuration_source(error.to_string(), error))?;let (route_clients, caller_auth) =- self.build_route_clients(route_name, config, &clients)?;+ self.build_route_clients(route_name, config, &clients, &decision_clients)?;let anthropic_auxiliary_target =self.build_anthropic_auxiliary_target(config, &clients);let responses_auxiliary_target =@@ -343,10 +380,49 @@ impl DeploymentConfig {Ok(clients)}+ fn build_decision_clients(+ &self,+ provider_api_keys: &mut Vec<String>,+ ) -> RunnerResult<BTreeMap<String, Arc<dyn RoutedDecisionClient>>> {+ self.decision_clients+ .iter()+ .map(|(name, config)| {+ validate_value("decision client name", name)?;+ let DecisionClientConfig::SystemOne {+ endpoint,+ api_key_env,+ timeout_ms,+ } = config;+ if *timeout_ms == 0 {+ return Err(RunnerError::configuration(format!(+ "decision client {name} timeout_ms must be at least 1"+ )));+ }+ let api_key = read_api_key(&format!("decision client {name}"), api_key_env)?;+ let client = SystemOneClient::new(+ endpoint.0.clone(),+ api_key.clone(),+ Duration::from_millis(*timeout_ms),+ )+ .map_err(|error| RunnerError::configuration(error.to_string()))?;+ provider_api_keys.push(api_key);+ Ok((+ name.clone(),+ Arc::new(client) as Arc<dyn RoutedDecisionClient>,+ ))+ })+ .collect()+ }+fn build_targets(&self) -> BTreeMap<String, ModelId> {self.targets.iter().map(|(name, config)| (name.clone(), config.id.clone()))+ .chain(+ self.decision_targets+ .iter()+ .map(|(name, config)| (name.clone(), config.id.clone())),+ ).collect()}@@ -358,6 +434,7 @@ impl DeploymentConfig {route_name: &str,route: &RouteConfig,clients: &BTreeMap<String, Arc<TranslatingLlmClient>>,+ decision_clients: &BTreeMap<String, Arc<dyn RoutedDecisionClient>>,) -> RunnerResult<(ClientRouter, Option<CallerAuthKind>)> {let TargetPromptPolicy {prompts,@@ -368,7 +445,20 @@ impl DeploymentConfig {let mut caller_auth = None;let mut has_mixed_families = false;let mut forwarding_origins = BTreeSet::new();+ let mut decisions_by_model = HashMap::new();for name in route.callable_target_names() {+ if route+ .algorithm+ .decision_judge()+ .is_some_and(|(judge, _)| judge == name)+ {+ let target = &self.decision_targets[name];+ decisions_by_model.insert(+ target.id.clone(),+ decision_clients[&target.decision_client].clone(),+ );+ continue;+ }let target = self.targets.get(name).ok_or_else(|| {RunnerError::configuration(format!("route references unknown target {name}"))})?;@@ -417,8 +507,9 @@ impl DeploymentConfig {.into_iter().map(|name| self.targets[name].id.clone()).collect::<Vec<_>>();- let router = ClientRouter::new_with_completion_targets(+ let router = ClientRouter::new_with_decision_clients(by_model,+ decisions_by_model,prompts,routing_answer_target,&completion_targets,@@ -599,6 +690,23 @@ struct LlmClientConfig {timeout_ms: Option<u64>,}+#[derive(Debug, Deserialize)]+#[serde(tag = "format", rename_all = "snake_case", deny_unknown_fields)]+enum DecisionClientConfig {+ SystemOne {+ endpoint: HttpBaseUrl,+ api_key_env: String,+ timeout_ms: u64,+ },+}++#[derive(Debug, Deserialize)]+#[serde(deny_unknown_fields)]+struct DecisionTargetConfig {+ id: ModelId,+ decision_client: String,+}+#[derive(Debug, Deserialize)]#[serde(deny_unknown_fields)]struct TargetConfig {@@ -691,24 +799,7 @@ fn build_backend(let api_key = config.api_key_env.as_deref()- .map(|variable| {- if variable.trim().is_empty() {- return Err(RunnerError::configuration(format!(- "llm client {client_name} api_key_env must not be empty"- )));- }- let api_key = std::env::var(variable).map_err(|error| {- RunnerError::configuration(format!(- "llm client {client_name} could not read api_key_env {variable}: {error}"- ))- })?;- if api_key.trim().is_empty() {- return Err(RunnerError::configuration(format!(- "llm client {client_name} api_key_env {variable} is empty"- )));- }- Ok(api_key)- })+ .map(|variable| read_api_key(&format!("llm client {client_name}"), variable)).transpose()?;let http = HttpBackendConfig {base_url: config.base_url.as_str().to_string(),@@ -730,6 +821,25 @@ fn build_backend(Ok(backend)}+fn read_api_key(client: &str, variable: &str) -> RunnerResult<String> {+ if variable.trim().is_empty() {+ return Err(RunnerError::configuration(format!(+ "{client} api_key_env must not be empty"+ )));+ }+ let api_key = std::env::var(variable).map_err(|error| {+ RunnerError::configuration(format!(+ "{client} could not read api_key_env {variable}: {error}"+ ))+ })?;+ if api_key.trim().is_empty() {+ return Err(RunnerError::configuration(format!(+ "{client} api_key_env {variable} is empty"+ )));+ }+ Ok(api_key)+}+// A function so that serde default can use it.const fn default_max_retries() -> u32 {DEFAULT_MAX_RETRIESdiff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rsindex a488b31a7..115e8ce83 100644--- a/crates/switchyard-runner/src/lib.rs+++ b/crates/switchyard-runner/src/lib.rs@@ -12,7 +12,8 @@ mod runner;pub use algorithm::{AdvisorTriggerConfig, AlgorithmConfigError, AlgorithmSpec, CategoryModelConfig, ClassifierMode,- ClassifierPolicyConfig, LlmClassifierRouteConfig, StageClassifierConfig, SubagentRouteConfig,+ ClassifierPolicyConfig, DecisionJudgeRouteConfig, LlmClassifierRouteConfig,+ StageClassifierConfig, SubagentRouteConfig,};pub use failure::{RouteErrorKind, RouteErrorPhase, RouteErrorSummary, stream_error_summary};// Re-exported because `Route::new` takes it, so a host wiring routes does not need a libsy dep.diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.mdindex f7458bc95..275718ece 100644--- a/crates/switchyard-server/CONFIGURATION.md+++ b/crates/switchyard-server/CONFIGURATION.md@@ -32,6 +32,47 @@ outside normal assistant `content`.To support another wire format, add its `ClientFormat` variant and explicit construction match in`../switchyard-runner/src/config.rs`. Add a client type only when a second implementation exists.+## Use a decision model as the capability judge++With `quality` and `economy` already defined as LLM targets:++```toml+[decision_clients.jev]+format = "system_one"+endpoint = "https://api.typesafe.ai/v1/systemone"+api_key_env = "TYPESAFE_API_KEY"+timeout_ms = 5000++[decision_targets.judge]+id = "jev-latest"+decision_client = "jev"++[routes.capability]+id = "switchyard/capability"+type = "llm_classifier"+mode = "capability"+classifier_target = "judge"+strong_target = "quality"+weak_target = "economy"++[routes.capability.decision]+cutoff = 0.4+candidates = { a = "quality", b = "economy" }+evidence = { candidates = { a = "Higher-quality model", b = "Lower-cost model" } }+```++- `endpoint` is the full System One URL. Set the named key environment variable+ before startup; caller credentials are not forwarded to the decision provider.+- Candidate labels must cover both tiers. Extra candidates add context, not routes.+ Supply evidence from your evaluations and choose a cutoff using those results.+- `decision.instructions` can replace the default relative-advantage instructions.+ It must preserve the meaning of `advantage` and `no_advantage`. Evidence is passed unchanged.+- The capable tier is selected only when its advantage score exceeds `cutoff`.+ Client failures and deadlines use the capable tier by default; set `fail_open = false`+ on the route to propagate those errors.+- Decision targets cannot serve final answers. Both normal requests and `/v1/decision`+ use this configuration. Omit `decision` to keep the existing LLM judge settings.+## Add an algorithm1. Implement and export the algorithm from `libsy`.