Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(routing): support decision models in composite routes (#981)
ryan-lempka7fa4b00cff5a
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex 94b2a54a9..646b37a43 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -110,6 +110,49 @@ struct CapabilityClassifierRouteConfig {recent_turn_window: Option<usize>,}+impl CapabilityClassifierRouteConfig {+ fn into_task_config(+ self,+ route_name: &str,+ targets: &BTreeMap<String, ModelId>,+ ) -> AlgorithmResult<TaskClassifierConfig> {+ let config = self;+ let judge = match config.judge {+ CapabilityJudgeRouteConfig::Llm(judge) => CapabilityJudgeConfig::Llm(judge),+ CapabilityJudgeRouteConfig::Decision(judge) => {+ for target in [&config.strong_target, &config.weak_target] {+ if !judge.candidates.values().any(|name| name == target) {+ return Err(AlgorithmConfigError::new(format!(+ "route {route_name}: decision.candidates must include target {target}"+ )));+ }+ }+ let candidates = judge+ .candidates+ .into_iter()+ .map(|(label, name)| {+ resolve_target_model_id(route_name, &name, targets)+ .map(|model| (label, model))+ })+ .collect::<AlgorithmResult<_>>()?;+ CapabilityJudgeConfig::Decision(DecisionJudgeConfig {+ cutoff: judge.cutoff,+ instructions: judge.instructions,+ candidates,+ evidence: judge.evidence,+ })+ }+ };+ Ok(TaskClassifierConfig {+ judge,+ fail_open: config.fail_open,+ classify_trigger: config.classify_trigger,+ message_hash_fallback: config.message_hash_fallback,+ recent_turn_window: config.recent_turn_window,+ })+ }+}+#[derive(Clone, Debug)]enum CapabilityJudgeRouteConfig {Llm(LlmCapabilityConfig),@@ -490,12 +533,16 @@ pub enum AdvisorTriggerConfig {pub struct StageClassifierConfig {/// Target the judge is called through. Not a routing destination.pub target: String,+ #[serde(default)]+ pub decision: Option<DecisionJudgeRouteConfig>,+ #[serde(default)]+ pub fail_open: Option<bool>,/// Lowest solve probability that still routes to the efficient tier, from 0 to 1.- pub base_threshold: f64,+ pub base_threshold: Option<f64>,/// How much to raise the threshold when the judge is uncertain. Added once/// for an uncertain verdict and twice for unsupported.#[serde(default)]- pub threshold_step: f64,+ pub threshold_step: Option<f64>,/// How often the judge runs. `new_session` has no effect here.#[serde(default)]pub classify_trigger: ClassifyTrigger,@@ -547,19 +594,33 @@ pub struct StageTierConfig {}impl StageClassifierConfig {- fn task_classifier_config(&self) -> TaskClassifierConfig {- TaskClassifierConfig {- judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {- base_threshold: self.base_threshold,- threshold_step: self.threshold_step,- contract: classifier_contract(self.prompt.as_deref())- .with_response_format_type(self.response_format_type),- max_output_tokens: self.max_output_tokens,- }),- fail_open: true,+ fn task_classifier_config(+ &self,+ route_name: &str,+ tiers: &StageTierConfig,+ targets: &BTreeMap<String, ModelId>,+ ) -> AlgorithmResult<TaskClassifierConfig> {+ let config = LlmClassifierRouteConfig {+ classifier_target: self.target.clone(),+ strong_target: Some(tiers.capable_target.clone()),+ weak_target: Some(tiers.efficient_target.clone()),+ base_threshold: self.base_threshold,+ threshold_step: self.threshold_step,+ fail_open: self.fail_open,+ decision: self.decision.clone(),classify_trigger: self.classify_trigger,message_hash_fallback: self.message_hash_fallback,recent_turn_window: self.recent_turn_window,+ prompt: self.prompt.clone(),+ response_format_type: self.response_format_type,+ max_output_tokens: self.max_output_tokens,+ ..Default::default()+ };+ match config.validated_classifier_mode(route_name)? {+ LlmClassifierModeConfig::Capability(config) => {+ config.into_task_config(route_name, targets)+ }+ _ => Err(AlgorithmConfigError::new("expected a capability judge")),}}}@@ -586,6 +647,14 @@ impl AlgorithmSpec {.decision.as_ref().map(|decision| (config.classifier_target.as_str(), decision)),+ Self::Composite { classifier, .. }+ | Self::StageRouter {+ classifier: Some(classifier),+ ..+ } => classifier+ .decision+ .as_ref()+ .map(|decision| (classifier.target.as_str(), decision)),_ => None,}}@@ -1330,39 +1399,7 @@ fn build_algorithm(let mode = classifier_config.validated_classifier_mode(route_name)?;let algorithm = match mode {LlmClassifierModeConfig::Capability(config) => {- let judge = match config.judge {- CapabilityJudgeRouteConfig::Llm(judge) => CapabilityJudgeConfig::Llm(judge),- CapabilityJudgeRouteConfig::Decision(judge) => {- for target in [&config.strong_target, &config.weak_target] {- if !judge.candidates.values().any(|name| name == target) {- return Err(AlgorithmConfigError::new(format!(- "llm_classifier route {route_name}: decision.candidates must include target {target}"- )));- }- }- let candidates = judge- .candidates- .into_iter()- .map(|(label, name)| {- resolve_target_model_id(route_name, &name, targets)- .map(|model| (label, model))- })- .collect::<AlgorithmResult<_>>()?;- CapabilityJudgeConfig::Decision(DecisionJudgeConfig {- cutoff: judge.cutoff,- instructions: judge.instructions,- candidates,- evidence: judge.evidence,- })- }- };- let classifier_config = TaskClassifierConfig {- judge,- fail_open: config.fail_open,- classify_trigger: config.classify_trigger,- message_hash_fallback: config.message_hash_fallback,- recent_turn_window: config.recent_turn_window,- };+ let classifier_config = config.into_task_config(route_name, targets)?;LlmTaskClassifier::new(LlmClassifierConfig::Capability {config: classifier_config,})@@ -1439,9 +1476,14 @@ fn build_algorithm(config.handoff_notes = handoff_notes.clone();// The judge is called through its own target, so it is not a routing// destination and stays out of the tier pair.- config.llm_fallback = classifier.as_ref().map(|classifier| LlmFallback {- config: classifier.task_classifier_config(),- });+ config.llm_fallback = classifier+ .as_ref()+ .map(|classifier| {+ classifier+ .task_classifier_config(route_name, tiers, targets)+ .map(|config| LlmFallback { config })+ })+ .transpose()?;let algorithm = StageRouter::new(config).map_err(|error| {AlgorithmConfigError::with_source(format!("stage_router route {route_name}: {error}"),@@ -1475,7 +1517,7 @@ fn build_algorithm(stage_config.tool_semantics = stage.tool_semantics.clone();stage_config.handoff_notes = stage.handoff_notes.clone();let config = CompositeRouterConfig {- judge: classifier.task_classifier_config(),+ judge: classifier.task_classifier_config(route_name, stage, targets)?,stage: stage_config,};let algorithm = CompositeRouter::new(config).map_err(|error| {diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex d0cd12286..e8918efe0 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -1191,6 +1191,39 @@ new = ["send_message"])}+ #[test]+ fn composite_decision_judge_reuses_capability_validation() {+ let source = composite_config().replace(+ "[routes.composed.classifier]\ntarget = \"tier_judge\"\nbase_threshold = 0.5",+ "[routes.composed.classifier]\ntarget = \"tier_judge\"",+ ) + r#"+[routes.composed.classifier.decision]+cutoff = 0.4+candidates = { a = "strong", b = "weak" }+evidence = { description = "candidate comparison" }+"#;+ for (source, valid) in [+ (source.clone(), true),+ (+ source.replace(+ "target = \"tier_judge\"",+ "target = \"tier_judge\"\nbase_threshold = 0.5",+ ),+ false,+ ),+ (source.replace("b = \"weak\"", "b = \"strong\""), false),+ ] {+ let deployment: DeploymentConfig = toml::from_str(&source).unwrap();+ let spec = &deployment.routes["composed"].algorithm;+ assert_eq!(spec.decision_judge().unwrap().0, "tier_judge");+ assert_eq!(+ spec.build_with_runtime_models("composed", &deployment.build_targets())+ .is_ok(),+ valid+ );+ }+ }+#[test]fn composite_route_builds_and_claims_both_tiers_and_its_judge() -> RunnerResult<()> {let runner = runner_from_toml(&composite_config())?;diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 91f1f90f2..dd0ca0765 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -391,9 +391,11 @@ configuration. Today a classifier sets the tier a stage router falls open to whe| Key | Required | Default | Meaning ||---|:---:|---|---|| `classifier.target` | Yes | — | Target the tier judge is called through. Not a routing destination. |-| `classifier.base_threshold` | Yes | — | `p_solve` floor that still routes to the efficient tier. In `[0, 1]`. |+| `classifier.base_threshold` | For LLM judges | unset | `p_solve` floor that still routes to the efficient tier. In `[0, 1]`. Omit with `classifier.decision`. || `classifier.threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. || `classifier.classify_trigger` | Yes | — | `user_turn` re-picks the tier whenever the user speaks, `new_session` picks once and holds it. `every_request` is rejected here: a judge call per tool step is the cost this route exists to avoid. |+| `classifier.decision` | No | unset | Use a decision model instead of an LLM judge. The classifier target must be in `decision_targets`. |+| `classifier.fail_open` | No | `true` | On judge failure, retain the last classified tier or default to efficient. Set `false` to return the error. || `classifier.message_hash_fallback` | No | `false` | Retains the tier by hashing the first user message, for clients that send no session ID. Works with either supported trigger. Conversations opening with the same text share a tier. || `classifier.recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. || `classifier.prompt` | No | packaged prompt | Replaces the classifier judge prompt. The verdict schema is unchanged. |@@ -411,6 +413,10 @@ configuration. Today a classifier sets the tier a stage router falls open to whe| `stage.handoff_notes` | No | unset | Optional guidance appended to forwarded requests. Uses the same fields and behavior as [stage-router handoff notes](../routing_algorithms/stage_router_routing.md#optional-handoff-notes). || `subagents` | No | unset | Nested policy used only for delegated sub-agent work. |+For decision classifier configuration, see+[Decision model classifier](../routing_algorithms/composite_routing.md#decision-model-classifier).+The same decision settings are accepted by the optional `stage_router` classifier.+The tier is retained per session. A deployment that sends no session ID needs`classifier.message_hash_fallback = true`, which keys on the first user messageinstead. The outer `[routes.<name>.classifier]` table is required. The nesteddiff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.mdindex e70e96c14..aeab7e976 100644--- a/docs/routing_algorithms/composite_routing.md+++ b/docs/routing_algorithms/composite_routing.md@@ -7,12 +7,16 @@ fit the same route type.## What exists today-One pairing: an LLM classifier sets the tier a stage router falls open to when+One pairing: a classifier sets the tier a stage router falls open to whenits own signals are not confident. The stage router's `picker` mode is unchanged,and so is how it scores signals, escalates, and de-escalates. Both algorithms and therelationship between them are fixed, and the configuration below names themdirectly rather than composing arbitrary algorithms.+The classifier can use an LLM judge or a decision model. A decision+model is configured through a decision client and target, separately from the+capable and efficient completion targets.+The general form, where any algorithm can target another and set itsconfiguration, is the direction rather than the current behaviour. Expect thisroute's configuration to change as more pairings are added.@@ -49,6 +53,39 @@ nor a second `classifier`. The outer classifier supplies the default tier.Custom `tool_semantics` remain stage-local and have the same additive behavior ason a standalone stage route.+## Decision model classifier++Use a decision target as the classifier. Keep the stage block above+and replace its LLM classifier settings with:++```toml+[decision_clients.decisions]+format = "system_one"+endpoint = "https://openrouter.ai/api/alpha/decisions"+api_key_env = "OPENROUTER_API_KEY"+timeout_ms = 5000++[decision_targets.judge]+id = "inception/mercury-decide:free"+decision_client = "decisions"++[routes.switchyard.classifier]+target = "judge"+classify_trigger = "user_turn"++[routes.switchyard.classifier.decision]+cutoff = 0.4+candidates = { a = "strong", b = "weak" }+evidence = { description = "Add task-relevant evidence comparing a and b." }+```++Both candidates must be completion targets. The judge must be a decision target.+The classifier selects capable when `probabilities.advantage` exceeds `cutoff`.+Calibrate the cutoff for your tasks and models. Omit LLM-only settings such as+`base_threshold`, `threshold_step` and `prompt` when using a decision model.+On judge failure, the default is to retain the last classified tier or use+efficient for a new session. Set `classifier.fail_open = false` to return the error.+## SessionsThe tier is retained per session, so send `x-switchyard-session-id` and each