Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): make the unmatched verdict threshold step configurable (#846)
Hiroshi MORISHIGE3fbfdff83389
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 189fc9c3d..2bd98cbef 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -77,10 +77,15 @@ impl TaskClassifierVerdict {}/// Returns the number of threshold steps assigned to this capability boundary.- fn boundary_steps(&self) -> Option<u8> {+ ///+ /// `unmatched_steps` comes from the route configuration: a request that matches no+ /// capability rule has no evidence behind its solve probability, so operators can+ /// require more confidence for it than for an uncertain verdict.+ fn boundary_steps(&self, unmatched_steps: u8) -> Option<u8> {match self.capability_boundary.as_str() {"supported" => Some(0),- "uncertain" | "unmatched" => Some(1),+ "uncertain" => Some(1),+ "unmatched" => Some(unmatched_steps),"unsupported" => Some(2),_ => None,}@@ -232,6 +237,7 @@ impl ClassifierInput for TaskInput {struct TaskClassifierPolicy {base_threshold: f64,threshold_step: f64,+ unmatched_steps: u8,}impl TaskClassifierPolicy {@@ -239,12 +245,14 @@ impl TaskClassifierPolicy {Self {base_threshold: config.base_threshold,threshold_step: config.threshold_step,+ unmatched_steps: config.unmatched_steps,}}/// Returns the required solve probability for one validated verdict.fn threshold(&self, verdict: &TaskClassifierVerdict) -> Option<f64> {- Some(self.base_threshold + f64::from(verdict.boundary_steps()?) * self.threshold_step)+ let steps = verdict.boundary_steps(self.unmatched_steps)?;+ Some(self.base_threshold + f64::from(steps) * self.threshold_step)}}@@ -346,8 +354,12 @@ pub struct LlmCapabilityConfig {/// Lowest solve probability that routes a supported task to the efficient target.pub base_threshold: f64,/// Amount added per capability-boundary step: zero for supported, one for- /// uncertain or unmatched, and two for unsupported.+ /// uncertain, `unmatched_steps` for unmatched, and two for unsupported.pub threshold_step: f64,+ /// Threshold steps applied to an unmatched verdict (no capability rule applies).+ ///+ /// Defaults to 1, the same as an uncertain verdict. 2 treats it like unsupported.+ pub unmatched_steps: u8,/// Prompt and verdict contract settings for the classifier judge.pub contract: ClassifierContractConfig,/// Maximum completion tokens available to the classifier verdict.@@ -359,6 +371,7 @@ impl Default for LlmCapabilityConfig {Self {base_threshold: 0.0,threshold_step: 0.0,+ unmatched_steps: DEFAULT_UNMATCHED_STEPS,contract: ClassifierContractConfig::default(),max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,}@@ -374,6 +387,8 @@ struct TaskClassifierConfigWire {fail_open: bool,#[serde(default)]threshold_step: f64,+ #[serde(default = "default_unmatched_steps")]+ unmatched_steps: u8,#[serde(default)]classify_trigger: ClassifyTrigger,#[serde(default)]@@ -403,6 +418,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig {judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {base_threshold: wire.base_threshold,threshold_step: wire.threshold_step,+ unmatched_steps: wire.unmatched_steps,contract,max_output_tokens: wire.max_output_tokens,}),@@ -418,6 +434,13 @@ const fn default_fail_open() -> bool {true}+/// Unmatched verdicts share the uncertain step unless the route says otherwise.+pub const DEFAULT_UNMATCHED_STEPS: u8 = 1;++const fn default_unmatched_steps() -> u8 {+ DEFAULT_UNMATCHED_STEPS+}+const fn default_judge_max_output_tokens() -> u64 {DEFAULT_JUDGE_MAX_OUTPUT_TOKENS}@@ -452,6 +475,14 @@ impl LlmCapabilityConfig {),});}+ if self.unmatched_steps > 2 {+ return Err(LibsyError::AlgorithmError {+ message: format!(+ "unmatched_steps must be 0, 1, or 2, got {}",+ self.unmatched_steps+ ),+ });+ }let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step;if unsupported_threshold > 1.0 && unsupported_threshold - 1.0 > f64::EPSILON {return Err(LibsyError::AlgorithmError {@@ -1723,6 +1754,55 @@ mod tests {Ok(())}+ #[test]+ fn unmatched_steps_raise_only_the_unmatched_threshold() -> Result<()> {+ let policy = TaskClassifierPolicy::new(&LlmCapabilityConfig {+ threshold_step: 0.1,+ unmatched_steps: 2,+ ..llm_config(0.75)+ });++ // 0.85 clears one step but not two: unmatched now routes to the capable target.+ assert_eq!(+ selected(&policy, Some(&verdict(0.85, "unmatched", "none")))?,+ "capable"+ );+ assert_eq!(+ selected(&policy, Some(&verdict(0.95, "unmatched", "none")))?,+ "efficient"+ );+ // Other boundaries keep their steps.+ assert_eq!(+ selected(&policy, Some(&verdict(0.85, "uncertain", "UNC-1")))?,+ "efficient"+ );+ assert_eq!(+ selected(&policy, Some(&verdict(0.75, "supported", "SUP-2")))?,+ "efficient"+ );+ Ok(())+ }++ #[test]+ fn unmatched_steps_default_to_one_and_reject_values_above_two() {+ let parsed: TaskClassifierConfig =+ serde_json::from_value(serde_json::json!({ "base_threshold": 0.5 }))+ .expect("config without unmatched_steps parses");+ let CapabilityJudgeConfig::Llm(judge) = &parsed.judge else {+ panic!("a flat capability config builds an LLM judge");+ };+ assert_eq!(judge.unmatched_steps, 1);++ let invalid = TaskClassifierConfig {+ judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {+ unmatched_steps: 3,+ ..llm_config(0.5)+ }),+ ..TaskClassifierConfig::default()+ };+ assert!(invalid.validate().is_err());+ }+/// The text of each message a judge with `recent_turn_window` would be sent./// The no-window case is covered by `capability_judge_builds_a_structured_request`.fn capability_judge(recent_turn_window: Option<usize>) -> Result<CapabilityJudge> {diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 6c5711555..1ebc94567 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -22,8 +22,9 @@ pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger};pub use algorithms::composite::{CompositeJudgeConfig, CompositeRouter, CompositeRouterConfig};pub use algorithms::infra::InfraPolicy;pub use algorithms::llm_class::{- CapabilityJudgeConfig, CustomClassifierConfig, CustomClassifierPolicy, DecisionJudgeConfig,- LlmCapabilityConfig, LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig,+ CapabilityJudgeConfig, CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS,+ DecisionJudgeConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmTaskClassifier,+ TaskClassifierConfig,};pub use algorithms::noop::Noop;pub use algorithms::passthrough::Passthrough;diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 3a307dec9..5fc7f5230 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -15,10 +15,11 @@ use serde_json::Value;use switchyard_libsy::{Algorithm, CallDecision, CallModel, CapabilityJudgeConfig, ClassifierContractConfig,ClassifierResponseFormat, ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy,- DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig, HandoffNoteConfig,- LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback,- LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter,- StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics,+ DEFAULT_UNMATCHED_STEPS, DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig,+ HandoffNoteConfig, LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig,+ LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels,+ StageRouter, StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig,+ ToolSemantics,};use switchyard_protocol::{Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,@@ -371,6 +372,7 @@ impl PyTaskClassifierConfig {judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {base_threshold,threshold_step,+ unmatched_steps: DEFAULT_UNMATCHED_STEPS,contract: classifier_contract(prompt, response_format_type)?,max_output_tokens,}),diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rsindex c7d0882dc..c2ca4cc3a 100644--- a/crates/switchyard-runner/src/algorithm.rs+++ b/crates/switchyard-runner/src/algorithm.rs@@ -13,11 +13,11 @@ use libsy::{AdvisorGate, AdvisorGateConfig, Algorithm, BreakerConfig, CapabilityJudgeConfig,ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeJudgeConfig,CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy,- DecisionJudgeConfig, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig,- LlmCapabilityConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough,- PickerMode, PlanExecute, PlanExecuteConfig, Random, RuntimeModels, StageRouter,- StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics,- Vgr, VgrConfig,+ DEFAULT_UNMATCHED_STEPS, DecisionJudgeConfig, EscalationJudgeConfig, GateTrigger,+ HandoffNoteConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier,+ Noop, Passthrough, PickerMode, PlanExecute, PlanExecuteConfig, Random, RuntimeModels,+ StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig,+ ToolSemantics, Vgr, VgrConfig,};use serde::Deserialize;use serde_json::Value;@@ -301,6 +301,9 @@ pub struct LlmClassifierRouteConfig {/// Capability mode: how much to raise the threshold when the judge is/// uncertain. Added once for an uncertain verdict and twice for unsupported.pub threshold_step: Option<f64>,+ /// Capability mode: threshold steps for an unmatched verdict (no capability rule+ /// applies). 1 by default, the same as uncertain; 2 treats it like unsupported.+ pub unmatched_steps: Option<u8>,/// Capability mode: routes to the capable tier on judge client failures and deadlines./// Defaults to true when capability mode is selected.pub fail_open: Option<bool>,@@ -1181,6 +1184,7 @@ impl LlmClassifierRouteConfig {weak_target,base_threshold,threshold_step,+ unmatched_steps,fail_open,classify_trigger,message_hash_fallback,@@ -1234,6 +1238,7 @@ impl LlmClassifierRouteConfig {let judge = if let Some(decision) = decision {if base_threshold.is_some()|| threshold_step.is_some()+ || unmatched_steps.is_some()|| prompt.is_some()|| *response_format_type != ClassifierResponseFormat::JsonSchema|| *max_output_tokens != default_classifier_max_output_tokens()@@ -1251,6 +1256,7 @@ impl LlmClassifierRouteConfig {base_threshold,)?,threshold_step: threshold_step.unwrap_or_default(),+ unmatched_steps: unmatched_steps.unwrap_or(DEFAULT_UNMATCHED_STEPS),contract: classifier_contract(prompt.as_deref()).with_response_format_type(*response_format_type),max_output_tokens: *max_output_tokens,@@ -1296,7 +1302,10 @@ impl LlmClassifierRouteConfig {|| fail_open.is_some()|| *message_hash_fallback|| recent_turn_window.is_some();- if mode.is_some() && has_capability_settings {+ // `unmatched_steps` is new, so no existing escalation configuration carries it:+ // reject it even when the mode is implied by `escalation`. The older capability+ // keys stay tolerated in that implicit form for compatibility.+ if unmatched_steps.is_some() || (mode.is_some() && has_capability_settings) {return Err(AlgorithmConfigError::new(format!("llm_classifier route {route_name} mode escalation cannot use capability routing settings")));@@ -1334,6 +1343,7 @@ impl LlmClassifierRouteConfig {|| weak_target.is_some()|| base_threshold.is_some()|| threshold_step.is_some()+ || unmatched_steps.is_some()|| escalation.is_some()|| *response_format_type != ClassifierResponseFormat::JsonSchema{diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex c01793053..285009d8f 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -1748,6 +1748,34 @@ classify_trigger"#,Ok(())}+ #[test]+ fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> {+ // Accepted on a capability route; the value range is checked by the algorithm.+ runner_from_toml(&VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nunmatched_steps = 2",+ ))?;+ assert!(+ error_message(&VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nunmatched_steps = 3",+ ))+ .contains("unmatched_steps must be 0, 1, or 2")+ );++ // Rejected on an escalation route, including the implicit form where the+ // `escalation` table alone selects the mode: the key is new, so no existing+ // configuration relies on it being ignored there.+ assert!(+ error_message(&VALID_CONFIG.replace(+ "base_threshold = 0.5",+ "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }",+ ))+ .contains("mode escalation cannot use capability routing settings")+ );+ Ok(())+ }+#[test]fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> {let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";diff --git a/crates/switchyard-runner/tests/algorithm_factory.rs b/crates/switchyard-runner/tests/algorithm_factory.rsindex 6b4c1363b..0ee584d3c 100644--- a/crates/switchyard-runner/tests/algorithm_factory.rs+++ b/crates/switchyard-runner/tests/algorithm_factory.rs@@ -141,6 +141,13 @@ fn invalid_routing_configuration_is_rejected() {),"bogus",),+ (+ DEPLOYMENT.replace(+ "weak_target = \"fast\"\n",+ "weak_target = \"fast\"\nunmatched_steps = 2\n",+ ),+ "decision cannot use LLM judge settings",+ ),] {let error = AlgorithmFactory::from_toml(&source).and_then(|factory| factory.build())diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex c3edb7454..9b1a74d65 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -178,7 +178,8 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts| Key | Default | Meaning ||---|---|---|| `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. |-| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |+| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |+| `unmatched_steps` | `1` | Threshold steps for an unmatched verdict. `0`, `1`, or `2`; `2` treats it like unsupported. || `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request including tool continuations, `user_turn` judges each new user message and holds that target across the tool calls between, `new_session` judges once and reuses that target for the session. || `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `classify_trigger = "new_session"` or `"user_turn"`. |diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 4bd97fb19..cc8820264 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -351,7 +351,8 @@ Capability mode classifies before serving. See| `weak_target` | Yes | — | Efficient tier. || `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. || `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid verdicts use the capable tier with either setting. |-| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |+| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |+| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. || `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. || `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. || `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.mdindex 45a1a6a0e..e43225b19 100644--- a/docs/routing_algorithms/llm_classifier_routing.md+++ b/docs/routing_algorithms/llm_classifier_routing.md@@ -71,7 +71,8 @@ greater than or equal to the applicable threshold. Otherwise it routes to`strong_target`:- `supported` uses `base_threshold`.-- `uncertain` and `unmatched` use `base_threshold + threshold_step`.+- `uncertain` uses `base_threshold + threshold_step`.+- `unmatched` uses `base_threshold + unmatched_steps * threshold_step` (`unmatched_steps` defaults to `1`).- `unsupported` uses `base_threshold + 2 * threshold_step`.An invalid, inconsistent, or unparseable verdict routes to@@ -122,6 +123,7 @@ for the server merge behavior.|---|---|---|| `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. || `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. |+| `unmatched_steps` | `1` | Boundary steps applied to an `unmatched` verdict. `0`, `1`, or `2`; `2` requires the same confidence as an `unsupported` verdict. || `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. || `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. || `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. |