Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): make the unmatched verdict threshold step configurable (#846)

Hiroshi MORISHIGE3fbfdff83389
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 189fc9c3d..2bd98cbef 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -77,10 +77,15 @@ impl TaskClassifierVerdict {
}
/// Returns the number of threshold steps assigned to this capability boundary.
- fn boundary_steps(&self) -> Option<u8> {
+ ///
+ /// `unmatched_steps` comes from the route configuration: a request that matches no
+ /// capability rule has no evidence behind its solve probability, so operators can
+ /// require more confidence for it than for an uncertain verdict.
+ fn boundary_steps(&self, unmatched_steps: u8) -> Option<u8> {
match self.capability_boundary.as_str() {
"supported" => Some(0),
- "uncertain" | "unmatched" => Some(1),
+ "uncertain" => Some(1),
+ "unmatched" => Some(unmatched_steps),
"unsupported" => Some(2),
_ => None,
}
@@ -232,6 +237,7 @@ impl ClassifierInput for TaskInput {
struct TaskClassifierPolicy {
base_threshold: f64,
threshold_step: f64,
+ unmatched_steps: u8,
}
impl TaskClassifierPolicy {
@@ -239,12 +245,14 @@ impl TaskClassifierPolicy {
Self {
base_threshold: config.base_threshold,
threshold_step: config.threshold_step,
+ unmatched_steps: config.unmatched_steps,
}
}
/// Returns the required solve probability for one validated verdict.
fn threshold(&self, verdict: &TaskClassifierVerdict) -> Option<f64> {
- Some(self.base_threshold + f64::from(verdict.boundary_steps()?) * self.threshold_step)
+ let steps = verdict.boundary_steps(self.unmatched_steps)?;
+ Some(self.base_threshold + f64::from(steps) * self.threshold_step)
}
}
@@ -346,8 +354,12 @@ pub struct LlmCapabilityConfig {
/// Lowest solve probability that routes a supported task to the efficient target.
pub base_threshold: f64,
/// Amount added per capability-boundary step: zero for supported, one for
- /// uncertain or unmatched, and two for unsupported.
+ /// uncertain, `unmatched_steps` for unmatched, and two for unsupported.
pub threshold_step: f64,
+ /// Threshold steps applied to an unmatched verdict (no capability rule applies).
+ ///
+ /// Defaults to 1, the same as an uncertain verdict. 2 treats it like unsupported.
+ pub unmatched_steps: u8,
/// Prompt and verdict contract settings for the classifier judge.
pub contract: ClassifierContractConfig,
/// Maximum completion tokens available to the classifier verdict.
@@ -359,6 +371,7 @@ impl Default for LlmCapabilityConfig {
Self {
base_threshold: 0.0,
threshold_step: 0.0,
+ unmatched_steps: DEFAULT_UNMATCHED_STEPS,
contract: ClassifierContractConfig::default(),
max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
}
@@ -374,6 +387,8 @@ struct TaskClassifierConfigWire {
fail_open: bool,
#[serde(default)]
threshold_step: f64,
+ #[serde(default = "default_unmatched_steps")]
+ unmatched_steps: u8,
#[serde(default)]
classify_trigger: ClassifyTrigger,
#[serde(default)]
@@ -403,6 +418,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: wire.base_threshold,
threshold_step: wire.threshold_step,
+ unmatched_steps: wire.unmatched_steps,
contract,
max_output_tokens: wire.max_output_tokens,
}),
@@ -418,6 +434,13 @@ const fn default_fail_open() -> bool {
true
}
+/// Unmatched verdicts share the uncertain step unless the route says otherwise.
+pub const DEFAULT_UNMATCHED_STEPS: u8 = 1;
+
+const fn default_unmatched_steps() -> u8 {
+ DEFAULT_UNMATCHED_STEPS
+}
+
const fn default_judge_max_output_tokens() -> u64 {
DEFAULT_JUDGE_MAX_OUTPUT_TOKENS
}
@@ -452,6 +475,14 @@ impl LlmCapabilityConfig {
),
});
}
+ if self.unmatched_steps > 2 {
+ return Err(LibsyError::AlgorithmError {
+ message: format!(
+ "unmatched_steps must be 0, 1, or 2, got {}",
+ self.unmatched_steps
+ ),
+ });
+ }
let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step;
if unsupported_threshold > 1.0 && unsupported_threshold - 1.0 > f64::EPSILON {
return Err(LibsyError::AlgorithmError {
@@ -1723,6 +1754,55 @@ mod tests {
Ok(())
}
+ #[test]
+ fn unmatched_steps_raise_only_the_unmatched_threshold() -> Result<()> {
+ let policy = TaskClassifierPolicy::new(&LlmCapabilityConfig {
+ threshold_step: 0.1,
+ unmatched_steps: 2,
+ ..llm_config(0.75)
+ });
+
+ // 0.85 clears one step but not two: unmatched now routes to the capable target.
+ assert_eq!(
+ selected(&policy, Some(&verdict(0.85, "unmatched", "none")))?,
+ "capable"
+ );
+ assert_eq!(
+ selected(&policy, Some(&verdict(0.95, "unmatched", "none")))?,
+ "efficient"
+ );
+ // Other boundaries keep their steps.
+ assert_eq!(
+ selected(&policy, Some(&verdict(0.85, "uncertain", "UNC-1")))?,
+ "efficient"
+ );
+ assert_eq!(
+ selected(&policy, Some(&verdict(0.75, "supported", "SUP-2")))?,
+ "efficient"
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn unmatched_steps_default_to_one_and_reject_values_above_two() {
+ let parsed: TaskClassifierConfig =
+ serde_json::from_value(serde_json::json!({ "base_threshold": 0.5 }))
+ .expect("config without unmatched_steps parses");
+ let CapabilityJudgeConfig::Llm(judge) = &parsed.judge else {
+ panic!("a flat capability config builds an LLM judge");
+ };
+ assert_eq!(judge.unmatched_steps, 1);
+
+ let invalid = TaskClassifierConfig {
+ judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
+ unmatched_steps: 3,
+ ..llm_config(0.5)
+ }),
+ ..TaskClassifierConfig::default()
+ };
+ assert!(invalid.validate().is_err());
+ }
+
/// The text of each message a judge with `recent_turn_window` would be sent.
/// The no-window case is covered by `capability_judge_builds_a_structured_request`.
fn capability_judge(recent_turn_window: Option<usize>) -> Result<CapabilityJudge> {
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 6c5711555..1ebc94567 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -22,8 +22,9 @@ pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger};
pub use algorithms::composite::{CompositeJudgeConfig, CompositeRouter, CompositeRouterConfig};
pub use algorithms::infra::InfraPolicy;
pub use algorithms::llm_class::{
- CapabilityJudgeConfig, CustomClassifierConfig, CustomClassifierPolicy, DecisionJudgeConfig,
- LlmCapabilityConfig, LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig,
+ CapabilityJudgeConfig, CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS,
+ DecisionJudgeConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmTaskClassifier,
+ TaskClassifierConfig,
};
pub use algorithms::noop::Noop;
pub use algorithms::passthrough::Passthrough;
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 3a307dec9..5fc7f5230 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -15,10 +15,11 @@ use serde_json::Value;
use switchyard_libsy::{
Algorithm, CallDecision, CallModel, CapabilityJudgeConfig, ClassifierContractConfig,
ClassifierResponseFormat, ClassifyTrigger, CustomClassifierConfig, CustomClassifierPolicy,
- DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig, HandoffNoteConfig,
- LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback,
- LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter,
- StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics,
+ DEFAULT_UNMATCHED_STEPS, DecisionJudgeConfig, DeescalationConfig, EscalationJudgeConfig,
+ HandoffNoteConfig, LibsyError as RustLibsyError, LlmCapabilityConfig, LlmClassifierConfig,
+ LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels,
+ StageRouter, StageRouterConfig, State, Step as RustStep, StepStream, TaskClassifierConfig,
+ ToolSemantics,
};
use switchyard_protocol::{
Category, DecisionResponse, LlmClientError, LlmResponse, LlmResponseStream,
@@ -371,6 +372,7 @@ impl PyTaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold,
threshold_step,
+ unmatched_steps: DEFAULT_UNMATCHED_STEPS,
contract: classifier_contract(prompt, response_format_type)?,
max_output_tokens,
}),
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index c7d0882dc..c2ca4cc3a 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -13,11 +13,11 @@ use libsy::{
AdvisorGate, AdvisorGateConfig, Algorithm, BreakerConfig, CapabilityJudgeConfig,
ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeJudgeConfig,
CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy,
- DecisionJudgeConfig, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig,
- LlmCapabilityConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough,
- PickerMode, PlanExecute, PlanExecuteConfig, Random, RuntimeModels, StageRouter,
- StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics,
- Vgr, VgrConfig,
+ DEFAULT_UNMATCHED_STEPS, DecisionJudgeConfig, EscalationJudgeConfig, GateTrigger,
+ HandoffNoteConfig, LlmCapabilityConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier,
+ Noop, Passthrough, PickerMode, PlanExecute, PlanExecuteConfig, Random, RuntimeModels,
+ StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig,
+ ToolSemantics, Vgr, VgrConfig,
};
use serde::Deserialize;
use serde_json::Value;
@@ -301,6 +301,9 @@ pub struct LlmClassifierRouteConfig {
/// Capability mode: how much to raise the threshold when the judge is
/// uncertain. Added once for an uncertain verdict and twice for unsupported.
pub threshold_step: Option<f64>,
+ /// Capability mode: threshold steps for an unmatched verdict (no capability rule
+ /// applies). 1 by default, the same as uncertain; 2 treats it like unsupported.
+ pub unmatched_steps: Option<u8>,
/// Capability mode: routes to the capable tier on judge client failures and deadlines.
/// Defaults to true when capability mode is selected.
pub fail_open: Option<bool>,
@@ -1181,6 +1184,7 @@ impl LlmClassifierRouteConfig {
weak_target,
base_threshold,
threshold_step,
+ unmatched_steps,
fail_open,
classify_trigger,
message_hash_fallback,
@@ -1234,6 +1238,7 @@ impl LlmClassifierRouteConfig {
let judge = if let Some(decision) = decision {
if base_threshold.is_some()
|| threshold_step.is_some()
+ || unmatched_steps.is_some()
|| prompt.is_some()
|| *response_format_type != ClassifierResponseFormat::JsonSchema
|| *max_output_tokens != default_classifier_max_output_tokens()
@@ -1251,6 +1256,7 @@ impl LlmClassifierRouteConfig {
base_threshold,
)?,
threshold_step: threshold_step.unwrap_or_default(),
+ unmatched_steps: unmatched_steps.unwrap_or(DEFAULT_UNMATCHED_STEPS),
contract: classifier_contract(prompt.as_deref())
.with_response_format_type(*response_format_type),
max_output_tokens: *max_output_tokens,
@@ -1296,7 +1302,10 @@ impl LlmClassifierRouteConfig {
|| fail_open.is_some()
|| *message_hash_fallback
|| recent_turn_window.is_some();
- if mode.is_some() && has_capability_settings {
+ // `unmatched_steps` is new, so no existing escalation configuration carries it:
+ // reject it even when the mode is implied by `escalation`. The older capability
+ // keys stay tolerated in that implicit form for compatibility.
+ if unmatched_steps.is_some() || (mode.is_some() && has_capability_settings) {
return Err(AlgorithmConfigError::new(format!(
"llm_classifier route {route_name} mode escalation cannot use capability routing settings"
)));
@@ -1334,6 +1343,7 @@ impl LlmClassifierRouteConfig {
|| weak_target.is_some()
|| base_threshold.is_some()
|| threshold_step.is_some()
+ || unmatched_steps.is_some()
|| escalation.is_some()
|| *response_format_type != ClassifierResponseFormat::JsonSchema
{
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index c01793053..285009d8f 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1748,6 +1748,34 @@ classify_trigger"#,
Ok(())
}
+ #[test]
+ fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> {
+ // Accepted on a capability route; the value range is checked by the algorithm.
+ runner_from_toml(&VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nunmatched_steps = 2",
+ ))?;
+ assert!(
+ error_message(&VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nunmatched_steps = 3",
+ ))
+ .contains("unmatched_steps must be 0, 1, or 2")
+ );
+
+ // Rejected on an escalation route, including the implicit form where the
+ // `escalation` table alone selects the mode: the key is new, so no existing
+ // configuration relies on it being ignored there.
+ assert!(
+ error_message(&VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }",
+ ))
+ .contains("mode escalation cannot use capability routing settings")
+ );
+ Ok(())
+ }
+
#[test]
fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> {
let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";
diff --git a/crates/switchyard-runner/tests/algorithm_factory.rs b/crates/switchyard-runner/tests/algorithm_factory.rs
index 6b4c1363b..0ee584d3c 100644
--- a/crates/switchyard-runner/tests/algorithm_factory.rs
+++ b/crates/switchyard-runner/tests/algorithm_factory.rs
@@ -141,6 +141,13 @@ fn invalid_routing_configuration_is_rejected() {
),
"bogus",
),
+ (
+ DEPLOYMENT.replace(
+ "weak_target = \"fast\"\n",
+ "weak_target = \"fast\"\nunmatched_steps = 2\n",
+ ),
+ "decision cannot use LLM judge settings",
+ ),
] {
let error = AlgorithmFactory::from_toml(&source)
.and_then(|factory| factory.build())
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index c3edb7454..9b1a74d65 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -178,7 +178,8 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts
| Key | Default | Meaning |
|---|---|---|
| `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. |
-| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `unmatched_steps` | `1` | Threshold steps for an unmatched verdict. `0`, `1`, or `2`; `2` treats it like unsupported. |
| `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request including tool continuations, `user_turn` judges each new user message and holds that target across the tool calls between, `new_session` judges once and reuses that target for the session. |
| `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 4bd97fb19..cc8820264 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -351,7 +351,8 @@ Capability mode classifies before serving. See
| `weak_target` | Yes | — | Efficient tier. |
| `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. |
| `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid verdicts use the capable tier with either setting. |
-| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. |
| `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. |
| `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. |
| `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |
diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md
index 45a1a6a0e..e43225b19 100644
--- a/docs/routing_algorithms/llm_classifier_routing.md
+++ b/docs/routing_algorithms/llm_classifier_routing.md
@@ -71,7 +71,8 @@ greater than or equal to the applicable threshold. Otherwise it routes to
`strong_target`:
- `supported` uses `base_threshold`.
-- `uncertain` and `unmatched` use `base_threshold + threshold_step`.
+- `uncertain` uses `base_threshold + threshold_step`.
+- `unmatched` uses `base_threshold + unmatched_steps * threshold_step` (`unmatched_steps` defaults to `1`).
- `unsupported` uses `base_threshold + 2 * threshold_step`.
An invalid, inconsistent, or unparseable verdict routes to
@@ -122,6 +123,7 @@ for the server merge behavior.
|---|---|---|
| `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. |
| `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. |
+| `unmatched_steps` | `1` | Boundary steps applied to an `unmatched` verdict. `0`, `1`, or `2`; `2` requires the same confidence as an `unsupported` verdict. |
| `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. |
| `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. |
| `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. |