Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(routing): support decision models in composite routes (#981)

ryan-lempka7fa4b00cff5a
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index 94b2a54a9..646b37a43 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -110,6 +110,49 @@ struct CapabilityClassifierRouteConfig {
recent_turn_window: Option<usize>,
}
+impl CapabilityClassifierRouteConfig {
+ fn into_task_config(
+ self,
+ route_name: &str,
+ targets: &BTreeMap<String, ModelId>,
+ ) -> AlgorithmResult<TaskClassifierConfig> {
+ let config = self;
+ let judge = match config.judge {
+ CapabilityJudgeRouteConfig::Llm(judge) => CapabilityJudgeConfig::Llm(judge),
+ CapabilityJudgeRouteConfig::Decision(judge) => {
+ for target in [&config.strong_target, &config.weak_target] {
+ if !judge.candidates.values().any(|name| name == target) {
+ return Err(AlgorithmConfigError::new(format!(
+ "route {route_name}: decision.candidates must include target {target}"
+ )));
+ }
+ }
+ let candidates = judge
+ .candidates
+ .into_iter()
+ .map(|(label, name)| {
+ resolve_target_model_id(route_name, &name, targets)
+ .map(|model| (label, model))
+ })
+ .collect::<AlgorithmResult<_>>()?;
+ CapabilityJudgeConfig::Decision(DecisionJudgeConfig {
+ cutoff: judge.cutoff,
+ instructions: judge.instructions,
+ candidates,
+ evidence: judge.evidence,
+ })
+ }
+ };
+ Ok(TaskClassifierConfig {
+ judge,
+ fail_open: config.fail_open,
+ classify_trigger: config.classify_trigger,
+ message_hash_fallback: config.message_hash_fallback,
+ recent_turn_window: config.recent_turn_window,
+ })
+ }
+}
+
#[derive(Clone, Debug)]
enum CapabilityJudgeRouteConfig {
Llm(LlmCapabilityConfig),
@@ -490,12 +533,16 @@ pub enum AdvisorTriggerConfig {
pub struct StageClassifierConfig {
/// Target the judge is called through. Not a routing destination.
pub target: String,
+ #[serde(default)]
+ pub decision: Option<DecisionJudgeRouteConfig>,
+ #[serde(default)]
+ pub fail_open: Option<bool>,
/// Lowest solve probability that still routes to the efficient tier, from 0 to 1.
- pub base_threshold: f64,
+ pub base_threshold: Option<f64>,
/// How much to raise the threshold when the judge is uncertain. Added once
/// for an uncertain verdict and twice for unsupported.
#[serde(default)]
- pub threshold_step: f64,
+ pub threshold_step: Option<f64>,
/// How often the judge runs. `new_session` has no effect here.
#[serde(default)]
pub classify_trigger: ClassifyTrigger,
@@ -547,19 +594,33 @@ pub struct StageTierConfig {
}
impl StageClassifierConfig {
- fn task_classifier_config(&self) -> TaskClassifierConfig {
- TaskClassifierConfig {
- judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
- base_threshold: self.base_threshold,
- threshold_step: self.threshold_step,
- contract: classifier_contract(self.prompt.as_deref())
- .with_response_format_type(self.response_format_type),
- max_output_tokens: self.max_output_tokens,
- }),
- fail_open: true,
+ fn task_classifier_config(
+ &self,
+ route_name: &str,
+ tiers: &StageTierConfig,
+ targets: &BTreeMap<String, ModelId>,
+ ) -> AlgorithmResult<TaskClassifierConfig> {
+ let config = LlmClassifierRouteConfig {
+ classifier_target: self.target.clone(),
+ strong_target: Some(tiers.capable_target.clone()),
+ weak_target: Some(tiers.efficient_target.clone()),
+ base_threshold: self.base_threshold,
+ threshold_step: self.threshold_step,
+ fail_open: self.fail_open,
+ decision: self.decision.clone(),
classify_trigger: self.classify_trigger,
message_hash_fallback: self.message_hash_fallback,
recent_turn_window: self.recent_turn_window,
+ prompt: self.prompt.clone(),
+ response_format_type: self.response_format_type,
+ max_output_tokens: self.max_output_tokens,
+ ..Default::default()
+ };
+ match config.validated_classifier_mode(route_name)? {
+ LlmClassifierModeConfig::Capability(config) => {
+ config.into_task_config(route_name, targets)
+ }
+ _ => Err(AlgorithmConfigError::new("expected a capability judge")),
}
}
}
@@ -586,6 +647,14 @@ impl AlgorithmSpec {
.decision
.as_ref()
.map(|decision| (config.classifier_target.as_str(), decision)),
+ Self::Composite { classifier, .. }
+ | Self::StageRouter {
+ classifier: Some(classifier),
+ ..
+ } => classifier
+ .decision
+ .as_ref()
+ .map(|decision| (classifier.target.as_str(), decision)),
_ => None,
}
}
@@ -1330,39 +1399,7 @@ fn build_algorithm(
let mode = classifier_config.validated_classifier_mode(route_name)?;
let algorithm = match mode {
LlmClassifierModeConfig::Capability(config) => {
- let judge = match config.judge {
- CapabilityJudgeRouteConfig::Llm(judge) => CapabilityJudgeConfig::Llm(judge),
- CapabilityJudgeRouteConfig::Decision(judge) => {
- for target in [&config.strong_target, &config.weak_target] {
- if !judge.candidates.values().any(|name| name == target) {
- return Err(AlgorithmConfigError::new(format!(
- "llm_classifier route {route_name}: decision.candidates must include target {target}"
- )));
- }
- }
- let candidates = judge
- .candidates
- .into_iter()
- .map(|(label, name)| {
- resolve_target_model_id(route_name, &name, targets)
- .map(|model| (label, model))
- })
- .collect::<AlgorithmResult<_>>()?;
- CapabilityJudgeConfig::Decision(DecisionJudgeConfig {
- cutoff: judge.cutoff,
- instructions: judge.instructions,
- candidates,
- evidence: judge.evidence,
- })
- }
- };
- let classifier_config = TaskClassifierConfig {
- judge,
- fail_open: config.fail_open,
- classify_trigger: config.classify_trigger,
- message_hash_fallback: config.message_hash_fallback,
- recent_turn_window: config.recent_turn_window,
- };
+ let classifier_config = config.into_task_config(route_name, targets)?;
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
config: classifier_config,
})
@@ -1439,9 +1476,14 @@ fn build_algorithm(
config.handoff_notes = handoff_notes.clone();
// The judge is called through its own target, so it is not a routing
// destination and stays out of the tier pair.
- config.llm_fallback = classifier.as_ref().map(|classifier| LlmFallback {
- config: classifier.task_classifier_config(),
- });
+ config.llm_fallback = classifier
+ .as_ref()
+ .map(|classifier| {
+ classifier
+ .task_classifier_config(route_name, tiers, targets)
+ .map(|config| LlmFallback { config })
+ })
+ .transpose()?;
let algorithm = StageRouter::new(config).map_err(|error| {
AlgorithmConfigError::with_source(
format!("stage_router route {route_name}: {error}"),
@@ -1475,7 +1517,7 @@ fn build_algorithm(
stage_config.tool_semantics = stage.tool_semantics.clone();
stage_config.handoff_notes = stage.handoff_notes.clone();
let config = CompositeRouterConfig {
- judge: classifier.task_classifier_config(),
+ judge: classifier.task_classifier_config(route_name, stage, targets)?,
stage: stage_config,
};
let algorithm = CompositeRouter::new(config).map_err(|error| {
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index d0cd12286..e8918efe0 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -1191,6 +1191,39 @@ new = ["send_message"]
)
}
+ #[test]
+ fn composite_decision_judge_reuses_capability_validation() {
+ let source = composite_config().replace(
+ "[routes.composed.classifier]\ntarget = \"tier_judge\"\nbase_threshold = 0.5",
+ "[routes.composed.classifier]\ntarget = \"tier_judge\"",
+ ) + r#"
+[routes.composed.classifier.decision]
+cutoff = 0.4
+candidates = { a = "strong", b = "weak" }
+evidence = { description = "candidate comparison" }
+"#;
+ for (source, valid) in [
+ (source.clone(), true),
+ (
+ source.replace(
+ "target = \"tier_judge\"",
+ "target = \"tier_judge\"\nbase_threshold = 0.5",
+ ),
+ false,
+ ),
+ (source.replace("b = \"weak\"", "b = \"strong\""), false),
+ ] {
+ let deployment: DeploymentConfig = toml::from_str(&source).unwrap();
+ let spec = &deployment.routes["composed"].algorithm;
+ assert_eq!(spec.decision_judge().unwrap().0, "tier_judge");
+ assert_eq!(
+ spec.build_with_runtime_models("composed", &deployment.build_targets())
+ .is_ok(),
+ valid
+ );
+ }
+ }
+
#[test]
fn composite_route_builds_and_claims_both_tiers_and_its_judge() -> RunnerResult<()> {
let runner = runner_from_toml(&composite_config())?;
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 91f1f90f2..dd0ca0765 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -391,9 +391,11 @@ configuration. Today a classifier sets the tier a stage router falls open to whe
| Key | Required | Default | Meaning |
|---|:---:|---|---|
| `classifier.target` | Yes | — | Target the tier judge is called through. Not a routing destination. |
-| `classifier.base_threshold` | Yes | — | `p_solve` floor that still routes to the efficient tier. In `[0, 1]`. |
+| `classifier.base_threshold` | For LLM judges | unset | `p_solve` floor that still routes to the efficient tier. In `[0, 1]`. Omit with `classifier.decision`. |
| `classifier.threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
| `classifier.classify_trigger` | Yes | — | `user_turn` re-picks the tier whenever the user speaks, `new_session` picks once and holds it. `every_request` is rejected here: a judge call per tool step is the cost this route exists to avoid. |
+| `classifier.decision` | No | unset | Use a decision model instead of an LLM judge. The classifier target must be in `decision_targets`. |
+| `classifier.fail_open` | No | `true` | On judge failure, retain the last classified tier or default to efficient. Set `false` to return the error. |
| `classifier.message_hash_fallback` | No | `false` | Retains the tier by hashing the first user message, for clients that send no session ID. Works with either supported trigger. Conversations opening with the same text share a tier. |
| `classifier.recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |
| `classifier.prompt` | No | packaged prompt | Replaces the classifier judge prompt. The verdict schema is unchanged. |
@@ -411,6 +413,10 @@ configuration. Today a classifier sets the tier a stage router falls open to whe
| `stage.handoff_notes` | No | unset | Optional guidance appended to forwarded requests. Uses the same fields and behavior as [stage-router handoff notes](../routing_algorithms/stage_router_routing.md#optional-handoff-notes). |
| `subagents` | No | unset | Nested policy used only for delegated sub-agent work. |
+For decision classifier configuration, see
+[Decision model classifier](../routing_algorithms/composite_routing.md#decision-model-classifier).
+The same decision settings are accepted by the optional `stage_router` classifier.
+
The tier is retained per session. A deployment that sends no session ID needs
`classifier.message_hash_fallback = true`, which keys on the first user message
instead. The outer `[routes.<name>.classifier]` table is required. The nested
diff --git a/docs/routing_algorithms/composite_routing.md b/docs/routing_algorithms/composite_routing.md
index e70e96c14..aeab7e976 100644
--- a/docs/routing_algorithms/composite_routing.md
+++ b/docs/routing_algorithms/composite_routing.md
@@ -7,12 +7,16 @@ fit the same route type.
## What exists today
-One pairing: an LLM classifier sets the tier a stage router falls open to when
+One pairing: a classifier sets the tier a stage router falls open to when
its own signals are not confident. The stage router's `picker` mode is unchanged,
and so is how it scores signals, escalates, and de-escalates. Both algorithms and the
relationship between them are fixed, and the configuration below names them
directly rather than composing arbitrary algorithms.
+The classifier can use an LLM judge or a decision model. A decision
+model is configured through a decision client and target, separately from the
+capable and efficient completion targets.
+
The general form, where any algorithm can target another and set its
configuration, is the direction rather than the current behaviour. Expect this
route's configuration to change as more pairings are added.
@@ -49,6 +53,39 @@ nor a second `classifier`. The outer classifier supplies the default tier.
Custom `tool_semantics` remain stage-local and have the same additive behavior as
on a standalone stage route.
+## Decision model classifier
+
+Use a decision target as the classifier. Keep the stage block above
+and replace its LLM classifier settings with:
+
+```toml
+[decision_clients.decisions]
+format = "system_one"
+endpoint = "https://openrouter.ai/api/alpha/decisions"
+api_key_env = "OPENROUTER_API_KEY"
+timeout_ms = 5000
+
+[decision_targets.judge]
+id = "inception/mercury-decide:free"
+decision_client = "decisions"
+
+[routes.switchyard.classifier]
+target = "judge"
+classify_trigger = "user_turn"
+
+[routes.switchyard.classifier.decision]
+cutoff = 0.4
+candidates = { a = "strong", b = "weak" }
+evidence = { description = "Add task-relevant evidence comparing a and b." }
+```
+
+Both candidates must be completion targets. The judge must be a decision target.
+The classifier selects capable when `probabilities.advantage` exceeds `cutoff`.
+Calibrate the cutoff for your tasks and models. Omit LLM-only settings such as
+`base_threshold`, `threshold_step` and `prompt` when using a decision model.
+On judge failure, the default is to retain the last classified tier or use
+efficient for a new session. Set `classifier.fail_open = false` to return the error.
+
## Sessions
The tier is retained per session, so send `x-switchyard-session-id` and each