Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

test(routing): add drift and ceiling tests with the tuning, troubleshooting and strategy docs (#1031)

eric-liu-nvidiaf7295b7aa547
diff --git a/README.md b/README.md
index d4e94b441..053e12812 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ Use Switchyard through a gateway integration, try it with a local proxy, or embe
2. The algorithm examines the request or the agent's recent tool activity. Some algorithms call a model to judge the task.
3. Your gateway or application sends the request to the selected model. Routing can change as the agent continues its work.
-Evaluate the complete agent, model pool, and routing configuration against your single-model baseline. A cheaper model call does not guarantee a cheaper successfully completed task.
+Evaluate the complete agent, model pool, and routing configuration against your single-model baseline. A cheaper model call does not guarantee a cheaper successfully completed task. The [tuning loop](docs/tuning/index.md) gives the steps for that evaluation, and [Troubleshooting](docs/operations/troubleshooting.md) lists the settings that disable routing without an error.
## How to use it
diff --git a/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml b/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml
index b128d6c3a..5d2617f61 100644
--- a/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml
+++ b/benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml
@@ -51,5 +51,4 @@ type = "llm_classifier"
classifier_target = "judge"
strong_target = "strong"
weak_target = "weak"
-base_threshold = 0.5
escalation = { confirmations = 2, recent_turn_window = 28, window_message_chars = 500 }
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 7e1e82c09..6dfc5a135 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -121,10 +121,11 @@ impl Classifier for FallOpen {
/// The capability judge a stage router falls through to.
pub struct LlmFallback {
- /// Judge configuration. `recent_turn_window` is worth setting to this router's
- /// `recent_window` so the judge reads the same span the signal scorer scored.
- /// Note: `classify_trigger = new_session` and `message_hash_fallback` have no effect here —
- /// the judge runs as a cascade classifier, not a standalone algorithm.
+ /// Judge configuration. Its `recent_turn_window` counts trailing messages, while
+ /// this router's `recent_window` counts tool results.
+ /// Note: `classify_trigger` and `message_hash_fallback` have no effect here. The judge
+ /// runs as a cascade classifier on every ambiguous turn, not as a standalone algorithm,
+ /// so no verdict is retained.
pub config: TaskClassifierConfig,
}
@@ -272,6 +273,7 @@ mod tests {
use parking_lot::Mutex;
use super::*;
+ use crate::algorithms::util::affinity::ClassifyTrigger;
use crate::algorithms::util::stage::{clear_fall_open, set_fall_open};
use crate::algorithms::util::tier_fixtures::{JUDGE, Recorder, turn_request};
use crate::core::testing::test_drive_with_models;
@@ -626,6 +628,37 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn the_fallback_judge_ignores_the_retaining_triggers() -> Result<()> {
+ for trigger in [ClassifyTrigger::UserTurn, ClassifyTrigger::NewSession] {
+ let recorder = Arc::new(Recorder::default());
+ let mut config = config_with_judge(&recorder, 0.9);
+ config
+ .llm_fallback
+ .as_mut()
+ .expect("config_with_judge sets a judge")
+ .config
+ .classify_trigger = trigger;
+ let router = recording_router(config)?;
+
+ // Two tool continuations of one user turn in one session. A standalone
+ // capability route with this trigger would judge once and retain the
+ // target; inside the stage cascade the judge is a plain classifier.
+ for _ in 0..2 {
+ test_drive_with_models(
+ router.clone(),
+ turn_request(false),
+ runtime_models(),
+ recorder.serve(),
+ )
+ .await?;
+ }
+
+ assert_eq!(recorder.judge_calls(), 2, "{trigger:?} retained a verdict");
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> {
let recorder = Arc::new(Recorder::default());
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 40fd2c5d8..76d688c6f 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -819,6 +819,59 @@ mod tests {
);
}
+ #[test]
+ fn scorer_ceilings_match_the_documented_values() {
+ // The stage-router guide's "How the score is built" quotes these four
+ // scores. A constant change that moves them must update the guide too.
+ let base = signal_from(json!([{"role": "user", "content": "hi"}]));
+ let cases = [
+ (
+ "hard error alone",
+ ToolSignals {
+ severity: HARD_SEVERITY as f32,
+ turn_depth: 1,
+ ..base.clone()
+ },
+ 0.4621,
+ ),
+ (
+ "hard error plus stall",
+ ToolSignals {
+ severity: HARD_SEVERITY as f32,
+ turn_depth: STALL_MIN_TURN_DEPTH,
+ ..base.clone()
+ },
+ 0.7616,
+ ),
+ (
+ "one write among three operations, no error",
+ ToolSignals {
+ recent_write_count: 1,
+ recent_read_count: 2,
+ turn_depth: 1,
+ ..base.clone()
+ },
+ -0.6911,
+ ),
+ (
+ "full clean production",
+ ToolSignals {
+ recent_write_count: 3,
+ turn_depth: 1,
+ ..base.clone()
+ },
+ -0.8741,
+ ),
+ ];
+ for (name, signal, expected) in cases {
+ let scored = score_signal(&signal);
+ assert!(
+ (scored.score - expected).abs() < 0.0005,
+ "{name}: {scored:?}, documented {expected}"
+ );
+ }
+ }
+
#[test]
fn pick_tier_treats_the_exact_threshold_score_as_ambiguous() {
// score == threshold maps to p == 0.5 + t/2 exactly, the closed
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index 67736cd29..b2f141ae2 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -598,7 +598,9 @@ pub struct StageClassifierConfig {
/// for an uncertain verdict and twice for unsupported.
#[serde(default)]
pub threshold_step: Option<f64>,
- /// How often the judge runs. `new_session` has no effect here.
+ /// How often the judge runs. Composite honors it. On a standalone stage route
+ /// it has no effect: the fallback judge runs on every ambiguous turn and
+ /// retains no verdict.
#[serde(default)]
pub classify_trigger: ClassifyTrigger,
/// Reuses the session's target by hashing the first user message when no
diff --git a/docs/assets/stage-router-threshold.svg b/docs/assets/stage-router-threshold.svg
index ffd209789..104e8adfb 100644
--- a/docs/assets/stage-router-threshold.svg
+++ b/docs/assets/stage-router-threshold.svg
@@ -1,6 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 330" role="img" aria-labelledby="title desc">
- <title id="title">Stage score distribution at confidence threshold 0.5</title>
- <desc id="desc">An illustrative histogram of signed Stage scores. Scores below negative 0.5 select efficient, scores from negative 0.5 through positive 0.5 form the ambiguous band and fall back to efficient in efficient-first mode, and scores above positive 0.5 select capable.</desc>
+ <title id="title">Stage score distribution at confidence threshold 0.5, with signal ceilings</title>
+ <desc id="desc">An illustrative histogram of signed Stage scores. Scores below negative 0.5 select efficient, scores from negative 0.5 through positive 0.5 form the ambiguous band and fall back to efficient in efficient-first mode, and scores above positive 0.5 select capable. Dotted markers show the most the signals can reach: positive 0.76 toward capable and negative 0.87 toward efficient.</desc>
<rect width="880" height="330" rx="12" fill="#ffffff"/>
<text x="440" y="30" text-anchor="middle" font-family="sans-serif" font-size="17" font-weight="700" fill="#1f2937">Illustrative score distribution · efficient_first · threshold = 0.5</text>
@@ -13,7 +13,6 @@
<text x="717" y="82" text-anchor="middle" font-family="sans-serif" font-size="14" font-weight="700" fill="#5b21b6">signal → capable</text>
<g fill="#4b5563" opacity="0.82">
- <rect x="85" y="220" width="24" height="20" rx="2"/>
<rect x="121" y="207" width="24" height="33" rx="2"/>
<rect x="157" y="188" width="24" height="52" rx="2"/>
<rect x="193" y="171" width="24" height="69" rx="2"/>
@@ -31,14 +30,16 @@
<rect x="625" y="180" width="24" height="60" rx="2"/>
<rect x="661" y="197" width="24" height="43" rx="2"/>
<rect x="697" y="211" width="24" height="29" rx="2"/>
- <rect x="733" y="220" width="24" height="20" rx="2"/>
- <rect x="769" y="228" width="24" height="12" rx="2"/>
</g>
<line x1="70" y1="240" x2="810" y2="240" stroke="#111827" stroke-width="2"/>
<line x1="255" y1="58" x2="255" y2="248" stroke="#92400e" stroke-width="2" stroke-dasharray="6 5"/>
<line x1="625" y1="58" x2="625" y2="248" stroke="#92400e" stroke-width="2" stroke-dasharray="6 5"/>
<line x1="440" y1="235" x2="440" y2="248" stroke="#111827" stroke-width="2"/>
+ <line x1="722" y1="92" x2="722" y2="248" stroke="#5b21b6" stroke-width="2" stroke-dasharray="2 4"/>
+ <line x1="117" y1="92" x2="117" y2="248" stroke="#1e40af" stroke-width="2" stroke-dasharray="2 4"/>
+ <text x="722" y="52" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#5b21b6">signal ceiling +0.76</text>
+ <text x="117" y="52" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1e40af">signal ceiling -0.87</text>
<g font-family="sans-serif" font-size="13" fill="#374151" text-anchor="middle">
<text x="70" y="265">-1.0</text>
diff --git a/docs/index.md b/docs/index.md
index 67c220ea0..3cd51914a 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -25,6 +25,8 @@ The Server Path builds and runs the standalone `switchyard-server` binary.
- [Server CLI Reference](cli_reference.md): inspect `switchyard-server` commands and options
- [Rust API](reference/rust_api.md): browse libsy and protocol crate documentation
- [Context-Window Handling](operations/context_window.md): configure eviction and retry behavior
+- [Tuning](tuning/index.md): the loop from verifying routing to recording a result
+- [Troubleshooting](operations/troubleshooting.md): symptoms, silent misconfigurations, and fixes
## Reference
diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md
new file mode 100644
index 000000000..bac4a1955
--- /dev/null
+++ b/docs/operations/troubleshooting.md
@@ -0,0 +1,88 @@
+# Troubleshooting
+
+Two tables. The first starts from what you see. The second lists settings that
+load without error while disabling what you meant to configure. Each check
+names where to look: a response header or body, a `/v1/stats` key, a routing
+log field, a log line, or a `--dry-run` message.
+
+## Symptoms
+
+| Symptom | Check | Likely causes, most common first | Fix |
+|---|---|---|---|
+| The first request lands on the efficient model | `x-model-router-selected-model` on the response | Auto and Execution decide from tool results, and a request without tool history has none. This is by design. | Send a request with a critical tool error, as in [Verify routing is active](../getting_started.md#verify-routing-is-active). |
+| Every request lands on one model | `curl -s localhost:4000/v1/stats | jq -c '.models | map_values(.calls)'` and `.algorithm_stats.stage_router.routing_decisions` | Chat-only traffic (`fall_open` only); both tiers name the same target or the same model ID; `capable_first` with a threshold the signals cannot reach; one tier is down and the other serves its fallbacks. | See the matching rows in [Silent misconfigurations](#silent-misconfigurations) and [When Auto does not route](../routing_algorithms/overview.md#when-auto-does-not-route). |
+| The judge never changes the outcome | `switchyard_classifier_fail_open_total` on `/metrics`, and the log line `judge verdict unavailable; routing without one` with its `reason` | The provider rejects JSON Schema mode (`json_object` needed); the verdict is truncated by `max_output_tokens`; the judge model returns prose instead of JSON. | [Judge model compatibility](../routing_algorithms/llm_classifier_routing.md#judge-model-compatibility). |
+| Escalation never latches | The log line `stateful escalation has no session ID; routing state will not persist` | The client sends no session ID. No escalation state survives a request without one, so the route never latches at any `confirmations`. The warning fires only when `confirmations` is 2 or more. | Send `x-switchyard-session-id`. See [Tuning options](../routing_algorithms/escalation_router_routing.md#tuning-options). |
+| Requests hang or take much longer than the model needs | Server log for retries and cooldown; `/v1/stats` call counts per model | A judge or tier without `timeout_ms`; a judge call on every tool step; a dead tier retried before fallback. | Give the judge its own `[llm_clients]` entry with `timeout_ms`. On a Task or Custom `llm_classifier` route, set `classify_trigger = "user_turn"` and send a session ID. See the dead-tier row below. |
+| HTTP 400 from the upstream when the client sends a thinking level | The upstream error body in the response | A Claude target on an OpenAI wire format behind a gateway. | [Thinking](../integrations/pi.md#thinking) and `omit_body_fields`. |
+| HTTP 400 with code `context_length_exceeded` | The response body | Every target on the route overflowed. | [Context-Window Handling](context_window.md). |
+| HTTP 400 on requests that contain images | The response body | The route sets `vision = false`. | Remove it, or declare `true` only when every target accepts images. See the [route keys](../reference/toml_schema.md#routesname). |
+| The server does not start: `unknown field` | The `--dry-run` message | An `auto` route with a tuning key; a typo; a key from another route type. | [Auto](../reference/toml_schema.md#auto) and the [validation errors](../reference/toml_schema.md#validation-errors). |
+| The server does not start: two targets `both name model` | The `--dry-run` message | Two targets share a model ID on one client with different `reasoning_effort`, `extra_body`, or `omit_body_fields`. | [Reasoning effort](../routing_algorithms/model_selection.md#reasoning-effort). |
+| The client shows zero input tokens on `/v1/messages` | `prompt_tokens` in the routing log | The streamed `message_start` event reports zero usage when the target is on an OpenAI format. The real counts arrive in `message_delta`. | [Streamed token counts](../routing_algorithms/model_selection.md#pin-the-provider-and-endpoint). |
+| Cost per run is higher than the single capable model | `cached_tokens` in the routing log on the turn after a model change; cost per arm | Each switch loses the prompt cache; the judge adds a call per turn; the efficient model is too weak and most turns escalate. | [Counter-examples](../routing_algorithms/model_selection.md#counter-examples) and the [inverted-pair check](../routing_algorithms/model_selection.md#check-for-an-inverted-pair). |
+| The prompt cache is never read | `cached_tokens` stays `0` on repeated prompts | The gateway does not cache on that endpoint, or the prompt is too short to cache. | [Prompt caching](../integrations/pi.md#prompt-caching). |
+| `/v1/stats` has no `tiers` key | The `/v1/stats` body | The Rust server reports decisions under `algorithm_stats.<algorithm>`. | Read `algorithm_stats.stage_router.routing_decisions`, as in [Verify routing is active](../getting_started.md#verify-routing-is-active). |
+| The routing log does not say why a model was chosen | A routing-log record | The log records the served model and token counts only. | Use the decision counts in `/v1/stats`. For overrides, the `stage router override` log line also names the reason. See [Decision sources](../routing_algorithms/stage_router_routing.md#decision-sources). |
+| Sub-agent requests ignore the `subagents` policy | The log line `this route has sub-agent routing but the calling harness does not send sub-agent identity`. It fires only for Claude Code versions known to lack child identity. For other harnesses, check whether delegated requests carry a session ID and an agent ID. | The harness sends no sub-agent identity, so delegated requests follow the parent route. | [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). |
+
+## Silent misconfigurations
+
+These load and pass `--dry-run`. Most answer every request with HTTP 200; the
+rows that return an error or hold a request open say so.
+
+| Setting | Why it is silent | Warning | Fix |
+|---|---|---|---|
+| `picker = "capable_first"` with `confidence_threshold` at `0.88` or above | The signals reach at most 0.8741 toward efficient, so they never offload. Between `0.49` and `0.8741` only clean production offloads. | Startup: `uses picker "capable_first", which is experimental`. Nothing about the threshold. | Use `efficient_first`, or lower the threshold. See [Reachable scores per picker](../routing_algorithms/stage_router_routing.md#reachable-scores-per-picker). |
+| `capable_target` and `efficient_target` name the same target, or two targets with the same model ID and identical settings | One model serves every decision. | None for the same target. Startup `reuses model id ...; only one target per id is kept` for two targets with one model ID. | Use two different models. See [When Auto does not route](../routing_algorithms/overview.md#when-auto-does-not-route). |
+| `llm_classifier` with an `escalation` table, no `mode`, and `base_threshold`, `threshold_step`, `fail_open`, `recent_turn_window`, or `message_hash_fallback` | The route runs in escalation mode because `escalation` is present. The capability keys are accepted and ignored. | None. | Set `mode` explicitly. With `mode = "escalation"` the same keys become a load error. |
+| `llm_classifier` capability with the default `classify_trigger = "every_request"` on a tool-calling agent | The judge runs on every tool continuation, so one task with twenty tool calls costs twenty-one judge calls. | None. | `classify_trigger = "user_turn"` with a session ID. See [When the judge runs](../routing_algorithms/llm_classifier_routing.md#when-the-judge-runs). |
+| Escalation with clients that send no session ID | No escalation state survives a request without a session ID, so the route never latches. With `confirmations = 1` each judged turn can escalate on its own, at the price of a weak, a judge, and a strong call per turn, and the warning goes away, but nothing is retained. | Runtime, once per route, when `confirmations` is 2 or more: `stateful escalation has no session ID; routing state will not persist`. | Send `x-switchyard-session-id`. |
+| `classify_trigger = "user_turn"` or `"new_session"` with clients that send no session ID and no `message_hash_fallback` | Nothing can be retained, so every request is judged. | Runtime, once per route: `affinity is enabled but this request carries no usable identity, so no affinity is applied`. | Send a session ID, or set `message_hash_fallback = true`. |
+| `response_format_type = "json_schema"` (the default) with a provider that supports only JSON object mode | Every judge call is rejected by the provider. Task routes to the strong target and Execution falls open to the picker default, both with HTTP 200. Escalation stops the request and returns the provider's error on the first attempt; a 400 is not retried. | Runtime, per call on Task and Execution routes: `judge verdict unavailable; routing without one`, and `switchyard_classifier_fail_open_total` rises. Escalation routes return the error instead. | `response_format_type = "json_object"`. See [Judge model compatibility](../routing_algorithms/llm_classifier_routing.md#judge-model-compatibility). |
+| A tier that refuses connections or returns HTTP 408, 429, or 5xx, with the default `max_retries = 2` | Each request retries that tier, then falls through to the route's other target, so one model serves everything, slowly. After the retries fail, a 5-second cooldown sends later requests straight to the other target. A tier that hangs behaves differently: without `timeout_ms` the request waits, and with it the request that reaches the deadline returns `504`. Only requests inside the cooldown that follows fall through. | Runtime log lines for the failed attempts and the fallback. | Fix the endpoint and watch `.models` call counts in `/v1/stats`. Set `timeout_ms` so a hang becomes a `504` instead of a wait. See [`[llm_clients.<name>]`](../reference/toml_schema.md#llm_clientsname). |
+| A judge client without `timeout_ms` | A judge that accepts the request and never finishes holds the request open. | None. | Give the judge its own `[llm_clients]` entry with `timeout_ms`. See [Tuning options](../routing_algorithms/llm_classifier_routing.md#tuning-options). |
+| `auto` or `stage_router` serving chat-only traffic | No tool results, so the signals abstain and every turn falls open to the efficient tier. | None. | Use Task or Composite. See [When routing will not help](../routing_algorithms/overview.md#when-routing-will-not-help). |
+| `classifier.classify_trigger = "user_turn"` or `"new_session"`, with or without `classifier.message_hash_fallback`, on a `stage_router` route | The stage fallback judge runs on every ambiguous turn and retains nothing, so the trigger changes nothing. The keys are accepted for compatibility. `message_hash_fallback` with the default trigger is a load error instead. | None. | Use a [composite](../routing_algorithms/composite_routing.md) route for a judge that runs once per user turn. |
+| `capable_first_threshold` on a standalone `stage_router` route | Only a composite classifier selects the tier that uses it. | None. | Use `confidence_threshold`, or a composite route. See the [knob reference](../routing_algorithms/stage_router_routing.md#knob-reference). |
+| A custom `llm_classifier` with one routing target | Every judge call is paid for and no choice is made. | Startup: `custom classifier has only one routing target; judge calls add cost without a routing choice`. | Add a second target, or use `passthrough`. |
+
+### Two of these rows, as files that pass `--dry-run`
+
+The capability keys below are ignored because the route runs in escalation
+mode:
+
+```toml
+[routes.agent]
+id = "agent"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 0.5 # ignored: the escalation table selects escalation mode
+threshold_step = 0.1 # ignored
+escalation = { confirmations = 2 }
+```
+
+Add `mode = "escalation"` and the same file fails to load with
+`mode escalation cannot use capability routing settings`.
+
+The threshold below can never be reached by the signals, so this route serves
+the capable tier for every ordinary turn:
+
+```toml
+[routes.stage]
+id = "stage"
+type = "stage_router"
+capable_target = "strong"
+efficient_target = "weak"
+picker = "capable_first"
+confidence_threshold = 0.9 # the signals reach 0.8741 at most toward efficient
+```
+
+## Related
+
+- [Verify routing is active](../getting_started.md#verify-routing-is-active)
+- [Tuning](../tuning/index.md)
+- [Model Selection](../routing_algorithms/model_selection.md)
+- [TOML Schema](../reference/toml_schema.md)
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index b70ba9531..fc1d6c09e 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -355,25 +355,60 @@ mode, or in the prompt in `json_object` mode.
### `stage_router`
Scores tool signals to pick a tier per turn. See
-[Stage-Router Routing](../routing_algorithms/stage_router_routing.md) for the
-optional `handoff_notes` and `classifier` tables and for tuning.
+[Stage-Router Routing](../routing_algorithms/stage_router_routing.md) for how
+the signals work and for tuning.
| Key | Required | Default | Meaning |
|---|:---:|---|---|
| `capable_target` | Yes | — | Capable tier. |
| `efficient_target` | Yes | — | Efficient tier. |
| `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. |
-| `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. |
+| `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. `efficient_first_threshold` is accepted as an alias. |
+| `capable_first_threshold` | No | `0.0` | Threshold used after an outer classifier selects the capable tier. Only a [composite](#composite) route sets that tier, so this key has no effect on a standalone `stage_router` route. |
| `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. |
-| `capable_hold_turns` | No | `2` | Requests kept on the capable tier after escalation. A clean test pass clears the hold early; `0` disables it. |
+| `capable_hold_turns` | No | `2` | Requests kept on the capable tier after an escalation the signals or a hard override picked. A clean test pass clears the hold early; `0` disables it. |
| `tool_semantics.observe` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as read-only investigation. |
| `tool_semantics.mutate` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as state-changing production. |
| `tool_semantics.plan` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as planning or task decomposition. |
| `tool_semantics.new` | No | `[]` | Exact ASCII case-insensitive domain tool names that demonstrate forward activity without favoring either tier. |
-| `classifier.classify_trigger` | No | `every_request` | When the judge runs. See the `llm_classifier` route. `new_session` has no effect here. |
-| `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. |
+| `handoff_notes` | No | unset | Table of notes appended to the forwarded request when the signals pick a tier. Keys below. |
+| `classifier` | No | unset | Table for the optional capability judge consulted on ambiguous turns. Keys below. |
| `subagents` | No | unset | Nested `passthrough` or custom `llm_classifier` policy used only for delegated sub-agent work. See [Sub-Agent-Aware Routing](../routing_algorithms/subagent_routing.md). |
+#### `handoff_notes` keys
+
+| Key | Required | Default | Meaning |
+|---|:---:|---|---|
+| `handoff_notes.escalation_note` | Yes, when the table is present | — | Note appended to the request when the signals or a hard override escalate to the capable tier. |
+| `handoff_notes.deescalation_note` | No | unset | Note appended to the request whenever the signals select the efficient tier, whether or not a capable turn came before. |
+| `handoff_notes.only_on_wrong_signal_escalation` | No | `true` | When `true`, only the escalating turn itself carries the escalation note. `false` also adds it to the capable-hold turns that follow. |
+
+#### `classifier` keys
+
+The judge runs when the signals are ambiguous or the request has no tool
+history. It runs as a step inside the stage cascade, not as a standalone route,
+so it is consulted on every ambiguous turn and never retains a verdict.
+
+| Key | Required | Default | Meaning |
+|---|:---:|---|---|
+| `classifier.target` | Yes, when the table is present | — | Target the judge is called through. Not a routing destination. |
+| `classifier.base_threshold` | For LLM judges | — | Lowest `p_solve` that still routes to the efficient tier. In `[0, 1]`. Omit with `classifier.decision`. |
+| `classifier.threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
+| `classifier.fail_open` | No | `true` | When `true`, judge client failures and deadlines let the turn fall open to the picker default and record fail-open evidence. When `false`, they stop the request. |
+| `classifier.decision` | No | unset | Use a decision model instead of an LLM judge. The classifier target must be in `decision_targets`. It cannot be combined with `base_threshold`, `threshold_step`, `prompt`, `response_format_type`, or `max_output_tokens`; the load error is `decision cannot use LLM judge settings`. See [Decision model classifier](../routing_algorithms/composite_routing.md#decision-model-classifier). |
+| `classifier.classify_trigger` | No | `every_request` | Accepted for compatibility with `llm_classifier`. No effect on the stage fallback judge: it runs on every ambiguous turn and retains no verdict, so `user_turn` and `new_session` change nothing here. Use a [composite](#composite) route to judge once per user turn. |
+| `classifier.message_hash_fallback` | No | `false` | Accepted for compatibility. No effect on the stage fallback judge. Setting it with the default trigger is a load error. |
+| `classifier.recent_turn_window` | No | unset | When unset, the judge sees the opening task and the latest user follow-up. When set, it also sees that many trailing messages. This counts messages, while the route's `recent_turn_window` counts tool results, so the two values are different units. |
+| `classifier.prompt` | No | packaged prompt | Replaces the capability judge prompt. The verdict schema is unchanged. |
+| `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the LLM judge. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Not with `classifier.decision`. |
+| `classifier.max_output_tokens` | No | `4096` | Maximum completion tokens for the LLM judge verdict. Must be at least `1`. Not with `classifier.decision`. |
+
+`classifier.response_schema`, `classifier.policy`, and any `classifier.mode`
+other than `capability` are rejected on a `stage_router` route with
+`custom classifier settings require composite with mode = custom`.
+`classifier.mode = "capability"` is accepted and changes nothing. Use a
+[composite](#composite) route for a custom judge.
+
### `auto`
`auto` is a preset. It builds a `stage_router` route with fixed settings and no
@@ -427,7 +462,7 @@ confidence threshold. Tool signals still drive escalation and recovery. See
| `classifier.base_threshold` | For LLM judges | unset | `p_solve` floor that still routes to the efficient tier. In `[0, 1]`. Omit with `classifier.decision`. |
| `classifier.threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
| `classifier.classify_trigger` | Yes | — | `user_turn` re-picks the tier whenever the user speaks, `new_session` picks once and holds it. `every_request` is rejected here: a judge call per tool step is the cost this route exists to avoid. |
-| `classifier.decision` | No | unset | Use a decision model instead of an LLM judge. The classifier target must be in `decision_targets`. |
+| `classifier.decision` | No | unset | Use a decision model instead of an LLM judge. The classifier target must be in `decision_targets`. It cannot be combined with the LLM judge keys (`base_threshold`, `threshold_step`, `prompt`, `response_format_type`, `max_output_tokens`). |
| `classifier.fail_open` | No | `true` | On judge failure, retain the last classified tier or default to efficient. Set `false` to return the error. |
| `classifier.message_hash_fallback` | No | `false` | Retains the tier by hashing the first user message, for clients that send no session ID. Works with either supported trigger. Conversations opening with the same text share a tier. |
| `classifier.recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |
diff --git a/docs/routing_algorithms/advisor_gate_routing.md b/docs/routing_algorithms/advisor_gate_routing.md
index 5af0077b3..6c78ad7e1 100644
--- a/docs/routing_algorithms/advisor_gate_routing.md
+++ b/docs/routing_algorithms/advisor_gate_routing.md
@@ -61,20 +61,17 @@ stamp every request of one evaluation with it, sub-agents included, so the
budget means "reviews for this task" even behind a gateway shared by many
tasks — then the session id resolved from harness headers (such as
`x-switchyard-session-id`), and one scope per server when neither is present.
-For hosts that return client errors to the core algorithm, a failed consult refunds
-the review budget and counts toward a separate cap of 3. The Rust runner stops on
-HTTP client errors before returning them to the algorithm, so failed HTTP calls
-do not refund this budget. An unparseable verdict still refunds the budget and
-passes the turn through as APPROVE.
+A failed consult refunds the review budget and counts toward a separate cap of 3.
+An unparseable verdict also refunds the budget and passes the turn through as
+APPROVE.
Long sessions are truncated middle-out before the consult: the transcript keeps
the task statement at the start and the most recent work at the end, marked
`...<middle of the conversation truncated>...`, capped at
-`transcript_max_chars`. `fail_open` controls failures returned to the core
-algorithm by its host. The Rust runner stops the request after an HTTP client's
-configured retries fail or its deadline expires, even with `fail_open = true`.
-HTTP client errors also stop the request when `timeout_ms` is unset. A timeout
-before the answer starts returns `504`.
+`transcript_max_chars`. `fail_open` (default `true`) lets the buffered executor
+turn through when the advisor call fails after its client's retries or deadline,
+and `/v1/stats` counts the failure under `advisor_gate.consult_failures`. With
+`fail_open = false` the request stops with a server error instead.
Gate behavior is observable at `/v1/stats` under `advisor_gate`: verdicts by
trigger, consult failures by reason, and REDO-discarded turns with their token
diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md
index 372de7db4..f0fd2bb3c 100644
--- a/docs/routing_algorithms/overview.md
+++ b/docs/routing_algorithms/overview.md
@@ -25,7 +25,123 @@ session. Execution uses the stage router, which reads tool-result history and
can optionally call an LLM judge. The TOML configuration keys are unchanged.
Whichever strategy you choose, the two models you put behind it matter more than
-any threshold. See [Model Selection](model_selection.md).
+any threshold. See [Model Selection](model_selection.md). To tune cost against
+quality, follow the [tuning loop](../tuning/index.md).
+
+### What the names mean
+
+| Name | Route `type` | What it does |
+|---|---|---|
+| [Auto](#auto) | `auto` | The Execution preset with fixed settings. |
+| [Task](llm_classifier_routing.md) | `llm_classifier`, `mode = "capability"` | A judge reads the task before the turn is served and picks the strong or weak model. |
+| [Execution](stage_router_routing.md) | `stage_router` | Tool results from the last few turns pick the capable or efficient model for each turn. |
+| [Composite](composite_routing.md) | `composite` | A judge picks the starting tier at each user turn. Execution runs the turns in between. |
+| [Plan/Execute](plan_execute_routing.md) | `plan_execute` | The capable model plans. The first file edit hands the session to the efficient model. |
+| [Escalation](escalation_router_routing.md) | `llm_classifier`, `mode = "escalation"` | The weak model serves until a judge reports the same failure on consecutive turns. Then the session latches to the strong model. |
+| [Custom](llm_classifier_routing.md#custom-multi-target-routing) | `llm_classifier`, `mode = "custom"` | Your own judge prompt and schema choose among any number of targets. |
+| [Advisor Gate](advisor_gate_routing.md) | `advisor` | One executor serves every turn. A stronger advisor reviews its plans and "done" claims. |
+| [Verification-Gated](vgr_routing.md) | `vgr` | A local model drafts. Verified answers stay local, and everything else goes to the cloud model. |
+| [Prefill](prefill_routing.md) | `prefill_router` | A classifier you train on the prompt's hidden states picks the target. Experimental: Switchyard supplies no checkpoint, exporter, or encoder assets. |
+| [Random](random_routing.md) | `random` | A fixed traffic split. |
+| [Fixed](#direct-model-routes) | `passthrough` | One target, no decision. |
+| [Sub-agent](subagent_routing.md) | `subagents` table | A separate policy for delegated sub-agent work on `passthrough`, `stage_router`, and `composite` routes. |
+
+### Compare the strategies
+
+| Route `type` | Decides from | Extra model calls | Switches | Session ID | When it cannot decide | Measured | Start from |
+|---|---|---|---|---|---|---|---|
+| `auto` | Tool results in the request | None | Every request | Optional. Keeps the capable hold. | Efficient tier | The preset of the measured `stage_router` settings, with a shorter window. See [Auto](#auto). | The [Getting Started](../getting_started.md#server-path) route file |
+| `stage_router` | Tool results in the request | Optional judge on ambiguous turns | Every request | Optional. Keeps the capable hold. | Picker default | [Terminal-Bench 2.1 and DeepSWE results](../benchmarks.md) with the [DeepSWE v1.1 profile](../../benchmark/routing-profiles/deepswe-v11-stage-router-luna-sol.toml) | That profile |
+| `llm_classifier`, capability | A judge's view of the task | One judge call per request, user turn, or session | Per trigger | Optional. Needed to retain a verdict. | Strong target | [Terminal-Bench 2.1 results](../benchmarks.md#terminal-bench-21) for an LLM judge and for Jev as a decision judge. The LLM-judge config is not published; a [smoke-test config](../../benchmark/server-configs/tb-lite-llm-classifier-opus-kimi-gemini.toml) is checked in. | That config |
+| `llm_classifier`, escalation | A judge's view of the finished weak turn | One judge call per unlatched turn | Latches once. Reversible with de-escalation. | Required to latch. Without it, `confirmations = 1` escalates single turns only. | An unreadable verdict keeps the current tier. A failed judge call stops the request in the server. | [Terminal-Bench 2.1 results](../benchmarks.md#terminal-bench-21) with the [checked-in profile](../../benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml), and a [public report](https://www.langchain.com/blog/switchyard-agent-routing-benchmark) | That profile |
+| `llm_classifier`, custom | Your judge's verdict | One judge call per trigger | Per trigger | Optional. Needed to retain a verdict. | `default_target` | [tau2 profiles](../../benchmark/routing-profiles/tau2-telecom-custom-opus-qwen-balanced.toml) | That profile |
+| `composite` | A judge at each user turn, tool results in between | One judge call per user turn or session | Every request, with a per-user-turn default | Required, or `message_hash_fallback` | Last tier, else efficient | Not benchmarked | [Composite Routing](composite_routing.md#configuration) |
+| `plan_execute` | The first edit or write tool call | None | Once per session | Recommended. Latches the handoff. | Stays on the planner | [DeepSWE results](../benchmarks.md#deepswe) with the [DeepSWE v1.1 profile](../../benchmark/routing-profiles/deepswe-v11-plan-execute-luna-sol.toml) | That profile |
+| `advisor` | The executor's terminal turn | One advisor review per gated turn, up to `max_reviews` | Never. One executor serves. | Scopes the review budget | An unreadable verdict or a failed advisor call lets the buffered executor turn through (`fail_open`, default `true`). `fail_open = false` stops the request. | [DeepSWE results](../benchmarks.md#deepswe) with the [DeepSWE v1.1 profile](../../benchmark/routing-profiles/deepswe-v11-advisor-gate-luna-sol.toml) | That profile |
+| `vgr` | Evidence about a local candidate | Verifier and judge calls | Per final answer | See [the page](vgr_routing.md) | Cloud tier | Not benchmarked. Planned for 0.4.0. | [`examples/vgr-local-servers.toml`](../../examples/vgr-local-servers.toml) |
+| `prefill_router` | Hidden states of the prompt | Encoder inference, no provider call | Every request | Not required | Checkpoint output | Experimental. No checkpoint is supplied. | [Schema example](../reference/toml_schema.md#prefill_router) |
+| `random` | Weights | None | Every request | Not required | Not applicable | Utility | [Random Routing](random_routing.md) |
+| `passthrough` | Nothing | None | Never | Not required | Not applicable | Utility | [Schema](../reference/toml_schema.md#passthrough) |
+| `noop` | Nothing | None | Never | Not required | Not applicable | Smoke test | [Schema](../reference/toml_schema.md#noop) |
+
+Every route streams the served model's reply. Three hold a turn back first.
+Escalation buffers the weak reply until the judge rules on it. `advisor` holds a
+gated turn until the review finishes. `vgr` holds a local candidate until it is
+verified. Task judges before the turn is served, which adds the judge's latency
+to every judged request.
+
+"Measured" links the [Benchmarks](../benchmarks.md) page, the checked-in
+profile, or the public report a result comes from. The `capable_first` picker
+and `prefill_router` are experimental. See
+[Model Selection](model_selection.md#measured-pairs) for the measured model
+pairs.
+
+### When routing will not help
+
+Each strategy page lists the cases it does not fit:
+[stage router](stage_router_routing.md#when-not-to-use-stage-router) and
+[escalation](escalation_router_routing.md#when-not-to-use-escalation-routing).
+These cases apply across strategies:
+
+- **Requests carry no session ID.** Escalation never latches, at any
+ `confirmations`. Composite cannot retain the tier. Task with `user_turn` or
+ `new_session` judges every request. Plan/Execute needs the first edit to stay
+ in the request history. Execution keeps no capable hold. Send
+ `x-switchyard-session-id`, or use a strategy that decides from the request
+ alone.
+- **Both tiers resolve to the same model, or the efficient model is the stronger
+ one.** Routing then only adds cost. Run the
+ [inverted-pair check](model_selection.md#check-for-an-inverted-pair) first.
+- **Chat-only traffic on Auto or Execution.** With no tool results the signals
+ abstain and every turn lands on the efficient tier. Use Task or Composite,
+ which judge the request itself.
+- **Responses continuations by ID.** A client that sends `previous_response_id`
+ with new input, and no history, hides the tool calls that Execution and
+ Plan/Execute read. See
+ [Responses continuations by ID](llm_classifier_routing.md#responses-continuations-by-id)
+ and the
+ [Plan/Execute history requirement](plan_execute_routing.md#responses-api-history-requirement).
+- **A judge that cannot return schema-valid JSON.** Task then sends every request
+ to the strong target, Execution falls back to the picker default, and
+ escalation never latches. When the provider rejects the judge request
+ outright, escalation turns fail with that error instead. Run the
+ [structured output check](model_selection.md#2-structured-output) and see
+ [Judge model compatibility](llm_classifier_routing.md#judge-model-compatibility).
+- **A judge on every tool step.** Task's default `classify_trigger = "every_request"`
+ judges every tool continuation. An agent that makes twenty tool calls causes
+ twenty-one judge calls for one task. Use `user_turn` with a session ID, or
+ Execution, which reads the tool results without a model call. See
+ [When the judge runs](llm_classifier_routing.md#when-the-judge-runs).
+- **Sub-agent traffic without identity.** A `subagents` policy applies only when
+ the harness sends sub-agent identity. Otherwise delegated requests follow the
+ parent route. The server logs a warning for Claude Code versions known to lack
+ child identity; other harnesses give no log line. See
+ [Sub-Agent-Aware Routing](subagent_routing.md).
+
+### Start here
+
+Three starting points. Each names a strategy, a file in this repository to copy,
+and the one dial to move first.
+
+1. **A coding agent, and cost comes first.** Start with `auto`. When you need to
+ tune, copy the
+ [DeepSWE stage profile](../../benchmark/routing-profiles/deepswe-v11-stage-router-luna-sol.toml)
+ and move `confidence_threshold` as described in
+ [Tuning `confidence_threshold`](stage_router_routing.md#tuning-confidence_threshold).
+2. **Long sessions where the cheap model should run until a judge reports that it is failing.**
+ Use escalation. Copy the
+ [escalation profile](../../benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml),
+ send a session ID, and move `confirmations` as described in
+ [Tuning options](escalation_router_routing.md#tuning-options).
+3. **One executor you want to keep, with a stronger model as a quality gate.**
+ Use the advisor gate. Copy the
+ [DeepSWE advisor profile](../../benchmark/routing-profiles/deepswe-v11-advisor-gate-luna-sol.toml)
+ and move `max_reviews` together with `gate_min_tool_results` as described in
+ [Tuning](advisor_gate_routing.md#tuning).
+
+For multi-turn support traffic with few tool results, the
+[tau2 custom-classifier profiles](../../benchmark/routing-profiles/tau2-telecom-custom-opus-qwen-balanced.toml)
+are the measured starting point. Their dial is the judge rubric in `prompt`.
### Auto
@@ -76,28 +192,6 @@ Every published result comes from a coding agent whose requests carry tool
results. The [model selection guide](model_selection.md#measured-pairs) lists the
measured model pairs.
-## More options
-
-These options remain available when you need a different routing policy.
-
-| Strategy | Use it when | Route `type` |
-|---|---|---|
-| [Plan/Execute](plan_execute_routing.md) | Use a capable model to inspect and plan, then switch to an efficient model after the first file mutation. | `plan_execute` |
-| [Composite](composite_routing.md) | Combine Task and Execution. A classifier sets the stage router's default tier. | `composite` |
-| [Escalation](escalation_router_routing.md) | Start on the efficient model and escalate when an LLM judge detects trouble. | `llm_classifier` with `mode = "escalation"` |
-| [Custom](llm_classifier_routing.md#custom-multi-target-routing) | Route among two or more models using your own classification schema and rules. | `llm_classifier` with `mode = "custom"` |
-| [Advisor Gate](advisor_gate_routing.md) | Keep one executor model and have a stronger advisor review its plans and completion claims. | `advisor` |
-| [Sub-Agent-Aware Routing](subagent_routing.md) | Delegated sub-agents should use a separate routing policy from the parent agent. | `passthrough`, `stage_router`, or `composite` with `subagents` |
-| [Random Routing](random_routing.md) | You need a fixed traffic split for A/B tests, baselines, or cost experiments. | `random` |
-| [Verification-Gated Routing](vgr_routing.md) | Serve a local attempt only after bounded verification, and send the request to the capable model otherwise. | `vgr` |
-| [Fixed Model](#direct-model-routes) | Send every request to one target without a routing decision. | `passthrough` |
-
-### Experimental
-
-[Prefill Router](../reference/toml_schema.md#prefill_router) uses a
-checkpoint-backed classifier. It is experimental in v0.3.0. Switchyard does not
-provide or support a checkpoint, exporter, or compatible encoder assets.
-
## Common route shape
A deployment has three layers. LLM clients describe how to reach a provider,
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index 366ee9932..234b677fa 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -109,7 +109,9 @@ Both pickers read the same signals; only the default tier differs.
benchmarked, so there are no calibrated thresholds for it and no measured
accuracy or cost figures to set expectations against. The server logs a
warning at startup when a route selects it. Use `efficient_first` unless you
- are running your own calibration.
+ are running your own calibration. See
+ [Reachable scores per picker](#reachable-scores-per-picker) for what a
+ threshold can and cannot do with each picker.
## Tuning `confidence_threshold`
@@ -127,9 +129,48 @@ default.
`0.5` is the recommended starting point, derived from many coding benchmarks,
and what the example below uses.
+### Reachable scores
+
+The scorer adds a few signals from the recent tool results. Error severity and
+stalls count toward the capable tier. Writes count toward the efficient tier.
+Each signal is bounded, so a turn's confidence can only reach a few values.
+These are the ones that matter when you choose a threshold:
+
+| Situation in the recent window | Confidence | Direction |
+|---|---|---|
+| A soft error alone | 0.21 | Capable |
+| A hard error alone, or a stall alone | 0.46 | Capable |
+| Every operation is a write, with a soft error | 0.49 | Efficient |
+| One write among three operations and no error, the smallest clean-production score at the default window | 0.69 | Efficient |
+| A hard error plus a stall | 0.76 | Capable. The most the signals reach in this direction (0.7616). |
+| Every operation is a write and no error | 0.87 | Efficient. The most the signals reach in this direction (0.8741). |
+
+Production turns the stall signals off, so with any production in the window
+the capable side stays below 0.46. An error in the window removes the
+clean-production bonus, so with a hard error the efficient side reaches at most
+0.24. The unit test `scorer_ceilings_match_the_documented_values` in
+[`crates/libsy/src/algorithms/util/stage.rs`](../../crates/libsy/src/algorithms/util/stage.rs)
+pins these values, so a change to the scorer fails CI until this table is
+updated.
+
+### Reachable scores per picker
+
+Both pickers use the same score. The picker only sets which direction the
+signals must prove.
+
+| Picker | The signals move a turn to | Reachable confidence | What a threshold means |
+|---|---|---|---|
+| `efficient_first` | Capable (escalation) | Up to 0.7616 | At `0.5`, a hard error alone (0.46) stays in the band, so an escalation needs a second signal, a hard override, or the classifier. Below 0.46, a hard error alone escalates. At `0.77` and above, the signals never escalate; only overrides, capable holds, and the classifier can. |
+| `capable_first` | Efficient (offload) | Up to 0.8741, and at most 0.49 when an error is in the window | At `0.5`, clean production offloads (0.69 and up at the default window) and nothing else does. Below 0.49, production with a soft error in the window also offloads. At `0.88` and above, the signals never offload. |
+
+Lower the threshold to let the signals decide more turns. Raise it to keep more
+turns on the picker's default tier. Hard overrides, capable holds, and the
+classifier do not use the threshold. The server warns at startup that
+`capable_first` has no calibrated threshold.
+
### What `0.5` means with `efficient_first`
-![Illustrative Stage score distribution with an ambiguous band from -0.5 to 0.5. Scores below -0.5 route efficient, scores in the band fall back to efficient, and scores above 0.5 route capable.](../assets/stage-router-threshold.svg)
+![Illustrative Stage score distribution with an ambiguous band from -0.5 to 0.5. Scores below -0.5 route efficient, scores in the band fall back to efficient, and scores above 0.5 route capable. Dotted markers show the most the signals can reach: +0.76 toward capable and -0.87 toward efficient.](../assets/stage-router-threshold.svg)
With `picker = "efficient_first"`, no classifier, and a threshold of `0.5`:
@@ -174,6 +215,49 @@ running the full task set.
## Route configuration
+### Knob reference
+
+Every key a `stage_router` route accepts. The `handoff_notes` and `classifier`
+tables are optional. The
+[TOML schema](../reference/toml_schema.md#stage_router) has the full table with
+required columns.
+
+| Key | Default | Meaning |
+|---|---|---|
+| `capable_target` | required | Capable tier. |
+| `efficient_target` | required | Efficient tier. |
+| `picker` | required | `efficient_first` or `capable_first` (experimental). Tier used when the signals are not confident. |
+| `confidence_threshold` | required | Corroboration a decisive pick needs, `0` to `1`. Alias `efficient_first_threshold`. |
+| `capable_first_threshold` | `0.0` | Used only after an outer `composite` classifier selects the capable tier. No effect on a standalone route. |
+| `recent_turn_window` | `3` | Trailing tool results the signals are computed over. |
+| `capable_hold_turns` | `2` | Requests kept on the capable tier after an escalation the signals or a hard override picked. `0` disables. |
+| `tool_semantics.observe` | `[]` | Domain tool names that count as investigation. |
+| `tool_semantics.mutate` | `[]` | Domain tool names that count as production. |
+| `tool_semantics.plan` | `[]` | Domain tool names that count as planning. |
+| `tool_semantics.new` | `[]` | Domain tool names that count as neutral forward activity. |
+| `handoff_notes.escalation_note` | required in the table | Note handed to the capable tier when the signals or a hard override escalate. |
+| `handoff_notes.deescalation_note` | unset | Note handed to the efficient tier whenever the signals select it, with or without a capable turn before. |
+| `handoff_notes.only_on_wrong_signal_escalation` | `true` | `false` also adds the escalation note to capable-hold turns. |
+| `classifier.target` | required in the table | Judge target. Not a routing destination. |
+| `classifier.base_threshold` | required for LLM judges | Lowest `p_solve` that still routes efficient. |
+| `classifier.threshold_step` | `0.0` | Added once for uncertain or unmatched verdicts, twice for unsupported. |
+| `classifier.fail_open` | `true` | On judge failure, fall open to the picker default instead of failing the request. |
+| `classifier.decision` | unset | Use a decision model instead of an LLM judge. Excludes the LLM judge keys below. |
+| `classifier.classify_trigger` | `every_request` | No effect here. The judge runs on every ambiguous turn. |
+| `classifier.message_hash_fallback` | `false` | No effect here. A load error with the default trigger. |
+| `classifier.recent_turn_window` | unset | Trailing messages the judge sees after the opening task. A different unit from the route's `recent_turn_window`, which counts tool results. |
+| `classifier.prompt` | packaged | Replaces the judge prompt. |
+| `classifier.response_format_type` | `json_schema` | `json_object` for providers without JSON Schema. |
+| `classifier.max_output_tokens` | `4096` | Token budget for the verdict. |
+| `subagents` | unset | Separate policy for delegated sub-agent work. |
+
+`classifier.response_schema`, `classifier.policy`, and any `classifier.mode`
+other than `capability` belong to
+[composite](composite_routing.md#custom-llm-classifier) routes and are rejected
+here. `classifier.mode = "capability"` is accepted and changes nothing.
+
+### Minimal route
+
```toml
schema_version = 1
@@ -312,14 +396,16 @@ header:
### Decision sources
-The router records an internal `decision_source` for each turn to distinguish the
-paths through its cascade:
+The router labels each resolved turn with a `decision_source`. `/v1/stats`
+counts turns by it under `algorithm_stats.stage_router.routing_decisions`, and
+the `stage router override` log line carries it for overrides:
| Source | When |
|---|---|
| `override` | A repeated failure, critical-error severity, or context-compaction marker forced the capable tier. Structured logs set `override_reason` to `repeated_failure`, `critical_error`, or `compaction`. |
| `capable_hold` | A recent escalation kept this recovery turn on the capable tier. |
| `dimensions` | The signed score fell outside the ambiguous band and selected a tier. |
+| `ambiguous` | The signed score stayed inside the band. This label never reaches `/v1/stats` or the logs: the turn continues to the classifier or the picker default, and that later source is recorded. |
| `llm-classifier` | The signals were ambiguous and the classifier returned a usable verdict. |
| `fall_open` | The signal scorer abstained and the classifier was absent or could not decide, so the picker default was used. |
diff --git a/docs/tuning/index.md b/docs/tuning/index.md
new file mode 100644
index 000000000..f4d1d740e
--- /dev/null
+++ b/docs/tuning/index.md
@@ -0,0 +1,98 @@
+# Tuning
+
+Routing has two outcomes you can measure: how many tasks the agent completes,
+and what a run costs. Tuning is the loop below. Every step links the page that
+explains it, names what it produces, and says what it costs to run.
+
+Start where your goal is:
+
+| Your goal | Start at |
+|---|---|
+| Lower cost at the same quality | Step 1, then step 4 for the baseline numbers |
+| Add a cheaper model safely | Step 2 |
+| Routing is on, and results got worse | Step 5 |
+
+## The loop
+
+1. **Verify routing is active.** Follow
+ [Verify routing is active](../getting_started.md#verify-routing-is-active).
+ Produces: the selected-model header from two requests and a `/v1/stats`
+ capture. Costs: two requests, no agent run.
+
+2. **Choose and check the models.** Run the
+ [five checks](../routing_algorithms/model_selection.md#check-the-efficient-model-before-you-route)
+ on the efficient model, then the
+ [inverted-pair check](../routing_algorithms/model_selection.md#check-for-an-inverted-pair).
+ Produces: a pass or fail per check, and solved tasks plus cost for each
+ fixed arm. Costs: single requests for most checks, one short agent run for
+ the tool-calling check, and one agent run per arm on at least 20 tasks.
+
+3. **Start from a measured profile.** Pick the closest row in
+ [Measured pairs](../routing_algorithms/model_selection.md#measured-pairs)
+ and copy its profile from `benchmark/routing-profiles/`. The
+ [strategy comparison](../routing_algorithms/overview.md#compare-the-strategies)
+ says which route type fits your traffic. Produces: a route file that passes
+ `switchyard-server --config routes.toml --dry-run`. Costs: none.
+
+4. **Run the routed arm against the fixed arms.** Use the same tasks, agent,
+ and provider for all three arms. The [benchmark guide](../../benchmark/README.md)
+ shows the harness the checked-in results used, and the
+ [Benchmarks](../benchmarks.md) page lists the published results with their
+ configs. Produces: solved tasks and
+ cost per run for each arm. Costs: `k` runs per arm. The plan/execute and
+ advisor profiles use three runs and report the mean with the sample standard
+ deviation, as in the
+ [plan/execute profile header](../../benchmark/routing-profiles/deepswe-v11-plan-execute-luna-sol.toml).
+ The stage profile records a single run. A single run cannot separate a small
+ change from noise.
+
+5. **Read the decision mix.** For stage routes, read
+ [decision sources](../routing_algorithms/stage_router_routing.md#decision-sources)
+ from `/v1/stats`. For escalation routes, read the
+ [observability](../routing_algorithms/escalation_router_routing.md#observability)
+ section. For advisor routes, `/v1/stats` reports verdicts and consult
+ failures under `advisor_gate`. Start the server with `--routing-log-file`
+ to keep one record per model call: the served call, and each judge call
+ under the judge's model ID. Filter by `model` before adding up usage.
+ Produces: the `routing_decisions` capture and the routing log. Costs: none
+ beyond the runs in step 4.
+
+6. **Move one dial.** Change one setting, then repeat step 4.
+
+ | Strategy | First dial | Guidance |
+ |---|---|---|
+ | Execution, Auto | `confidence_threshold` | [Tuning `confidence_threshold`](../routing_algorithms/stage_router_routing.md#tuning-confidence_threshold) and [Calibrating the threshold from run data](../routing_algorithms/stage_router_routing.md#calibrating-the-threshold-from-run-data) |
+ | Task | `base_threshold`, then `classify_trigger` | [Tuning options](../routing_algorithms/llm_classifier_routing.md#tuning-options) and [When the judge runs](../routing_algorithms/llm_classifier_routing.md#when-the-judge-runs) |
+ | Escalation | `confirmations` | [Tuning options](../routing_algorithms/escalation_router_routing.md#tuning-options) |
+ | Advisor Gate | `max_reviews` with `gate_min_tool_results` | [Tuning](../routing_algorithms/advisor_gate_routing.md#tuning) |
+ | Composite | `efficient_first_threshold` and `capable_first_threshold` | [What exists today](../routing_algorithms/composite_routing.md#what-exists-today) |
+
+ Produces: a before-and-after table with the arm results from step 4.
+ Costs: `k` runs per setting you try.
+
+7. **Record the result.** Write down what you ran, in the form the profile
+ headers use:
+
+ - the Switchyard commit;
+ - each model with its provider, wire format, and reasoning effort;
+ - the agent and its version;
+ - the benchmark, the task count, and `k`;
+ - the result band next to the fixed arms;
+ - the cost per run.
+
+ The
+ [plan/execute profile header](../../benchmark/routing-profiles/deepswe-v11-plan-execute-luna-sol.toml)
+ records all of these. Produces: the header. Costs: none.
+
+8. **Recalibrate on change.** A new model version, provider, wire format,
+ reasoning effort, agent version, or workload moves the operating point, and
+ the thresholds do not carry over. See
+ [Pin the provider and endpoint](../routing_algorithms/model_selection.md#pin-the-provider-and-endpoint).
+ Return to step 4 with the recorded header as your baseline. Produces: a new
+ header next to the old one. Costs: the runs of step 4.
+
+## Related
+
+- [Routing Overview](../routing_algorithms/overview.md): choose a strategy.
+- [Model Selection](../routing_algorithms/model_selection.md): choose and check the models.
+- [Troubleshooting](../operations/troubleshooting.md): symptoms and silent misconfigurations.
diff --git a/mkdocs.yml b/mkdocs.yml
index 8d6f51518..0de16cb87 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -42,7 +42,10 @@ nav:
- Escalation-Router Routing: routing_algorithms/escalation_router_routing.md
- Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md
- Verification-Gated Routing: routing_algorithms/vgr_routing.md
+ - Tuning:
+ - Tuning Loop: tuning/index.md
- Operations:
+ - Troubleshooting: operations/troubleshooting.md
- Context-Window Handling: operations/context_window.md
- Soak Testing: operations/soak_test.md
- Reference:
diff --git a/tests/test_stage_router_knob_docs.py b/tests/test_stage_router_knob_docs.py
new file mode 100644
index 000000000..4e995e3ef
--- /dev/null
+++ b/tests/test_stage_router_knob_docs.py
@@ -0,0 +1,161 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Stage-router knob reference drift tests.
+
+The TOML schema and the stage-router guide each list every key a
+``stage_router`` route accepts, and name the shared classifier keys the route
+rejects. These tests derive both lists from the Rust config types, so a new,
+renamed, or removed knob fails here until both pages catch up, and a table row
+for a key the code does not accept fails too.
+"""
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+RUNNER_ALGORITHM = ROOT / "crates" / "switchyard-runner" / "src" / "algorithm.rs"
+LIBSY_STAGE = ROOT / "crates" / "libsy" / "src" / "algorithms" / "util" / "stage.rs"
+LIBSY_TOOL_SIGNALS = ROOT / "crates" / "libsy" / "src" / "algorithms" / "util" / "tool_signals.rs"
+SCHEMA = ROOT / "docs" / "reference" / "toml_schema.md"
+GUIDE = ROOT / "docs" / "routing_algorithms" / "stage_router_routing.md"
+
+# `StageClassifierConfig` is shared with composite routes. A stage_router route
+# rejects these fields (see `StageClassifierConfig::task_classifier_config`), so
+# the pages name them as rejected rather than listing them as knobs.
+REJECTED_CLASSIFIER_KEYS = {"classifier.response_schema", "classifier.policy"}
+
+FIELD = re.compile(r"^\s*(?:pub(?:\([a-z]+\))?\s+)?([a-z_][a-z0-9_]*)\s*:", re.M)
+SERDE_NAME = re.compile(r'(?:rename|alias)\s*=\s*"([^"]+)"')
+
+
+def block(source: str, start: str) -> str:
+ """The body of the struct or enum variant whose declaration starts with `start`."""
+ begin = source.index(start) + len(start)
+ depth = 1
+ for position in range(begin, len(source)):
+ char = source[position]
+ if char == "{":
+ depth += 1
+ elif char == "}":
+ depth -= 1
+ if depth == 0:
+ return source[begin:position]
+ raise AssertionError(f"unterminated block after {start!r}")
+
+
+def toml_keys(body: str) -> list[str]:
+ """TOML keys a serde struct body accepts: field names, renames, and aliases.
+
+ Returns one entry per field; a field with aliases yields a `|`-joined entry.
+ Fields marked `#[serde(flatten)]` are skipped, since their keys come from
+ another struct.
+ """
+ keys: list[str] = []
+ pending_attrs: list[str] = []
+ for line in body.splitlines():
+ stripped = line.strip()
+ if stripped.startswith("#["):
+ pending_attrs.append(stripped)
+ continue
+ if stripped.startswith("//") or not stripped:
+ continue
+ match = FIELD.match(line)
+ if match is None:
+ pending_attrs = []
+ continue
+ attrs = " ".join(pending_attrs)
+ pending_attrs = []
+ if "flatten" in attrs:
+ continue
+ names = SERDE_NAME.findall(attrs)
+ renamed = re.search(r'rename\s*=\s*"([^"]+)"', attrs)
+ primary = renamed.group(1) if renamed else match.group(1)
+ keys.append("|".join([primary, *(name for name in names if name != primary)]))
+ return keys
+
+
+def expected_keys() -> set[str]:
+ runner = RUNNER_ALGORITHM.read_text()
+ libsy = LIBSY_STAGE.read_text()
+ signals = LIBSY_TOOL_SIGNALS.read_text()
+ keys: set[str] = set()
+ for key in toml_keys(block(runner, "StageRouter {")):
+ keys.update(key.split("|"))
+ for key in toml_keys(block(runner, "pub struct StageTierConfig {")):
+ field = key.split("|")[0]
+ if field == "tool_semantics":
+ keys.update(
+ f"tool_semantics.{name}"
+ for name in toml_keys(block(signals, "pub struct ToolSemantics {"))
+ )
+ elif field == "handoff_notes":
+ keys.add("handoff_notes")
+ keys.update(
+ f"handoff_notes.{name}"
+ for name in toml_keys(block(libsy, "pub struct HandoffNoteConfig {"))
+ )
+ else:
+ keys.update(key.split("|"))
+ keys.add("classifier")
+ for key in toml_keys(block(runner, "pub struct StageClassifierConfig {")):
+ keys.update(f"classifier.{name}" for name in key.split("|"))
+ return keys - REJECTED_CLASSIFIER_KEYS
+
+
+def decision_sources() -> list[str]:
+ body = block(LIBSY_STAGE.read_text(), "impl DecisionSource {")
+ names = re.findall(r'Self::\w+ => "([a-z_-]+)"', body)
+ assert names, "DecisionSource::as_str arms not found"
+ return names
+
+
+def section(text: str, start: str, end: str) -> str:
+ begin = text.index(start)
+ stop = text.index(end, begin + len(start))
+ return text[begin:stop]
+
+
+def table_keys(text: str) -> set[str]:
+ """Keys named in the first column of the markdown tables in `text`."""
+ return set(re.findall(r"^\| `([a-z_][a-z0-9_.]*)` \|", text, re.M))
+
+
+def check_section(text: str) -> None:
+ """Every accepted key is named somewhere in the section (a table row, or
+ prose for aliases and keys the route rejects), and every table row names an
+ accepted key."""
+ expected = expected_keys()
+ mentioned = {key for key in expected if f"`{key}`" in text}
+ assert sorted(expected - mentioned) == [], "keys the code accepts but the page omits"
+ unnamed = {key for key in REJECTED_CLASSIFIER_KEYS if f"`{key}`" not in text}
+ assert sorted(unnamed) == [], "rejected keys the page does not name"
+ unknown = table_keys(text) - expected
+ assert sorted(unknown) == [], "keys the tables list but the code does not accept"
+
+
+def test_derived_keys_include_the_known_shape() -> None:
+ keys = expected_keys()
+ assert {
+ "picker",
+ "confidence_threshold",
+ "efficient_first_threshold",
+ "tool_semantics.observe",
+ "handoff_notes.escalation_note",
+ "classifier.target",
+ "subagents",
+ } <= keys
+
+
+def test_schema_lists_exactly_the_stage_router_keys() -> None:
+ check_section(section(SCHEMA.read_text(), "### `stage_router`", "### `auto`"))
+
+
+def test_guide_knob_reference_lists_exactly_the_stage_router_keys() -> None:
+ check_section(section(GUIDE.read_text(), "### Knob reference", "### Minimal route"))
+
+
+def test_guide_lists_every_decision_source() -> None:
+ text = section(GUIDE.read_text(), "### Decision sources", "## When")
+ missing = [name for name in decision_sources() if f"`{name}`" not in text]
+ assert missing == []