Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): advisor route type (#382)
eric-liu-nvidia8372cc962cfe
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 561390235..a78695cd5 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -9,10 +9,10 @@ use std::path::Path;use std::sync::Arc;use libsy::{- Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig,- CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig,- LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter,- StageRouterConfig, TargetPrompts, TaskClassifierConfig,+ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,+ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger,+ HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough,+ PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts, TaskClassifierConfig,};use serde::Deserialize;use serde_json::Value;@@ -456,6 +456,53 @@ enum RouteConfig {#[serde(default)]classifier: Option<StageClassifierConfig>,},+ Advisor {+ id: ModelId,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,+ #[serde(default)]+ reasoning: Option<bool>,+ /// Serves every client-visible turn; also the count_tokens target.+ executor_target: String,+ /// Reviews the executor's first terminal turn. Judge-only, never a+ /// routing destination.+ advisor_target: String,+ #[serde(default)]+ reviewer_system_prompt: Option<String>,+ #[serde(default)]+ redo_feedback_prefix: Option<String>,+ #[serde(default)]+ gate_trigger: AdvisorTriggerConfig,+ #[serde(default)]+ gate_trigger_pattern: Option<String>,+ #[serde(default = "default_max_reviews")]+ max_reviews: u32,+ #[serde(default)]+ gate_stall_turns: u32,+ #[serde(default)]+ gate_min_tool_results: u32,+ #[serde(default = "default_advisor_max_tokens")]+ advisor_max_tokens: u64,+ #[serde(default)]+ advisor_temperature: Option<f64>,+ #[serde(default = "default_transcript_max_chars")]+ transcript_max_chars: usize,+ #[serde(default = "default_fail_open")]+ fail_open: bool,+ },+}++/// What fires an advisor route's review.+#[derive(Debug, Default, Deserialize, PartialEq)]+#[serde(rename_all = "snake_case")]+enum AdvisorTriggerConfig {+ /// The executor's first turn without tool calls.+ #[default]+ NoToolCall,+ /// The first turn whose text matches `gate_trigger_pattern`.+ Pattern,}/// The judge a `stage_router` route falls through to, and how it routes.@@ -504,7 +551,8 @@ impl RouteConfig {| Random { id, .. }| LlmClassifier { id, .. }| Passthrough { id, .. }- | StageRouter { id, .. } => id,+ | StageRouter { id, .. }+ | Advisor { id, .. } => id,}}@@ -543,6 +591,11 @@ impl RouteConfig {efficient_target,..} => vec![capable_target, efficient_target],+ // The advisor is judge-only: reviews go through its own client,+ // so it is not a completion (or count_tokens) destination.+ Self::Advisor {+ executor_target, ..+ } => vec![executor_target],}}@@ -560,6 +613,7 @@ impl RouteConfig {classifier: Some(classifier),..} => names.push(&classifier.target),+ Self::Advisor { advisor_target, .. } => names.push(advisor_target),_ => {}}names@@ -597,6 +651,12 @@ impl RouteConfig {tool_calling,reasoning,..+ }+ | Advisor {+ context_window,+ tool_calling,+ reasoning,+ ..} => ModelCapabilities {context_window: *context_window,tool_calling: *tool_calling,@@ -1004,9 +1064,76 @@ fn build_algorithm(})?;Ok(Arc::new(algorithm))}+ RouteConfig::Advisor {+ executor_target,+ advisor_target,+ reviewer_system_prompt,+ redo_feedback_prefix,+ gate_trigger,+ gate_trigger_pattern,+ max_reviews,+ gate_stall_turns,+ gate_min_tool_results,+ advisor_max_tokens,+ advisor_temperature,+ transcript_max_chars,+ fail_open,+ ..+ } => {+ let executor = resolve_target_model_id(route_name, executor_target, targets)?;+ let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;+ // A pattern set under the default trigger would be silently+ // ignored; reject the misconfiguration instead.+ if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {+ return Err(ServerError::new(format!(+ "advisor route {route_name}: gate_trigger_pattern requires \+ gate_trigger = \"pattern\""+ )));+ }+ let mut config = AdvisorGateConfig::default();+ if let Some(prompt) = reviewer_system_prompt {+ config.reviewer_system_prompt = prompt.clone();+ }+ if let Some(prefix) = redo_feedback_prefix {+ config.redo_feedback_prefix = prefix.clone();+ }+ config.gate_trigger = match gate_trigger {+ AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,+ AdvisorTriggerConfig::Pattern => {+ GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())+ }+ };+ config.max_reviews = *max_reviews;+ config.gate_stall_turns = *gate_stall_turns;+ config.gate_min_tool_results = *gate_min_tool_results;+ config.advisor_max_tokens = *advisor_max_tokens;+ config.advisor_temperature = *advisor_temperature;+ config.transcript_max_chars = *transcript_max_chars;+ config.fail_open = *fail_open;+ let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {+ ServerError::new(format!("advisor route {route_name}: {error}"))+ })?;+ Ok(Arc::new(algorithm))+ }}}+const fn default_max_reviews() -> u32 {+ 1+}++const fn default_advisor_max_tokens() -> u64 {+ 2048+}++const fn default_transcript_max_chars() -> usize {+ 200_000+}++const fn default_fail_open() -> bool {+ true+}+fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {prompt.map_or_else(ClassifierContractConfig::default, |prompt| {ClassifierContractConfig::default().with_prompt(prompt)@@ -1665,4 +1792,122 @@ target = "azure");}}++ const ADVISOR_CONFIG: &str = r#"+schema_version = 1++[llm_clients.anthropic]+format = "anthropic_messages"+base_url = "https://example.test"++[targets.executor]+id = "executor/model"+llm_client = "anthropic"++[targets.advisor]+id = "advisor/model"+llm_client = "anthropic"++[routes.gated]+id = "switchyard/advisor"+type = "advisor"+executor_target = "executor"+advisor_target = "advisor"+"#;++ #[test]+ fn advisor_route_parses_with_defaults_and_builds() -> ServerResult<()> {+ let state = server_state_from_toml(ADVISOR_CONFIG)?;+ assert_eq!(state.models().collect::<Vec<_>>(), ["switchyard/advisor"]);+ Ok(())+ }++ #[test]+ fn advisor_route_accepts_every_gate_knob() -> ServerResult<()> {+ let tuned = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ concat!(+ "advisor_target = \"advisor\"\n",+ "reviewer_system_prompt = \"review it\"\n",+ "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",+ "gate_trigger = \"pattern\"\n",+ "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",+ "max_reviews = 2\n",+ "gate_stall_turns = 40\n",+ "gate_min_tool_results = 1\n",+ "advisor_max_tokens = 1024\n",+ "advisor_temperature = 0.0\n",+ "transcript_max_chars = 100000\n",+ "fail_open = false\n",+ "context_window = 200000\n",+ "tool_calling = true\n",+ "reasoning = true",+ ),+ );+ server_state_from_toml(&tuned)?;+ Ok(())+ }++ #[test]+ fn advisor_route_rejects_unknown_keys() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\nbogus_field = 1",+ );+ assert!(error_message(&invalid).contains("bogus_field"));+ }++ #[test]+ fn advisor_route_requires_both_targets() {+ let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");+ assert!(error_message(&missing).contains("advisor_target"));+ }++ #[test]+ fn advisor_route_rejects_unknown_target() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"missing\"",+ );+ assert!(error_message(&invalid).contains("missing"));+ }++ #[test]+ fn advisor_route_rejects_invalid_pattern() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",+ );+ assert!(error_message(&invalid).contains("not a valid regex"));+ }++ #[test]+ fn advisor_route_rejects_pattern_without_pattern_trigger() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",+ );+ assert!(+ error_message(&invalid)+ .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")+ );+ }++ #[test]+ fn advisor_route_pattern_trigger_requires_pattern() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",+ );+ assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));+ }++ #[test]+ fn advisor_route_rejects_zero_max_reviews() {+ let invalid = ADVISOR_CONFIG.replace(+ "advisor_target = \"advisor\"",+ "advisor_target = \"advisor\"\nmax_reviews = 0",+ );+ assert!(error_message(&invalid).contains("max_reviews must be at least 1"));+ }}diff --git a/docs/routing_algorithms/advisor_gate_routing.md b/docs/routing_algorithms/advisor_gate_routing.mdnew file mode 100644index 000000000..00b6d3a69--- /dev/null+++ b/docs/routing_algorithms/advisor_gate_routing.md@@ -0,0 +1,131 @@+# Advisor-Gate Routing++Advisor-gate routing pairs an **executor** (the model under test, serving every+client-visible turn) with a stronger **advisor** that acts as a quality gate.+The executor works the task with its own tools; when it produces a terminal+turn — a plan before doing the work, or a claim that the task is complete — the+advisor reviews the session transcript and either lets the turn through+(APPROVE) or discards it and sends the executor back to work with a concrete+plan (REDO). The advisor is judge-only: it reviews turns but never serves one,+so clients only ever see executor output.++This differs from the classifier and router strategies: those decide *which+model serves a turn*, while the advisor gate keeps one model serving and spends+the stronger model only on verdicts at the moments that decide task success.++## How it works++Every request routes to the executor. The gate buffers each executor turn,+decides whether it needs review, and only then releases it to the client.++A review fires on one of two triggers:++- **`no_tool_call`** (default) — the executor's first turn without tool calls,+ the natural "I'm done or I have a plan" moment on function-calling agent+ harnesses. `gate_min_tool_results` skips early chatty turns: a no-tool-call+ turn is only reviewed once the conversation carries at least that many tool+ results.+- **`pattern`** — the first turn whose visible text matches+ `gate_trigger_pattern`, for text-protocol harnesses where every turn lacks+ tool calls and completion is declared with a textual marker instead.++Independently, `gate_stall_turns` adds a mid-task checkpoint: when a+conversation reaches that many assistant turns without ever triggering, the+next turn is reviewed once — catching executors that grind without declaring+completion.++The advisor receives the serialized transcript (task, actions, results, and the+gated turn) under a reviewer contract that demands APPROVE or REDO as the first+word of its reply. On APPROVE the buffered turn replays to the client verbatim,+preserved provider events included. On REDO the turn is discarded — the client+never sees it — and the advisor's plan is appended as user feedback, the+executor re-invoked, and its continuation served instead.++```mermaid+%%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%%+flowchart LR+ t["executor turn"] --> g{"trigger fires?"}+ g -->|no| r["replay to client"]+ g -->|yes| c["advisor reviews transcript"]+ c -->|APPROVE| r+ c -->|REDO| f["discard turn, inject plan,\nre-invoke executor"]+ f --> r++ classDef box font-family:monospace,fill:none,stroke:#9aa0a6,stroke-width:1px;+ class t,g,r,c,f box;+```++Reviews draw from a per-session budget of `max_reviews`. The budget scope is+the caller's `proxy_x_session_id` header when present — benchmark harnesses+stamp every request of one evaluation with it, sub-agents included, so the+budget means "reviews for this task" even behind a gateway shared by many+tasks — and falls back to one scope per server otherwise. Failed consults+refund the budget and count toward a separate cap of 3, which bounds consult+latency against a down advisor. An unparseable verdict also refunds and passes+the turn through as APPROVE.++Long sessions are truncated middle-out before the consult: the transcript keeps+the task statement at the start and the most recent work at the end, marked+`...<middle of the conversation truncated>...`, capped at+`transcript_max_chars`. With `fail_open = true` (default) any advisor failure+degrades to APPROVE; `fail_open = false` surfaces it as a server error instead.++Gate behavior is observable at `/v1/stats` under `advisor_gate`: verdicts by+trigger, consult failures by reason, and REDO-discarded turns with their token+counts (the client never saw those turns, so terminal usage accounting alone+would miss them).++## Configuration++```toml+[targets.executor]+id = "small/model"+llm_client = "provider"++[targets.advisor]+id = "frontier/model"+llm_client = "provider"++[routes.gated]+id = "switchyard/gated"+type = "advisor"+executor_target = "executor"+advisor_target = "advisor"+max_reviews = 3+gate_stall_turns = 30+gate_min_tool_results = 3+```++| Key | Default | Meaning |+|---|---|---|+| `executor_target` | required | Serves every client-visible turn. |+| `advisor_target` | required | Reviews gated turns; never a routing destination. |+| `gate_trigger` | `"no_tool_call"` | What fires a review: `no_tool_call` or `pattern`. |+| `gate_trigger_pattern` | unset | Regex for the `pattern` trigger; required by and exclusive to it. |+| `max_reviews` | `1` | Review budget per session scope; later triggers re-review until spent. |+| `gate_stall_turns` | `0` (off) | Mid-task checkpoint after this many assistant turns. |+| `gate_min_tool_results` | `0` | Minimum tool results before a `no_tool_call` turn is reviewable. |+| `advisor_max_tokens` | `2048` | Output cap for each advisor consult. |+| `advisor_temperature` | unset | Sampling temperature for consults; omitted when unset. |+| `transcript_max_chars` | `200000` | Middle-out cap on the serialized transcript (~50k tokens). |+| `fail_open` | `true` | Advisor failure passes the turn through instead of erroring. |+| `reviewer_system_prompt` | built-in | Overrides the APPROVE/REDO reviewer contract. |+| `redo_feedback_prefix` | built-in | Overrides the prefix injected before a REDO plan. |++## Tuning++The defaults gate once per session at the first terminal turn. On agentic+coding harnesses, the configuration that benchmarked best is `max_reviews = 3`,+`gate_stall_turns = 30`, `gate_min_tool_results = 3`: skip the early commentary+turns, keep a mid-task checkpoint for grinders, and allow a re-review after a+REDO.++Expect the gate's value to depend on the executor. On Terminal-Bench 2.1 with a+coding agent it lifted a weak executor by 11 points (43.8% → 54.7% ± 0.7,+k=3) by catching premature "done" claims and stalls, while on a strong executor+it only matched the accuracy of the takeover-style routers — strong executors+rarely produce the diligence failures a review can catch. If your executor is+already frontier-class, benchmark+[stage-router routing](stage_router_routing.md) first; reach for the advisor+gate when the executor is markedly weaker than the best model you can call, or+when you need a review trail rather than a model swap.diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.mdindex e5cd4299f..dad214d8e 100644--- a/docs/routing_algorithms/overview.md+++ b/docs/routing_algorithms/overview.md@@ -16,6 +16,7 @@ configuration and tuning. For the vocabulary these pages use, see| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm_classifier` || [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` || [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `llm_classifier` with `escalation` |+| [Advisor-Gate Routing](advisor_gate_routing.md) | One model should serve every turn, with a stronger reviewer approving its "done" claims or sending back a redo plan. | `advisor` |## Common route shapediff --git a/mkdocs.yml b/mkdocs.ymlindex d081dab58..fcf40f715 100644--- a/mkdocs.yml+++ b/mkdocs.yml@@ -29,6 +29,7 @@ nav:- LLM Classifier Routing: routing_algorithms/llm_classifier_routing.md- Stage-Router Routing: routing_algorithms/stage_router_routing.md- Escalation-Router Routing: routing_algorithms/escalation_router_routing.md+ - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md- Operations:- Context-Window Handling: operations/context_window.md- Reference: