Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): advisor route type (#382)

eric-liu-nvidia8372cc962cfe
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 561390235..a78695cd5 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -9,10 +9,10 @@ use std::path::Path;
use std::sync::Arc;
use libsy::{
- Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig,
- CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig,
- LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter,
- StageRouterConfig, TargetPrompts, TaskClassifierConfig,
+ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat,
+ CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger,
+ HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough,
+ PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts, TaskClassifierConfig,
};
use serde::Deserialize;
use serde_json::Value;
@@ -456,6 +456,53 @@ enum RouteConfig {
#[serde(default)]
classifier: Option<StageClassifierConfig>,
},
+ Advisor {
+ id: ModelId,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
+ #[serde(default)]
+ reasoning: Option<bool>,
+ /// Serves every client-visible turn; also the count_tokens target.
+ executor_target: String,
+ /// Reviews the executor's first terminal turn. Judge-only, never a
+ /// routing destination.
+ advisor_target: String,
+ #[serde(default)]
+ reviewer_system_prompt: Option<String>,
+ #[serde(default)]
+ redo_feedback_prefix: Option<String>,
+ #[serde(default)]
+ gate_trigger: AdvisorTriggerConfig,
+ #[serde(default)]
+ gate_trigger_pattern: Option<String>,
+ #[serde(default = "default_max_reviews")]
+ max_reviews: u32,
+ #[serde(default)]
+ gate_stall_turns: u32,
+ #[serde(default)]
+ gate_min_tool_results: u32,
+ #[serde(default = "default_advisor_max_tokens")]
+ advisor_max_tokens: u64,
+ #[serde(default)]
+ advisor_temperature: Option<f64>,
+ #[serde(default = "default_transcript_max_chars")]
+ transcript_max_chars: usize,
+ #[serde(default = "default_fail_open")]
+ fail_open: bool,
+ },
+}
+
+/// What fires an advisor route's review.
+#[derive(Debug, Default, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+enum AdvisorTriggerConfig {
+ /// The executor's first turn without tool calls.
+ #[default]
+ NoToolCall,
+ /// The first turn whose text matches `gate_trigger_pattern`.
+ Pattern,
}
/// The judge a `stage_router` route falls through to, and how it routes.
@@ -504,7 +551,8 @@ impl RouteConfig {
| Random { id, .. }
| LlmClassifier { id, .. }
| Passthrough { id, .. }
- | StageRouter { id, .. } => id,
+ | StageRouter { id, .. }
+ | Advisor { id, .. } => id,
}
}
@@ -543,6 +591,11 @@ impl RouteConfig {
efficient_target,
..
} => vec![capable_target, efficient_target],
+ // The advisor is judge-only: reviews go through its own client,
+ // so it is not a completion (or count_tokens) destination.
+ Self::Advisor {
+ executor_target, ..
+ } => vec![executor_target],
}
}
@@ -560,6 +613,7 @@ impl RouteConfig {
classifier: Some(classifier),
..
} => names.push(&classifier.target),
+ Self::Advisor { advisor_target, .. } => names.push(advisor_target),
_ => {}
}
names
@@ -597,6 +651,12 @@ impl RouteConfig {
tool_calling,
reasoning,
..
+ }
+ | Advisor {
+ context_window,
+ tool_calling,
+ reasoning,
+ ..
} => ModelCapabilities {
context_window: *context_window,
tool_calling: *tool_calling,
@@ -1004,9 +1064,76 @@ fn build_algorithm(
})?;
Ok(Arc::new(algorithm))
}
+ RouteConfig::Advisor {
+ executor_target,
+ advisor_target,
+ reviewer_system_prompt,
+ redo_feedback_prefix,
+ gate_trigger,
+ gate_trigger_pattern,
+ max_reviews,
+ gate_stall_turns,
+ gate_min_tool_results,
+ advisor_max_tokens,
+ advisor_temperature,
+ transcript_max_chars,
+ fail_open,
+ ..
+ } => {
+ let executor = resolve_target_model_id(route_name, executor_target, targets)?;
+ let advisor = resolve_target_model_id(route_name, advisor_target, targets)?;
+ // A pattern set under the default trigger would be silently
+ // ignored; reject the misconfiguration instead.
+ if *gate_trigger == AdvisorTriggerConfig::NoToolCall && gate_trigger_pattern.is_some() {
+ return Err(ServerError::new(format!(
+ "advisor route {route_name}: gate_trigger_pattern requires \
+ gate_trigger = \"pattern\""
+ )));
+ }
+ let mut config = AdvisorGateConfig::default();
+ if let Some(prompt) = reviewer_system_prompt {
+ config.reviewer_system_prompt = prompt.clone();
+ }
+ if let Some(prefix) = redo_feedback_prefix {
+ config.redo_feedback_prefix = prefix.clone();
+ }
+ config.gate_trigger = match gate_trigger {
+ AdvisorTriggerConfig::NoToolCall => GateTrigger::NoToolCall,
+ AdvisorTriggerConfig::Pattern => {
+ GateTrigger::Pattern(gate_trigger_pattern.clone().unwrap_or_default())
+ }
+ };
+ config.max_reviews = *max_reviews;
+ config.gate_stall_turns = *gate_stall_turns;
+ config.gate_min_tool_results = *gate_min_tool_results;
+ config.advisor_max_tokens = *advisor_max_tokens;
+ config.advisor_temperature = *advisor_temperature;
+ config.transcript_max_chars = *transcript_max_chars;
+ config.fail_open = *fail_open;
+ let algorithm = AdvisorGate::new(executor, advisor, config).map_err(|error| {
+ ServerError::new(format!("advisor route {route_name}: {error}"))
+ })?;
+ Ok(Arc::new(algorithm))
+ }
}
}
+const fn default_max_reviews() -> u32 {
+ 1
+}
+
+const fn default_advisor_max_tokens() -> u64 {
+ 2048
+}
+
+const fn default_transcript_max_chars() -> usize {
+ 200_000
+}
+
+const fn default_fail_open() -> bool {
+ true
+}
+
fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig {
prompt.map_or_else(ClassifierContractConfig::default, |prompt| {
ClassifierContractConfig::default().with_prompt(prompt)
@@ -1665,4 +1792,122 @@ target = "azure"
);
}
}
+
+ const ADVISOR_CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.anthropic]
+format = "anthropic_messages"
+base_url = "https://example.test"
+
+[targets.executor]
+id = "executor/model"
+llm_client = "anthropic"
+
+[targets.advisor]
+id = "advisor/model"
+llm_client = "anthropic"
+
+[routes.gated]
+id = "switchyard/advisor"
+type = "advisor"
+executor_target = "executor"
+advisor_target = "advisor"
+"#;
+
+ #[test]
+ fn advisor_route_parses_with_defaults_and_builds() -> ServerResult<()> {
+ let state = server_state_from_toml(ADVISOR_CONFIG)?;
+ assert_eq!(state.models().collect::<Vec<_>>(), ["switchyard/advisor"]);
+ Ok(())
+ }
+
+ #[test]
+ fn advisor_route_accepts_every_gate_knob() -> ServerResult<()> {
+ let tuned = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ concat!(
+ "advisor_target = \"advisor\"\n",
+ "reviewer_system_prompt = \"review it\"\n",
+ "redo_feedback_prefix = \"REVIEWER SAYS: \"\n",
+ "gate_trigger = \"pattern\"\n",
+ "gate_trigger_pattern = 'task_complete[\"\\s>:]*true'\n",
+ "max_reviews = 2\n",
+ "gate_stall_turns = 40\n",
+ "gate_min_tool_results = 1\n",
+ "advisor_max_tokens = 1024\n",
+ "advisor_temperature = 0.0\n",
+ "transcript_max_chars = 100000\n",
+ "fail_open = false\n",
+ "context_window = 200000\n",
+ "tool_calling = true\n",
+ "reasoning = true",
+ ),
+ );
+ server_state_from_toml(&tuned)?;
+ Ok(())
+ }
+
+ #[test]
+ fn advisor_route_rejects_unknown_keys() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\nbogus_field = 1",
+ );
+ assert!(error_message(&invalid).contains("bogus_field"));
+ }
+
+ #[test]
+ fn advisor_route_requires_both_targets() {
+ let missing = ADVISOR_CONFIG.replace("advisor_target = \"advisor\"\n", "");
+ assert!(error_message(&missing).contains("advisor_target"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_unknown_target() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"missing\"",
+ );
+ assert!(error_message(&invalid).contains("missing"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_invalid_pattern() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"\ngate_trigger_pattern = \"(unclosed\"",
+ );
+ assert!(error_message(&invalid).contains("not a valid regex"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_pattern_without_pattern_trigger() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger_pattern = \"done\"",
+ );
+ assert!(
+ error_message(&invalid)
+ .contains("gate_trigger_pattern requires gate_trigger = \"pattern\"")
+ );
+ }
+
+ #[test]
+ fn advisor_route_pattern_trigger_requires_pattern() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\ngate_trigger = \"pattern\"",
+ );
+ assert!(error_message(&invalid).contains("non-empty gate_trigger_pattern"));
+ }
+
+ #[test]
+ fn advisor_route_rejects_zero_max_reviews() {
+ let invalid = ADVISOR_CONFIG.replace(
+ "advisor_target = \"advisor\"",
+ "advisor_target = \"advisor\"\nmax_reviews = 0",
+ );
+ assert!(error_message(&invalid).contains("max_reviews must be at least 1"));
+ }
}
diff --git a/docs/routing_algorithms/advisor_gate_routing.md b/docs/routing_algorithms/advisor_gate_routing.md
new file mode 100644
index 000000000..00b6d3a69
--- /dev/null
+++ b/docs/routing_algorithms/advisor_gate_routing.md
@@ -0,0 +1,131 @@
+# Advisor-Gate Routing
+
+Advisor-gate routing pairs an **executor** (the model under test, serving every
+client-visible turn) with a stronger **advisor** that acts as a quality gate.
+The executor works the task with its own tools; when it produces a terminal
+turn — a plan before doing the work, or a claim that the task is complete — the
+advisor reviews the session transcript and either lets the turn through
+(APPROVE) or discards it and sends the executor back to work with a concrete
+plan (REDO). The advisor is judge-only: it reviews turns but never serves one,
+so clients only ever see executor output.
+
+This differs from the classifier and router strategies: those decide *which
+model serves a turn*, while the advisor gate keeps one model serving and spends
+the stronger model only on verdicts at the moments that decide task success.
+
+## How it works
+
+Every request routes to the executor. The gate buffers each executor turn,
+decides whether it needs review, and only then releases it to the client.
+
+A review fires on one of two triggers:
+
+- **`no_tool_call`** (default) — the executor's first turn without tool calls,
+ the natural "I'm done or I have a plan" moment on function-calling agent
+ harnesses. `gate_min_tool_results` skips early chatty turns: a no-tool-call
+ turn is only reviewed once the conversation carries at least that many tool
+ results.
+- **`pattern`** — the first turn whose visible text matches
+ `gate_trigger_pattern`, for text-protocol harnesses where every turn lacks
+ tool calls and completion is declared with a textual marker instead.
+
+Independently, `gate_stall_turns` adds a mid-task checkpoint: when a
+conversation reaches that many assistant turns without ever triggering, the
+next turn is reviewed once — catching executors that grind without declaring
+completion.
+
+The advisor receives the serialized transcript (task, actions, results, and the
+gated turn) under a reviewer contract that demands APPROVE or REDO as the first
+word of its reply. On APPROVE the buffered turn replays to the client verbatim,
+preserved provider events included. On REDO the turn is discarded — the client
+never sees it — and the advisor's plan is appended as user feedback, the
+executor re-invoked, and its continuation served instead.
+
+```mermaid
+%%{init: {"flowchart": {"nodeSpacing": 18, "rankSpacing": 26}}}%%
+flowchart LR
+ t["executor turn"] --> g{"trigger fires?"}
+ g -->|no| r["replay to client"]
+ g -->|yes| c["advisor reviews transcript"]
+ c -->|APPROVE| r
+ c -->|REDO| f["discard turn, inject plan,\nre-invoke executor"]
+ f --> r
+
+ classDef box font-family:monospace,fill:none,stroke:#9aa0a6,stroke-width:1px;
+ class t,g,r,c,f box;
+```
+
+Reviews draw from a per-session budget of `max_reviews`. The budget scope is
+the caller's `proxy_x_session_id` header when present — benchmark harnesses
+stamp every request of one evaluation with it, sub-agents included, so the
+budget means "reviews for this task" even behind a gateway shared by many
+tasks — and falls back to one scope per server otherwise. Failed consults
+refund the budget and count toward a separate cap of 3, which bounds consult
+latency against a down advisor. An unparseable verdict also refunds and passes
+the turn through as APPROVE.
+
+Long sessions are truncated middle-out before the consult: the transcript keeps
+the task statement at the start and the most recent work at the end, marked
+`...<middle of the conversation truncated>...`, capped at
+`transcript_max_chars`. With `fail_open = true` (default) any advisor failure
+degrades to APPROVE; `fail_open = false` surfaces it as a server error instead.
+
+Gate behavior is observable at `/v1/stats` under `advisor_gate`: verdicts by
+trigger, consult failures by reason, and REDO-discarded turns with their token
+counts (the client never saw those turns, so terminal usage accounting alone
+would miss them).
+
+## Configuration
+
+```toml
+[targets.executor]
+id = "small/model"
+llm_client = "provider"
+
+[targets.advisor]
+id = "frontier/model"
+llm_client = "provider"
+
+[routes.gated]
+id = "switchyard/gated"
+type = "advisor"
+executor_target = "executor"
+advisor_target = "advisor"
+max_reviews = 3
+gate_stall_turns = 30
+gate_min_tool_results = 3
+```
+
+| Key | Default | Meaning |
+|---|---|---|
+| `executor_target` | required | Serves every client-visible turn. |
+| `advisor_target` | required | Reviews gated turns; never a routing destination. |
+| `gate_trigger` | `"no_tool_call"` | What fires a review: `no_tool_call` or `pattern`. |
+| `gate_trigger_pattern` | unset | Regex for the `pattern` trigger; required by and exclusive to it. |
+| `max_reviews` | `1` | Review budget per session scope; later triggers re-review until spent. |
+| `gate_stall_turns` | `0` (off) | Mid-task checkpoint after this many assistant turns. |
+| `gate_min_tool_results` | `0` | Minimum tool results before a `no_tool_call` turn is reviewable. |
+| `advisor_max_tokens` | `2048` | Output cap for each advisor consult. |
+| `advisor_temperature` | unset | Sampling temperature for consults; omitted when unset. |
+| `transcript_max_chars` | `200000` | Middle-out cap on the serialized transcript (~50k tokens). |
+| `fail_open` | `true` | Advisor failure passes the turn through instead of erroring. |
+| `reviewer_system_prompt` | built-in | Overrides the APPROVE/REDO reviewer contract. |
+| `redo_feedback_prefix` | built-in | Overrides the prefix injected before a REDO plan. |
+
+## Tuning
+
+The defaults gate once per session at the first terminal turn. On agentic
+coding harnesses, the configuration that benchmarked best is `max_reviews = 3`,
+`gate_stall_turns = 30`, `gate_min_tool_results = 3`: skip the early commentary
+turns, keep a mid-task checkpoint for grinders, and allow a re-review after a
+REDO.
+
+Expect the gate's value to depend on the executor. On Terminal-Bench 2.1 with a
+coding agent it lifted a weak executor by 11 points (43.8% → 54.7% ± 0.7,
+k=3) by catching premature "done" claims and stalls, while on a strong executor
+it only matched the accuracy of the takeover-style routers — strong executors
+rarely produce the diligence failures a review can catch. If your executor is
+already frontier-class, benchmark
+[stage-router routing](stage_router_routing.md) first; reach for the advisor
+gate when the executor is markedly weaker than the best model you can call, or
+when you need a review trail rather than a model swap.
diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md
index e5cd4299f..dad214d8e 100644
--- a/docs/routing_algorithms/overview.md
+++ b/docs/routing_algorithms/overview.md
@@ -16,6 +16,7 @@ configuration and tuning. For the vocabulary these pages use, see
| [LLM Classifier Routing](llm_classifier_routing.md) | Request content should decide whether a turn needs the weak or strong tier. | `llm_classifier` |
| [Stage-Router Routing](stage_router_routing.md) | Tool-result and agent-progress signals should route most turns without an extra classifier call. | `stage_router` |
| [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `llm_classifier` with `escalation` |
+| [Advisor-Gate Routing](advisor_gate_routing.md) | One model should serve every turn, with a stronger reviewer approving its "done" claims or sending back a redo plan. | `advisor` |
## Common route shape
diff --git a/mkdocs.yml b/mkdocs.yml
index d081dab58..fcf40f715 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -29,6 +29,7 @@ nav:
- LLM Classifier Routing: routing_algorithms/llm_classifier_routing.md
- Stage-Router Routing: routing_algorithms/stage_router_routing.md
- Escalation-Router Routing: routing_algorithms/escalation_router_routing.md
+ - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md
- Operations:
- Context-Window Handling: operations/context_window.md
- Reference: