Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): add escalation as config knob on LlmTaskClassifier (#211)

Lin Jiac5041dd450e1
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index a4dc02aea..b5cb3834e 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -14,7 +14,7 @@ pub mod rand;
pub mod stage;
pub(crate) use fall_through::{DefaultTarget, FallThrough};
-pub use llm_class::{LlmTaskClassifier, TaskClassifierConfig};
+pub use llm_class::{EscalationJudgeConfig, LlmTaskClassifier, TaskClassifierConfig};
pub use noop::{Noop, NoopDecision};
pub use passthrough::{Passthrough, PassthroughDecision};
pub use rand::{Random, RandomClassifier, RandomDecision};
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 367748f51..64bda5984 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -11,15 +11,18 @@ use std::sync::Arc;
use async_trait::async_trait;
use serde::Deserialize;
-use switchyard_protocol::{LlmRequest, Message, OutputParams, Role};
+use switchyard_protocol::{LlmRequest, Message, OutputParams, Role, SimpleDecision};
+use super::util::escalation::{build_judge, EscalationJudge, EscalationPolicy};
+pub use super::util::EscalationJudgeConfig;
use super::util::{
load_judge_config, AffinityRouter, Judge, JudgeClassifier, JudgeConfig, JudgePolicy,
};
use super::{DefaultTarget, FallThrough};
use crate::{
- Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTarget, LlmTargetSet,
- Request, Response, Result, RoutedLlmClient, Score, State,
+ AggLlmResponse, Algorithm, Classification, Classifier, Context, Driver, LibsyError,
+ LlmResponse, LlmTarget, LlmTargetSet, Request, Response, Result, RoutedLlmClient, Score, State,
+ StateValue,
};
const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md");
@@ -263,10 +266,147 @@ struct TaskClassifier {
capable_target: String,
}
+// ── Escalation classifier ──────────────────────────────────────────────────
+
+/// Session-state key holding the consecutive-escalate streak.
+const STREAK_KEY: &str = "escalation_streak";
+
+fn streak(state: &State) -> u32 {
+ match state.extra.get(STREAK_KEY) {
+ Some(StateValue::Count(n)) => *n,
+ _ => 0,
+ }
+}
+
+fn decisive(target: &str) -> Classification {
+ Classification::Scores(vec![Score {
+ target: target.to_string(),
+ confidence: 1.0,
+ }])
+}
+
+fn assistant_message(response: &AggLlmResponse) -> Message {
+ Message {
+ role: Role::Assistant,
+ content: response
+ .first_output()
+ .map(|output| output.content.clone())
+ .unwrap_or_default(),
+ }
+}
+
+/// Calls the efficient model, judges its response, and latches to capable once the streak
+/// confirms. Returns the efficient response directly when not escalating so the caller does
+/// not pay for a second model call.
+struct EscalationClassifier {
+ judge: JudgeClassifier<EscalationJudge, EscalationPolicy>,
+ capable: LlmTarget,
+ efficient: LlmTarget,
+ /// Consecutive escalate verdicts required to latch.
+ confirmations: u32,
+}
+
+#[async_trait]
+impl Classifier<State> for EscalationClassifier {
+ fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
+ if self.capable.semantic_name == self.efficient.semantic_name {
+ None
+ } else if selected_model == self.capable.semantic_name {
+ Some("strong")
+ } else if selected_model == self.efficient.semantic_name {
+ Some("weak")
+ } else {
+ None
+ }
+ }
+
+ async fn score(
+ &self,
+ state: &mut State,
+ request: &mut Request,
+ driver: Option<&Driver>,
+ ) -> Result<(Classification, Option<Response>)> {
+ let Some(driver) = driver else {
+ return Err(LibsyError::AlgorithmError {
+ message: "escalation classifier requires a driver".into(),
+ });
+ };
+
+ // A confirmed session stays capable without a judge call.
+ if streak(state) >= self.confirmations {
+ return Ok((decisive(&self.capable.semantic_name), None));
+ }
+
+ // Call efficient model and buffer the response so the judge can read it.
+ // `Classifier::score` takes no `ctx`, so inner calls use Context::default() and their
+ // spans carry algorithm="" rather than the algorithm name. Known gap shared with the
+ // task classifier's judge consultation.
+ let efficient_response = driver
+ .call_llm_target(
+ Context::default(),
+ &self.efficient,
+ request.clone(),
+ Arc::new(SimpleDecision {
+ selected_model: self.efficient.semantic_name.clone(),
+ reasoning: Some("escalation classifier: efficient tier".into()),
+ }),
+ )
+ .await?;
+ let agg = efficient_response
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|e| LibsyError::AlgorithmError {
+ message: format!("failed to aggregate efficient response: {e}"),
+ })?;
+ // Append the efficient reply so the judge reads this turn's completed trajectory.
+ let mut judge_request = request.clone();
+ judge_request
+ .llm_request
+ .messages
+ .push(assistant_message(&agg));
+ let efficient_response = Response {
+ llm_response: if request.llm_request.stream {
+ LlmResponse::Stream(agg.into_stream())
+ } else {
+ LlmResponse::Agg(agg)
+ },
+ metadata: efficient_response.metadata,
+ };
+
+ let (classification, _) = self
+ .judge
+ .score(state, &mut judge_request, Some(driver))
+ .await?;
+
+ let held = streak(state);
+ let best = classification.argmax(false)?;
+ let (escalate, pending) = match &best {
+ Some(score) if score.target == self.capable.semantic_name => (true, held + 1),
+ Some(_) => (false, 0),
+ None => (false, held),
+ };
+ state
+ .extra
+ .insert(STREAK_KEY.to_string(), StateValue::Count(pending));
+
+ if escalate && pending >= self.confirmations {
+ // Streak confirmed: drop the efficient response, caller will serve capable.
+ return Ok((decisive(&self.capable.semantic_name), None));
+ }
+
+ Ok((
+ decisive(&self.efficient.semantic_name),
+ Some(efficient_response),
+ ))
+ }
+}
+
/// A task-level capability routing algorithm with an internal fall-through cascade.
pub struct LlmTaskClassifier {
route: FallThrough<State>,
- classifier: Arc<TaskClassifier>,
+ /// The active classifier — either capability-based or escalation-based.
+ inner: Arc<dyn Classifier<State>>,
}
impl LlmTaskClassifier {
@@ -288,16 +428,17 @@ impl LlmTaskClassifier {
config: judge_config,
recent_turn_window: config.recent_turn_window,
},
- judge_target,
+ judge_target.clone(),
TaskClassifierPolicy::new(
efficient_target.semantic_name.clone(),
capable_target.semantic_name.clone(),
config,
),
),
- efficient_target: efficient_target.semantic_name,
- capable_target: capable_target.semantic_name,
+ efficient_target: efficient_target.semantic_name.clone(),
+ capable_target: capable_target.semantic_name.clone(),
});
+ let inner: Arc<dyn Classifier<State>> = classifier.clone();
// Affinity comes first so a retained assignment short-circuits the judge call.
// Note: when this classifier is embedded inside another cascade (e.g. StageRouter)
@@ -320,9 +461,41 @@ impl LlmTaskClassifier {
let capable_fallback = DefaultTarget::new(classifier.capable_target.clone());
Ok(Self {
route: route
- .with_classifier(classifier.clone())
+ .with_classifier(inner.clone())
.with_classifier(Arc::new(capable_fallback)),
- classifier,
+ inner,
+ })
+ }
+
+ /// Constructs an escalation variant that calls the efficient model each turn, judges its
+ /// response, and latches to the capable tier once the streak confirms.
+ ///
+ /// Every unlatched turn calls the efficient model, buffers its reply, and consults the
+ /// trajectory judge. Once `config.confirmations` consecutive escalate verdicts accumulate
+ /// the session latches to the capable tier for its remainder. A judge outage always stays
+ /// efficient.
+ pub fn new_with_escalation(
+ judge_target: LlmTarget,
+ efficient_target: LlmTarget,
+ capable_target: LlmTarget,
+ config: EscalationJudgeConfig,
+ ) -> Result<Self> {
+ let capable_name = capable_target.semantic_name.clone();
+ let efficient_name = efficient_target.semantic_name.clone();
+ let confirmations = config.confirmations;
+ let esc = Arc::new(EscalationClassifier {
+ judge: build_judge(judge_target, capable_name, efficient_name, config)?,
+ capable: capable_target.clone(),
+ efficient: efficient_target.clone(),
+ confirmations,
+ });
+ let inner: Arc<dyn Classifier<State>> = esc.clone();
+ let targets = LlmTargetSet::new(vec![capable_target, efficient_target]);
+ Ok(Self {
+ route: FallThrough::<State>::new_with_state(targets)
+ .with_name(ALGORITHM_NAME)
+ .with_classifier(esc),
+ inner,
})
}
@@ -359,7 +532,7 @@ impl Classifier<State> for TaskClassifier {
#[async_trait]
impl Classifier<State> for LlmTaskClassifier {
fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
- self.classifier.routing_tier(selected_model)
+ self.inner.routing_tier(selected_model)
}
async fn score(
@@ -368,7 +541,7 @@ impl Classifier<State> for LlmTaskClassifier {
request: &mut Request,
driver: Option<&Driver>,
) -> Result<(Classification, Option<Response>)> {
- self.classifier.score(state, request, driver).await
+ self.inner.score(state, request, driver).await
}
}
@@ -947,4 +1120,125 @@ mod tests {
.any(|value| value.as_str() == Some("none")));
Ok(())
}
+
+ // ── with_escalation tests ──────────────────────────────────────────────
+
+ use std::collections::VecDeque;
+
+ use switchyard_protocol::LlmClientError as ClientError;
+
+ use crate::Decision;
+
+ /// Serves each call with the next queued reply.
+ struct QueuedClient {
+ replies: Mutex<VecDeque<String>>,
+ }
+
+ impl QueuedClient {
+ fn new(replies: impl IntoIterator<Item = &'static str>) -> Arc<Self> {
+ Arc::new(Self {
+ replies: Mutex::new(replies.into_iter().map(String::from).collect()),
+ })
+ }
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for QueuedClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ _decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, ClientError> {
+ let reply = self
+ .replies
+ .lock()
+ .pop_front()
+ .unwrap_or_else(|| "unexpected call".to_string());
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, reply)),
+ metadata: request.metadata,
+ })
+ }
+ }
+
+ /// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict).
+ fn escalation_router(
+ client: Arc<dyn RoutedLlmClient>,
+ judge_client: Arc<dyn RoutedLlmClient>,
+ ) -> Result<Arc<LlmTaskClassifier>> {
+ let target = |name: &str, c: Arc<dyn RoutedLlmClient>| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: Some(c),
+ };
+ Ok(Arc::new(LlmTaskClassifier::new_with_escalation(
+ target("judge", judge_client),
+ target("efficient", client.clone()),
+ target("capable", client),
+ EscalationJudgeConfig {
+ confirmations: 1,
+ ..EscalationJudgeConfig::default()
+ },
+ )?))
+ }
+
+ #[tokio::test]
+ async fn escalation_router_serves_efficient_when_judge_declines() -> Result<()> {
+ // Judge: no escalation. Expect the efficient response to be returned directly.
+ let judge_client = QueuedClient::new([r#"{"escalate":false,"reason":"progressing"}"#]);
+ let model_client = QueuedClient::new(["efficient answer"]);
+ let router = escalation_router(model_client, judge_client)?;
+ let request = classify_request();
+
+ let (trace, response) = router.run(Context::default(), request).await?;
+
+ // The efficient model is the serving target, and the response comes from its call.
+ assert_eq!(trace.last().map(|d| d.selected_model()), Some("efficient"));
+ assert_eq!(
+ response.llm_response.as_agg().map(completion_text),
+ Some("efficient answer".to_string())
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn escalation_router_upgrades_to_capable_when_judge_escalates() -> Result<()> {
+ // Judge: escalate. After the efficient call, the streak confirms and capable is served.
+ let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck in a loop"}"#]);
+ // Efficient is called first (by the classifier), then capable is called by FallThrough.
+ let model_client = QueuedClient::new(["efficient draft", "capable answer"]);
+ let router = escalation_router(model_client, judge_client)?;
+ let request = classify_request();
+
+ let (trace, response) = router.run(Context::default(), request).await?;
+
+ assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ assert_eq!(
+ response.llm_response.as_agg().map(completion_text),
+ Some("capable answer".to_string())
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn escalation_router_stays_capable_after_latch() -> Result<()> {
+ // First turn: judge escalates and the streak latches.
+ // Second turn: judge is not called again; capable is served directly.
+ let judge_client = QueuedClient::new([r#"{"escalate":true,"reason":"stuck"}"#]);
+ let model_client = QueuedClient::new(["efficient draft", "capable t1", "capable t2"]);
+ let router = escalation_router(model_client, judge_client)?;
+
+ let session_request = classify_session_request();
+ router
+ .clone()
+ .run(Context::default(), session_request.clone())
+ .await?;
+ let (trace, _) = router
+ .clone()
+ .run(Context::default(), session_request)
+ .await?;
+
+ assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
+ Ok(())
+ }
}
diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs
index e21dff5a3..131b72621 100644
--- a/crates/libsy/src/algorithms/util.rs
+++ b/crates/libsy/src/algorithms/util.rs
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
mod affinity;
+pub mod escalation;
mod llm_judge;
pub(crate) mod prompts;
pub(crate) mod stage;
@@ -9,6 +10,7 @@ mod subagent;
pub(crate) mod tool_signals;
pub use affinity::AffinityRouter;
+pub use escalation::EscalationJudgeConfig;
pub(crate) use llm_judge::{load_judge_config, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};
pub use prompts::{append_note, SystemPromptProcessor, TargetPrompts};
pub use subagent::SubagentOverride;
diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs
new file mode 100644
index 000000000..f71a8054c
--- /dev/null
+++ b/crates/libsy/src/algorithms/util/escalation.rs
@@ -0,0 +1,514 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Trajectory-judge components for the escalation router — the judge, its verdict policy, and
+//! the transcript condenser they read.
+//!
+//! [`build_judge`] is the whole surface; the confirmation policy that consumes its verdicts
+//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
+
+use serde::Deserialize;
+use switchyard_protocol::{ContentBlock, LlmRequest, Message, OutputParams, Role};
+
+use super::{load_judge_config, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};
+use crate::{Classification, LlmTarget, Request, Result, Score, State};
+
+const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md");
+const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json");
+
+/// Separator marking where [`truncate_middle`] dropped a message's interior.
+const TRIM_MARKER: &str = " ...[trimmed] ";
+
+/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`].
+const TRUNCATION_SUFFIX: &str = "...<truncated>";
+
+/// Per-message cap for system and developer anchors, which carry no trajectory signal but
+/// which coding-agent harnesses make very large.
+const SYSTEM_CHARS: usize = 1_000;
+
+/// Cap for the first user message — the task statement, so it gets the widest anchor budget.
+const FIRST_USER_CHARS: usize = 2_000;
+
+/// Backstop on the assembled transcript; the per-message caps normally bind first.
+const MAX_REQUEST_CHARS: usize = 18_000;
+
+/// Completion budget for one judge reply. A runaway guard, not a budget: the benchmarked
+/// verdict runs ~40 tokens, and a cap costs nothing it does not generate.
+///
+/// Headroom this wide is for a judge deliberately opted back into thinking — hosts ask judge
+/// targets to answer without thinking first, but a judge that thinks and is cut off before
+/// answering has no verdict in `content` and fails open on every turn.
+const JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096;
+
+/// The tuning surface for the trajectory judge.
+///
+/// Deliberately small: the three settings an audit of the Python router's twenty-odd knobs
+/// found to actually change routing outcomes. Everything else is a fixed invariant (the
+/// constants above). Defaults are the benchmarked configuration.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(default)]
+pub struct EscalationJudgeConfig {
+ /// Consecutive escalate verdicts required before a turn moves to the capable tier, which
+ /// is also the turn that latches the session. Any decline clears the streak.
+ /// `1` escalates on the first verdict; the router's main cost dial.
+ /// `2` or higher needs a session id, since the streak is retained per session.
+ pub confirmations: u32,
+ /// Trailing messages shown on top of the anchors. A loop longer than this is invisible.
+ pub recent_turn_window: usize,
+ /// Per-message cap inside the trailing window.
+ pub window_message_chars: usize,
+}
+
+impl EscalationJudgeConfig {
+ /// Rejects settings that would leave the judge with nothing useful to read.
+ fn validate(&self) -> Result<()> {
+ let reject = |message: String| Err(crate::LibsyError::AlgorithmError { message });
+ if self.confirmations == 0 {
+ return reject("confirmations must be at least 1".to_string());
+ }
+ if self.recent_turn_window == 0 {
+ return reject("recent_turn_window must be at least 1".to_string());
+ }
+ if self.window_message_chars < 50 {
+ return reject(format!(
+ "window_message_chars must be at least 50, got {}",
+ self.window_message_chars
+ ));
+ }
+ Ok(())
+ }
+}
+
+impl Default for EscalationJudgeConfig {
+ fn default() -> Self {
+ Self {
+ confirmations: 2,
+ recent_turn_window: 28,
+ window_message_chars: 500,
+ }
+ }
+}
+
+/// The judge's verdict. The schema also requires a `reason`, which makes the judge state its
+/// case and measurably sharpens the verdict — routing reads only the boolean, so it is
+/// deserialized away rather than carried.
+#[derive(Deserialize)]
+pub(crate) struct EscalationVerdict {
+ escalate: bool,
+}
+
+/// Builds the judge request from the conversation so far.
+pub(crate) struct EscalationJudge {
+ rubric: JudgeConfig,
+ config: EscalationJudgeConfig,
+}
+
+impl Judge for EscalationJudge {
+ type Verdict = EscalationVerdict;
+
+ /// Renders the rubric as the system message and the condensed trajectory as the user
+ /// message. `state` is unused: the judge reads only the live request.
+ fn build_request(&self, _state: &State, request: &Request) -> Request {
+ let messages = &request.llm_request.messages;
+ let summary = summarize_for_judge(messages, conversation_turn(request), &self.config);
+ Request {
+ llm_request: LlmRequest {
+ model: request.llm_request.model.clone(),
+ messages: vec![
+ Message::text(Role::System, self.rubric.system_prompt.clone()),
+ Message::text(Role::User, summary),
+ ],
+ output: OutputParams {
+ response_format: self.rubric.response_schema.clone(),
+ max_output_tokens: Some(JUDGE_MAX_OUTPUT_TOKENS),
+ },
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: request.metadata.clone(),
+ }
+ }
+}
+
+/// Maps the judge's verdict to a classification. A verdict names the tier to serve — capable
+/// on escalate, efficient on decline — so the caller reads it straight off the winning score.
+/// [`Classification::Ambiguous`] carries the unavailable case, which names no tier: both a
+/// decline and an outage stay efficient, but only a decline is evidence, so only a decline
+/// clears the streak.
+pub(crate) struct EscalationPolicy {
+ capable: String,
+ efficient: String,
+}
+
+impl JudgePolicy for EscalationPolicy {
+ type Verdict = EscalationVerdict;
+
+ fn to_classification(&self, verdict: Option<&EscalationVerdict>) -> Classification {
+ match verdict {
+ Some(verdict) if verdict.escalate => Classification::Scores(vec![Score {
+ target: self.capable.clone(),
+ confidence: 1.0,
+ }]),
+ Some(_) => Classification::Scores(vec![Score {
+ target: self.efficient.clone(),
+ confidence: 1.0,
+ }]),
+ None => Classification::Ambiguous(Vec::new()),
+ }
+ }
+}
+
+/// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates.
+///
+/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
+/// fails here rather than on the first request.
+pub(crate) fn build_judge(
+ judge_target: LlmTarget,
+ capable: String,
+ efficient: String,
+ config: EscalationJudgeConfig,
+) -> Result<JudgeClassifier<EscalationJudge, EscalationPolicy>> {
+ config.validate()?;
+ let rubric = load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?;
+ Ok(JudgeClassifier::new(
+ EscalationJudge { rubric, config },
+ judge_target,
+ EscalationPolicy { capable, efficient },
+ ))
+}
+
+/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
+///
+/// The judge reads the turn *including* the reply it is judging, so the newest assistant
+/// message is this turn — no `+ 1`. Counting the caller's request instead would report the
+/// turn after the one under judgement.
+///
+/// libsy receives messages already normalized by `switchyard-protocol`, so unlike the
+/// wire-format-aware Python equivalent this needs no per-format branching.
+pub(crate) fn conversation_turn(request: &Request) -> usize {
+ request
+ .llm_request
+ .messages
+ .iter()
+ .filter(|message| message.role == Role::Assistant)
+ .count()
+}
+
+/// Flattens a message to plain text, tool calls and tool results included.
+///
+/// [`Message::text_content`] is deliberately not used here: it keeps only text and refusal
+/// blocks, which would erase exactly the repeated-command signal the judge's loop detection
+/// relies on.
+fn message_text(message: &Message) -> String {
+ let mut parts = Vec::new();
+ collect_text(&message.content, &mut parts);
+ parts.join(" ")
+}
+
+/// Appends the judge-relevant text of each block, descending into tool results.
+fn collect_text(content: &[ContentBlock], parts: &mut Vec<String>) {
+ for block in content {
+ match block {
+ ContentBlock::Text { text } | ContentBlock::Refusal { text } => {
+ parts.push(text.clone());
+ }
+ ContentBlock::ToolCall(call) => {
+ parts.push(format!("tool_call {}({})", call.name, call.arguments));
+ }
+ ContentBlock::ToolResult(result) => collect_text(&result.content, parts),
+ _ => {}
+ }
+ }
+}
+
+/// Keeps the head and tail of `text` within `limit` characters.
+///
+/// The head gets two thirds of the surviving budget: for a trajectory judge the command or
+/// error signature that opens a message carries more signal than its trailing output.
+fn truncate_middle(text: &str, limit: usize) -> String {
+ let chars: Vec<char> = text.chars().collect();
+ if chars.len() <= limit {
+ return text.to_string();
+ }
+ let keep = limit
+ .saturating_sub(TRIM_MARKER.chars().count())
+ .max(20)
+ .min(chars.len());
+ let head = keep * 2 / 3;
+ let tail = keep - head;
+ let mut out: String = chars[..head].iter().collect();
+ out.push_str(TRIM_MARKER);
+ out.extend(chars[chars.len() - tail..].iter());
+ out
+}
+
+/// Renders a compact role-labelled transcript for the judge.
+///
+/// The framing anchors — system/developer messages and the first user message, where agent
+/// harnesses put the task statement — are kept unconditionally and capped individually. The
+/// trailing window carries recent activity. A coverage header states how much history is not
+/// shown, so the judge can reason about pace rather than assuming it sees everything.
+///
+/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go
+/// first: for a trajectory judge the newest evidence is strictly the most valuable.
+fn summarize_for_judge(
+ messages: &[Message],
+ turn: usize,
+ config: &EscalationJudgeConfig,
+) -> String {
+ let mut anchors: Vec<String> = Vec::new();
+ let mut window: Vec<String> = Vec::new();
+ let mut first_user_seen = false;
+
+ for message in messages {
+ let text = message_text(message);
+ match message.role {
+ Role::System | Role::Developer => anchors.push(format!(
+ "[{}] {}",
+ role_label(message.role),
+ truncate_middle(&text, SYSTEM_CHARS)
+ )),
+ Role::User if !first_user_seen => {
+ first_user_seen = true;
+ anchors.push(format!(
+ "[user (task)] {}",
+ truncate_middle(&text, FIRST_USER_CHARS)
+ ));
+ }
+ role => window.push(format!(
+ "[{}] {}",
+ role_label(role),
+ truncate_middle(&text, config.window_message_chars)
+ )),
+ }
+ }
+
+ if window.len() > config.recent_turn_window {
+ window.drain(..window.len() - config.recent_turn_window);
+ }
+
+ let assemble = |window: &[String]| {
+ let header = format!(
+ "Conversation turn {turn}; showing the last {} of {} messages after the task framing.",
+ window.len(),
+ messages.len(),
+ );
+ std::iter::once(header)
+ .chain(anchors.iter().cloned())
+ .chain(window.iter().cloned())
+ .collect::<Vec<_>>()
+ .join("\n")
+ };
+
+ let mut text = assemble(&window);
+ while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() {
+ window.remove(0);
+ text = assemble(&window);
+ }
+ if text.chars().count() > MAX_REQUEST_CHARS {
+ let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1);
+ text = text.chars().take(keep).collect::<String>() + TRUNCATION_SUFFIX;
+ }
+ text
+}
+
+/// The transcript label for a role.
+fn role_label(role: Role) -> &'static str {
+ match role {
+ Role::System => "system",
+ Role::Developer => "developer",
+ Role::User => "user",
+ Role::Assistant => "assistant",
+ Role::Tool => "tool",
+ }
+}
+
+/// A request whose conversation sits at `turn`: `turn - 1` prior assistant replies, each
+/// answered by a further user message.
+///
+/// Shared with the assembled router's tests, which drive the same conversation shape.
+#[cfg(test)]
+pub(crate) fn request_at_turn(session_id: Option<&str>, turn: usize) -> Request {
+ use switchyard_protocol::Metadata;
+
+ let mut messages = vec![Message::text(Role::User, "What is 2+2?")];
+ for attempt in 1..turn {
+ messages.push(Message::text(Role::Assistant, format!("attempt {attempt}")));
+ messages.push(Message::text(Role::User, format!("still wrong {attempt}")));
+ }
+ Request {
+ llm_request: LlmRequest {
+ model: Some("auto".to_string()),
+ messages,
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: session_id.map(|id| Metadata {
+ session_id: Some(id.to_string()),
+ ..Metadata::default()
+ }),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use serde_json::json;
+ use switchyard_protocol::{ContentBlock, Message, Role, ToolCall, ToolResult};
+
+ use super::*;
+
+ #[test]
+ fn judge_request_is_rubric_plus_summary_under_a_completion_cap() -> Result<()> {
+ let judge = EscalationJudge {
+ rubric: load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)?,
+ config: EscalationJudgeConfig::default(),
+ };
+
+ // As the classifier calls it: the turn's reply is already on the transcript.
+ let mut judged = request_at_turn(None, 4);
+ judged
+ .llm_request
+ .messages
+ .push(Message::text(Role::Assistant, "this turn's reply"));
+ let built = judge.build_request(&State::default(), &judged);
+
+ // Two messages: the rubric as system, the condensed trajectory as user.
+ assert_eq!(built.llm_request.messages.len(), 2);
+ assert_eq!(built.llm_request.messages[0].role, Role::System);
+ assert_eq!(built.llm_request.messages[1].role, Role::User);
+ assert!(built.llm_request.messages[1]
+ .text_content("")
+ .is_some_and(|text| text.contains("Conversation turn 4")));
+ // Bounded output, so a reasoning judge cannot run away mid-verdict.
+ assert_eq!(
+ built.llm_request.output.max_output_tokens,
+ Some(JUDGE_MAX_OUTPUT_TOKENS)
+ );
+ assert!(built.llm_request.output.response_format.is_some());
+ Ok(())
+ }
+
+ #[test]
+ fn conversation_turn_counts_assistant_replies() {
+ // The judge is handed the transcript with this turn's reply already appended, which
+ // is the shape asserted here: a request entering turn N plus its reply *is* turn N.
+ for turn in [1, 5] {
+ let mut judged = request_at_turn(None, turn);
+ judged
+ .llm_request
+ .messages
+ .push(Message::text(Role::Assistant, "this turn's reply"));
+ assert_eq!(conversation_turn(&judged), turn);
+ }
+ }
+
+ #[test]
+ fn message_text_keeps_tool_calls_and_results() {
+ let call = Message {
+ role: Role::Assistant,
+ content: vec![
+ ContentBlock::Text {
+ text: "running it".to_string(),
+ },
+ ContentBlock::ToolCall(ToolCall {
+ id: "call-1".to_string(),
+ name: "bash".to_string(),
+ arguments: json!({"cmd": "ls"}),
+ }),
+ ],
+ };
+ let text = message_text(&call);
+ assert!(text.contains("running it"), "{text}");
+ assert!(text.contains(r#"tool_call bash({"cmd":"ls"})"#), "{text}");
+
+ let result = Message {
+ role: Role::Tool,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "call-1".to_string(),
+ content: vec![ContentBlock::Text {
+ text: "no such file".to_string(),
+ }],
+ is_error: Some(true),
+ })],
+ };
+ assert_eq!(message_text(&result), "no such file");
+ }
+
+ #[test]
+ fn truncate_middle_keeps_head_and_tail() {
+ let text = "a".repeat(40) + &"z".repeat(40);
+ let trimmed = truncate_middle(&text, 50);
+ assert!(trimmed.chars().count() <= 50, "{trimmed}");
+ assert!(trimmed.starts_with('a'));
+ assert!(trimmed.ends_with('z'));
+ assert!(trimmed.contains("[trimmed]"));
+
+ // Under the limit the text is returned untouched.
+ assert_eq!(truncate_middle("short", 50), "short");
+ }
+
+ #[test]
+ fn summary_keeps_anchors_and_the_recent_window() {
+ let mut messages = vec![
+ Message::text(Role::System, "you are a coding agent"),
+ Message::text(Role::User, "fix the failing test"),
+ ];
+ for i in 0..10 {
+ messages.push(Message::text(Role::Assistant, format!("step {i}")));
+ }
+ let config = EscalationJudgeConfig {
+ recent_turn_window: 3,
+ ..EscalationJudgeConfig::default()
+ };
+
+ let summary = summarize_for_judge(&messages, 11, &config);
+
+ assert!(
+ summary.contains("[system] you are a coding agent"),
+ "{summary}"
+ );
+ assert!(
+ summary.contains("[user (task)] fix the failing test"),
+ "{summary}"
+ );
+ assert!(summary.contains("Conversation turn 11; showing the last 3 of 12 messages"));
+ // Only the newest window entries survive.
+ assert!(summary.contains("step 9"), "{summary}");
+ assert!(summary.contains("step 7"), "{summary}");
+ assert!(!summary.contains("step 6"), "{summary}");
+ }
+
+ #[test]
+ fn summary_drops_oldest_window_lines_under_the_char_cap() {
+ // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps
+ // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually
+ // wide per-message cap. That is the point — it only fires on pathological input.
+ let mut messages = vec![
+ Message::text(Role::System, "framing"),
+ Message::text(Role::User, "task"),
+ ];
+ for i in 0..20 {
+ messages.push(Message::text(
+ Role::Assistant,
+ format!("{i} {}", "x".repeat(2_000)),
+ ));
+ }
+ let config = EscalationJudgeConfig {
+ window_message_chars: 2_000,
+ ..EscalationJudgeConfig::default()
+ };
+
+ let summary = summarize_for_judge(&messages, 21, &config);
+
+ assert!(
+ summary.chars().count() <= MAX_REQUEST_CHARS,
+ "{}",
+ summary.chars().count()
+ );
+ // Anchors are never dropped, and the newest activity outlives the oldest.
+ assert!(summary.contains("[system] framing"), "{summary}");
+ assert!(summary.contains("[user (task)] task"), "{summary}");
+ assert!(summary.contains("19 xxx"), "{summary}");
+ assert!(!summary.contains("0 xxx"), "{summary}");
+ }
+}
diff --git a/crates/libsy/src/prompts/escalation/prompt.md b/crates/libsy/src/prompts/escalation/prompt.md
new file mode 100644
index 000000000..6d57369bb
--- /dev/null
+++ b/crates/libsy/src/prompts/escalation/prompt.md
@@ -0,0 +1,178 @@
+You are an escalation judge inside an agentic coding router. The session
+started on the EFFICIENT tier (a cheap but top-class 2026 model). Your
+job is to detect when the run is genuinely in trouble so the router can
+escalate the rest of the task to the STRONG tier (frontier, expensive).
+
+You see a condensed view of one session: the task framing (system prompt
++ first user message) and the most recent turns of activity (assistant
+messages and tool results). Judge the *trajectory* — is the agent making
+real progress toward the stated task — not the difficulty of the task
+itself. Return exactly one JSON object:
+
+{"escalate": boolean, "reason": "one short sentence naming the pattern"}
+
+Escalation is one-way for the rest of the task and expensive. Escalate
+only on a clear PATTERN of trouble, never on a single failed command.
+When the evidence is thin or ambiguous, return {"escalate": false}.
+
+The bar is not "is there friction" — agentic coding is full of friction
+the efficient tier works through on its own. The bar is "is this run
+likely DOOMED without intervention": the agent is stuck in place and
+its recent behavior shows no mechanism by which the next few turns
+would look different.
+
+# Is the stuck point beyond the efficient tier?
+
+Escalation pays only when the trouble is the KIND the strong tier is
+better at. The efficient tier handles routine coding, file
+exploration, single-file edits, normal debugging, dependency and
+environment setup, and most refactors on its own — being stuck on
+those is usually temporary. Weigh the kind of stuck point:
+
+Escalate sooner — the stuck point exceeds the efficient tier's
+capability, and a stronger model would likely break the loop:
+- Cross-module or cross-codebase synthesis: the fix requires learning
+ a convention, contract, or invariant from elsewhere in the codebase
+ and applying it consistently — even when the edit itself is
+ single-file.
+- Subtle invariants: plausible-looking fixes keep failing the same
+ test because the root cause hinges on a behavior contract none of
+ the attempts have touched.
+- Root causes genuinely spanning modules, or multi-step algorithmic /
+ formal reasoning the agent keeps getting almost-right.
+
+Hold weak — the efficient tier resolves these with iteration:
+- Procedural or mechanical friction: tool availability, installs,
+ service startup, recipe-following scaffolding, localized one-file
+ test fixes.
+
+Hold weak — no model can fix these, so escalation is pure waste:
+- The blocker is external: required data or files that simply do not
+ exist in the environment, a permanently broken or missing service,
+ or requirements the environment contradicts. A stronger model
+ changes nothing about a missing resource. One boundary to respect:
+ when producing, recovering, or decoding that very artifact IS the
+ stated task, its absence is the work itself, not a blocker — judge
+ the trajectory on it like any other work.
+
+# Trouble patterns — escalate when you see these
+
+Repetition and loops (the most common way agent runs die):
+- The same command or edit failing 2+ times with materially the same
+ error, especially with unrelated changes in between.
+- Near-identical tool calls repeated, or the same files re-read, without
+ new information gained — including longer cycles (A -> B -> C -> A).
+- Fighting the environment: repeatedly invoking a missing executable,
+ retrying installs that fail the same way, or trying variations of a
+ command the environment has already rejected, instead of adapting.
+
+False progress (looks like progress, is not):
+- Declaring success or moving on while the latest visible evidence
+ (test output, exit code, error text) shows failure.
+- Finishing without running the verification the task specifies, when
+ the task states how success is checked (e.g. "make the provided
+ tests pass") and running it was possible.
+- A reproduction or test the agent wrote that passes trivially without
+ exercising the actual issue, then building on that false signal.
+- The agent's stated reading of a tool result contradicts what the
+ result actually says (treating an error or empty output as success).
+
+Drift and dead ends:
+- Recent activity no longer serves the task in the first user message
+ (e.g. polishing style while the required feature is unstarted).
+ A debugging detour that plausibly unblocks the task — fixing the
+ environment, starting a required service, investigating an error in
+ a dependency — is NOT drift; call drift only when the detour has
+ produced nothing useful for many turns AND the task's real
+ verification remains untouched.
+- Violating an explicit task constraint (modifying files the task says
+ not to touch, changing the tests instead of the code under test).
+- Editing or reasoning about code without ever having opened the files
+ the errors point to — acting on guessed file contents.
+- Contradicting or re-deriving something already established earlier in
+ the session (forgetting its own findings).
+- Many turns elapsed with nothing durable produced (no successful
+ writes, no passing checks) and no visible narrowing of the problem —
+ the run is on pace to exhaust its turn budget.
+
+Desperation:
+- Giving up: declaring the task impossible, asking to stop, or drifting
+ into restating the problem instead of acting on it.
+- Destructive flailing: rm -rf, wholesale reinstalls, chmod -R, or
+ reverting everything as a reaction to being stuck rather than a
+ reasoned step.
+
+# Expected friction — do NOT escalate on these
+
+Agentic coding is full of failures that are part of healthy work:
+- A test written to fail first (TDD) or a bug being reproduced on
+ purpose.
+- A compile, lint, or test error fixed or meaningfully acted on in the
+ immediately following turn.
+- Exploration dead-ends early in a session (grep with no matches,
+ reading a file that turns out to be irrelevant) while the agent is
+ still orienting.
+- A missing tool handled adaptively (tries `rg`, falls back to `grep`).
+- Sequential alternatives: trying a DIFFERENT library, tool, or
+ approach after one fails is adaptation, not a loop — even when
+ several alternatives fail in a row. The loop pattern requires the
+ SAME approach retried without material change.
+- A service that is unreachable or not yet running (server not
+ started, port closed, connection refused) while the agent is still
+ actively working to start, configure, or replace it.
+- Planning activity: todo-list and plan updates (TodoWrite,
+ update_plan) are routine agent workflow — planning is neither drift
+ nor struggle, and harness-injected skill/instruction reading early
+ in a session is orientation, not off-task work.
+- Zero-count summaries: "0 failed", "0 errors", "0 warnings" are
+ CLEAN results. Read failure keywords together with their counts —
+ only a nonzero count is a failure.
+- A long-running command (build, install, test suite) that simply has
+ not finished, or the agent waiting on information it asked for.
+
+The distinguishing question: is each failure producing new information
+that changes the next action? Failing forward is fine; failing in place
+is trouble. Also weigh the session's own recovery record: if this same
+session already shows friction the agent subsequently cleared (a failure
+followed by a verified fix or passing check), lean toward holding — a
+session that has recovered before will usually recover again.
+
+# Worked examples (none drawn from any benchmark task set)
+
+* Turn 3; the agent ran the test suite, 4 tests fail, and it is now
+ reading the first failing test. -> {"escalate": false} — reproducing
+ failures is the job.
+* The agent has run `pytest tests/test_api.py` 4 times with the same
+ ImportError, editing an unrelated config file between attempts. ->
+ {"escalate": true, "reason": "same ImportError 4 times while editing
+ unrelated files"}
+* `conda` is not installed; the agent has tried `conda install` five
+ ways instead of using the `pip` that earlier output showed present.
+ -> {"escalate": true, "reason": "fighting missing executable instead
+ of adapting"}
+* Task: "make the provided integration tests pass." Recent turns:
+ renaming variables and reformatting docstrings; tests not run in 8
+ turns. -> {"escalate": true, "reason": "drifted to cosmetic edits,
+ verification abandoned"}
+* The agent says "All tests pass, task complete" but the last visible
+ test output shows "2 failed, 11 passed". -> {"escalate": true,
+ "reason": "claims success contradicted by latest test output"}
+* The agent wrote a reproduction script that exits 0 without invoking
+ the code path the issue describes, concluded "bug not reproducible",
+ and is wrapping up. -> {"escalate": true, "reason": "reproduction
+ never exercised the reported code path"}
+* Two turns of edits, one failed build, then a fixed build and a
+ passing test. -> {"escalate": false}
+* `npm install` has been running for one turn with no output yet. ->
+ {"escalate": false} — slow command, not a stall.
+* Four different serialization libraries failed to import; the agent
+ is now writing the converter with a fifth approach it has not tried
+ before. -> {"escalate": false} — sequential alternatives are
+ adaptation, even when none has succeeded yet.
+* Task: tune a slow batch pipeline. The agent is investigating why
+ the message broker fails to start, since the pipeline cannot be
+ measured without it. -> {"escalate": false} — unblocking
+ verification serves the task.
+
+Do not emit markdown, commentary, or chain-of-thought — only the JSON
+object.
diff --git a/crates/libsy/src/prompts/escalation/schema.json b/crates/libsy/src/prompts/escalation/schema.json
new file mode 100644
index 000000000..977c41882
--- /dev/null
+++ b/crates/libsy/src/prompts/escalation/schema.json
@@ -0,0 +1,22 @@
+{
+ "type": "json_schema",
+ "json_schema": {
+ "name": "EscalationVerdict",
+ "strict": true,
+ "schema": {
+ "type": "object",
+ "properties": {
+ "escalate": {
+ "type": "boolean",
+ "description": "True when the run is likely doomed without escalation to the strong tier."
+ },
+ "reason": {
+ "type": "string",
+ "description": "One short sentence naming the trouble pattern, or stating why the run is progressing."
+ }
+ },
+ "required": ["escalate", "reason"],
+ "additionalProperties": false
+ }
+ }
+}
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 7130397d8..e496c07a3 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -128,6 +128,27 @@ pub trait Decision: Send + Sync {
fn as_any(&self) -> &dyn std::any::Any;
}
+/// A minimal [`Decision`] implementation for one-off calls that don't belong to a
+/// named algorithm step — judge side calls, classifier side calls, etc.
+pub struct SimpleDecision {
+ pub selected_model: String,
+ pub reasoning: Option<String>,
+}
+
+impl Decision for SimpleDecision {
+ fn selected_model(&self) -> &str {
+ &self.selected_model
+ }
+
+ fn reasoning(&self) -> Option<&str> {
+ self.reasoning.as_deref()
+ }
+
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+}
+
/// Performs the actual model call for a target. This is the one piece of I/O the
/// library does not own — a host implements it over its own transport (HTTP SDK,
/// in-process model, mock). It serves a call the stream consumer chose not to
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs
index 0db72066a..4e7eb10d3 100644
--- a/crates/protocol/src/stream.rs
+++ b/crates/protocol/src/stream.rs
@@ -88,6 +88,62 @@ impl LlmResponse {
}
}
+impl AggLlmResponse {
+ /// Converts a fully-buffered response into a synthetic chunk stream.
+ ///
+ /// Useful when a caller had to buffer the response (e.g. to judge it) but the
+ /// downstream expects `LlmResponse::Stream` — for instance when `stream: true` was
+ /// requested and the algorithm had to aggregate before it could return.
+ pub fn into_stream(self) -> LlmResponseStream {
+ let mut chunks: Vec<LlmResponseChunk> = Vec::new();
+ chunks.push(LlmResponseChunk::MessageStart {
+ id: self.id,
+ model: self.model,
+ });
+ let mut tool_call_index = 0usize;
+ for (output_index, output) in self.outputs.into_iter().enumerate() {
+ for block in output.content {
+ match block {
+ ContentBlock::Text { text } => {
+ chunks.push(LlmResponseChunk::TextDelta {
+ index: output_index,
+ text,
+ });
+ }
+ ContentBlock::Reasoning { text, .. } => {
+ chunks.push(LlmResponseChunk::ReasoningDelta {
+ index: output_index,
+ text,
+ });
+ }
+ ContentBlock::ToolCall(tool) => {
+ let args = serde_json::to_string(&tool.arguments).unwrap_or_default();
+ chunks.push(LlmResponseChunk::ToolCallDelta {
+ index: tool_call_index,
+ id: Some(tool.id),
+ name: Some(tool.name),
+ arguments_delta: Some(args),
+ });
+ tool_call_index += 1;
+ }
+ // Other variants (Image, ToolResult, Refusal, etc.) don't have
+ // a streaming chunk representation and don't appear in assistant outputs.
+ _ => {}
+ }
+ }
+ chunks.push(LlmResponseChunk::MessageStop {
+ reason: output.stop_reason.and_then(|r| {
+ serde_json::to_value(r)
+ .ok()
+ .and_then(|v| v.as_str().map(String::from))
+ }),
+ });
+ }
+ chunks.push(LlmResponseChunk::Usage(self.usage));
+ Box::pin(futures::stream::iter(chunks.into_iter().map(Ok)))
+ }
+}
+
/// One provider-neutral streaming event — the normalized counterpart to
/// [`AggLlmResponse`](crate::AggLlmResponse), sitting between stream decoders and
/// encoders. `switchyard-translation` re-exports it as `ConversationStreamEvent`.
@@ -362,6 +418,36 @@ mod tests {
);
}
+ #[test]
+ fn into_stream_round_trips_through_into_agg() {
+ let original = AggLlmResponse {
+ id: Some("id1".to_string()),
+ model: Some("m".to_string()),
+ outputs: vec![ResponseOutput {
+ role: Role::Assistant,
+ content: vec![ContentBlock::Text {
+ text: "hello".to_string(),
+ }],
+ stop_reason: Some(StopReason::EndTurn),
+ }],
+ usage: Usage {
+ output_tokens: Some(3),
+ ..Usage::default()
+ },
+ ..AggLlmResponse::default()
+ };
+ let stream = LlmResponse::Stream(original.clone().into_stream());
+ let recovered = block_on(stream.into_agg()).expect("into_agg failed");
+ assert_eq!(recovered.id, original.id);
+ assert_eq!(recovered.model, original.model);
+ assert_eq!(recovered.usage.output_tokens, original.usage.output_tokens);
+ assert_eq!(
+ recovered.outputs[0].stop_reason,
+ original.outputs[0].stop_reason
+ );
+ assert_eq!(recovered.outputs[0].content, original.outputs[0].content);
+ }
+
#[test]
fn into_agg_preserves_stream_item_error() {
let response = LlmResponse::Stream(Box::pin(stream::once(async {
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index d389ae2f3..351f122ea 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -9,8 +9,8 @@ use std::path::Path;
use std::sync::Arc;
use libsy::algorithms::{
- LlmTaskClassifier, Noop, Passthrough, Random, StageRouter, StageRouterConfig, TargetPrompts,
- TaskClassifierConfig,
+ EscalationJudgeConfig, LlmTaskClassifier, Noop, Passthrough, Random, StageRouter,
+ StageRouterConfig, TargetPrompts, TaskClassifierConfig,
};
use libsy::stage_router::{HandoffNoteConfig, LlmFallback, PickerMode};
use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
@@ -194,6 +194,14 @@ enum RouteConfig {
weak_target: String,
#[serde(flatten)]
classifier_config: TaskClassifierConfig,
+ /// Present to route by escalation instead of up-front classification.
+ ///
+ /// The classifier target becomes a trajectory judge: every unlatched turn is served by
+ /// the weak tier and judged, and the session latches to the strong tier once
+ /// `confirmations` consecutive escalate verdicts accumulate. Absent, the classifier
+ /// picks a tier before the turn's call, as usual.
+ #[serde(default)]
+ escalation: Option<EscalationJudgeConfig>,
},
StageRouter {
id: String,
@@ -325,17 +333,27 @@ fn build_algorithm(
strong_target,
weak_target,
classifier_config,
+ escalation,
..
} => {
let classifier = resolve_target(route_name, classifier_target, targets)?;
let strong = resolve_target(route_name, strong_target, targets)?;
let weak = resolve_target(route_name, weak_target, targets)?;
// The weak model is the efficient tier; the strong model is the capable one.
- let algorithm =
- LlmTaskClassifier::new(classifier, weak, strong, classifier_config.clone())
- .map_err(|error| {
- ServerError::new(format!("llm_classifier route {route_name}: {error}"))
- })?;
+ // With `escalation`, the classifier target judges the weak tier's reply each turn
+ // instead of picking a tier ahead of it.
+ let algorithm = match escalation {
+ Some(judge_config) => LlmTaskClassifier::new_with_escalation(
+ classifier,
+ weak,
+ strong,
+ judge_config.clone(),
+ ),
+ None => LlmTaskClassifier::new(classifier, weak, strong, classifier_config.clone()),
+ }
+ .map_err(|error| {
+ ServerError::new(format!("llm_classifier route {route_name}: {error}"))
+ })?;
Ok(Arc::new(algorithm))
}
RouteConfig::StageRouter {
@@ -510,6 +528,27 @@ target = "weak"
Ok(())
}
+ #[test]
+ fn an_escalation_table_switches_the_classifier_route_to_escalation() -> ServerResult<()> {
+ // Present: the classifier target judges the weak tier's reply each turn instead of
+ // picking a tier ahead of it. The route builds either way, so the assertion is that
+ // the knob parses and its settings reach the algorithm's validation.
+ let escalating = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { confirmations = 2 }",
+ );
+ server_state_from_toml(&escalating)?;
+
+ // A setting that would starve the judge is rejected here rather than on the first
+ // request, the same as any other unusable route configuration.
+ let starved = VALID_CONFIG.replace(
+ "base_threshold = 0.5",
+ "base_threshold = 0.5\nescalation = { confirmations = 0 }",
+ );
+ assert!(error_message(&starved).contains("confirmations must be at least 1"));
+ Ok(())
+ }
+
#[test]
fn rejects_unknown_fields_and_algorithm_types() {
let unknown_field =