Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): avoid zero exit tool-signal matches (#651)

sabhatinasd5cfe5bdfd2a
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 4638f5e9b..c2f576287 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -140,6 +140,9 @@ pub struct StageRouterConfig {
pub recent_window: Option<usize>,
/// Exact tool-name semantics added to the built-in coding vocabulary.
pub tool_semantics: ToolSemantics,
+ /// Requests to keep on the capable tier after an escalation. A clean test
+ /// pass clears the hold early.
+ pub capable_hold_turns: u32,
/// Note handed to the model on a signal-driven escalation, and on a
/// hand-back to the efficient tier when a de-escalation note is configured.
pub handoff_notes: Option<HandoffNoteConfig>,
@@ -161,6 +164,7 @@ impl StageRouterConfig {
confidence_threshold,
recent_window: None,
tool_semantics: ToolSemantics::default(),
+ capable_hold_turns: 2,
handoff_notes: None,
capable_system_prompt: None,
efficient_system_prompt: None,
@@ -218,7 +222,8 @@ pub(crate) fn build_stage_route(config: StageRouterConfig) -> Result<FallThrough
let default_tier = config.mode.default_tier();
let fall_open = FallOpen { default_tier };
- let mut classifier = StageClassifier::new(config.mode, config.confidence_threshold);
+ let mut classifier = StageClassifier::new(config.mode, config.confidence_threshold)
+ .with_capable_hold_turns(config.capable_hold_turns);
if let Some(notes) = config.handoff_notes {
classifier = classifier.with_handoff_notes(notes);
}
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index b654ee8b6..2338f7b01 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -45,6 +45,8 @@ const HARD_SEVERITY: f64 = 0.7;
const SIGNAL_UNIT: f64 = 0.10;
/// Critical severity forces the capable tier regardless of the scorer.
const SEVERITY_CRITICAL: f32 = 1.0;
+/// Session-state key for the remaining capable-tier recovery turns.
+const CAPABLE_HOLD_KEY: &str = "capable_hold_turns";
/// Counts final stage-router choices by decision source and semantic target.
const ROUTING_DECISIONS_METRIC: &str = "switchyard.stage_router.routing_decisions";
@@ -164,8 +166,8 @@ pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource)
pub enum DecisionSource {
/// Hard override (critical severity or context compaction).
Override,
- /// Settled run: recent tests passed with recent production and no error.
- TestsPassed,
+ /// A recent escalation is being held on the capable tier.
+ CapableHold,
/// Scorer crossed `confidence_threshold`.
Dimensions,
/// Scorer was not confident, so the signals did not decide this turn.
@@ -181,7 +183,7 @@ impl DecisionSource {
pub fn as_str(self) -> &'static str {
match self {
Self::Override => "override",
- Self::TestsPassed => "tests_passed",
+ Self::CapableHold => "capable_hold",
Self::Dimensions => "dimensions",
Self::Ambiguous => "ambiguous",
Self::LlmClassifier => "llm-classifier",
@@ -344,54 +346,57 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult {
}
}
-/// Hard **escalate** — force the capable tier no matter what the scorer would
-/// say. Fires on a critical error or a compacted context.
-fn should_escalate(signal: &ToolSignals) -> bool {
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum OverrideReason {
+ RepeatedFailure,
+ CriticalError,
+ Compaction,
+}
+
+impl OverrideReason {
+ fn as_str(self) -> &'static str {
+ match self {
+ Self::RepeatedFailure => "repeated_failure",
+ Self::CriticalError => "critical_error",
+ Self::Compaction => "compaction",
+ }
+ }
+}
+
+/// Hard **escalate** — force the capable tier no matter what the scorer would say.
+fn override_reason(signal: &ToolSignals) -> Option<OverrideReason> {
// Compaction wipes the accumulated signals, so a task that had escalated
// would snap back to efficient — a context big enough to overflow belongs capable.
if signal.compacted {
- return true;
+ return Some(OverrideReason::Compaction);
}
// A critical error is unambiguous.
- signal.severity >= SEVERITY_CRITICAL
-}
-
-/// Hard **de-escalate** — drop to the cheap tier on a settled turn: tests
-/// passed, code was just written or edited, and nothing errored in the window.
-fn should_deescalate(signal: &ToolSignals) -> bool {
- signal.tests_passed
- && (signal.recent_write_count + signal.recent_edit_count) >= 1
- && signal.severity <= 0.0
+ if signal.severity >= SEVERITY_CRITICAL {
+ return Some(OverrideReason::CriticalError);
+ }
+ signal
+ .repeated_failure
+ .then_some(OverrideReason::RepeatedFailure)
}
/// Decide a turn's tier from its signal.
///
/// The rules run in order; the first that fires wins:
///
-/// 1. **Escalate** — a hard reason to go capable (critical error / compaction).
-/// 2. **De-escalate** — a hard reason to go cheap (a settled turn).
-/// 3. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it.
-/// 4. **Fall open** — not confident: hand to the classifier, else the default.
+/// 1. **Escalate** — repeated failure, critical error, or compaction.
+/// 2. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it.
+/// 3. **Fall open** — not confident: hand to the classifier, else the default.
///
-/// Rules 1 and 2 are the two hard shortcuts that skip the scorer — one always
-/// escalates, one always de-escalates. **Escalate is checked first**, so a
-/// critical error still wins on a turn whose tests also happened to pass.
-///
-/// Deterministic and pure: the async classifier lives in the caller, so rule 4
+/// Deterministic and pure: the async classifier lives in the caller, so rule 3
/// returns [`PickOutcome::ConsultClassifier`] instead of calling it here. The
/// `no_signal` case (no tool activity yet) is handled one level up.
pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f64) -> PickOutcome {
// 1. Escalate — a hard reason to go capable, ahead of everything else.
- if should_escalate(signal) {
+ if override_reason(signal).is_some() {
return resolved(Tier::Capable, DecisionSource::Override, 0.5, Some(1.0));
}
- // 2. De-escalate — a hard reason to go cheap (the turn is winding down).
- if should_deescalate(signal) {
- return resolved(Tier::Efficient, DecisionSource::TestsPassed, 0.5, None);
- }
-
- // 3. Scorer — no hard reason either way, so weigh error vs production.
+ // 2. Scorer — no hard reason, so weigh error vs production.
// Resolve outside the closed ambiguous band [0.5 - t/2, 0.5 + t/2].
let scored = score_signal(signal);
let probability = scored.probability();
@@ -410,7 +415,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f
);
}
- // 4. Fall open — the signals didn't corroborate enough to be sure. Hand off
+ // 3. Fall open — the signals didn't corroborate enough to be sure. Hand off
// to the caller's classifier; with none, land on the picker's default.
PickOutcome::ConsultClassifier {
probability,
@@ -513,6 +518,7 @@ impl HandoffNoteConfig {
pub struct StageClassifier {
mode: PickerMode,
confidence_threshold: f64,
+ capable_hold_turns: u32,
handoff_notes: Option<HandoffNoteConfig>,
}
@@ -523,10 +529,18 @@ impl StageClassifier {
Self {
mode,
confidence_threshold,
+ capable_hold_turns: 2,
handoff_notes: None,
}
}
+ /// Keep the capable tier for this many requests after an escalation. A
+ /// clean passing test clears the hold early. Set to zero to disable.
+ pub fn with_capable_hold_turns(mut self, turns: u32) -> Self {
+ self.capable_hold_turns = turns;
+ self
+ }
+
/// Hand the routed model a note on a signal-driven escalation, and on a
/// hand-back to the efficient tier when a de-escalation note is configured.
pub fn with_handoff_notes(mut self, config: HandoffNoteConfig) -> Self {
@@ -549,6 +563,42 @@ impl StageClassifier {
prompts::append_note(request, note);
}
}
+
+ fn capable_hold_key(request: &Request) -> String {
+ request
+ .metadata
+ .as_ref()
+ .and_then(|metadata| metadata.agent_id.as_deref())
+ .map_or_else(
+ || CAPABLE_HOLD_KEY.to_string(),
+ |agent_id| format!("{CAPABLE_HOLD_KEY}:{agent_id}"),
+ )
+ }
+
+ fn consume_capable_hold(state: &mut State, key: &str) -> bool {
+ let Some(StateValue::Count(remaining)) = state.extra.get_mut(key) else {
+ return false;
+ };
+ if *remaining == 0 {
+ state.extra.remove(key);
+ return false;
+ }
+ *remaining -= 1;
+ if *remaining == 0 {
+ state.extra.remove(key);
+ }
+ true
+ }
+
+ fn set_capable_hold(&self, state: &mut State, key: &str) {
+ if self.capable_hold_turns == 0 {
+ state.extra.remove(key);
+ } else {
+ state
+ .extra
+ .insert(key.to_string(), StateValue::Count(self.capable_hold_turns));
+ }
+ }
}
#[async_trait]
@@ -559,15 +609,23 @@ impl Classifier<State> for StageClassifier {
request: &mut Request,
driver: &Driver,
) -> Result<(Classification, Option<switchyard_protocol::Response>)> {
- let tool_signals = &state.tool_signals;
- let Some(signal) = tool_signals else {
+ let Some(signal) = state.tool_signals.clone() else {
// No tool activity yet — nothing to score, so the signals have no
// opinion, same as a below-threshold turn.
return Ok((Self::abstain(state), None));
};
- let outcome = pick_tier(signal, self.mode, self.confidence_threshold);
- record_score_metrics(signal, &outcome);
+ let capable_hold_key = Self::capable_hold_key(request);
+ let clean_test_pass = signal.tests_passed && signal.no_error_streak > 0;
+ if clean_test_pass {
+ state.extra.remove(&capable_hold_key);
+ }
+ let outcome = if !clean_test_pass && Self::consume_capable_hold(state, &capable_hold_key) {
+ resolved(Tier::Capable, DecisionSource::CapableHold, 0.5, Some(1.0))
+ } else {
+ pick_tier(&signal, self.mode, self.confidence_threshold)
+ };
+ record_score_metrics(&signal, &outcome);
match outcome {
PickOutcome::Resolved {
tier,
@@ -580,8 +638,26 @@ impl Classifier<State> for StageClassifier {
Tier::Efficient => Category::Efficient,
};
let target = driver.first_model_for(&category)?;
+ if tier == Tier::Capable
+ && matches!(
+ source,
+ DecisionSource::Override | DecisionSource::Dimensions
+ )
+ {
+ self.set_capable_hold(state, &capable_hold_key);
+ }
record_decision_source(state, source);
record_routing_decision(source, target);
+ if source == DecisionSource::Override
+ && let Some(reason) = override_reason(&signal)
+ {
+ tracing::info!(
+ decision_source = source.as_str(),
+ override_reason = reason.as_str(),
+ target = %target,
+ "stage router override"
+ );
+ }
// Only a resolved turn routes on this classifier's target, so it
// is the only branch whose tier the signals actually chose — an
// ambiguous turn is decided further down the cascade.
@@ -674,6 +750,48 @@ mod tests {
));
}
+ #[test]
+ fn repeated_failure_overrides_to_capable() {
+ let signal = ToolSignals {
+ severity: HARD_SEVERITY as f32,
+ repeated_failure: true,
+ ..Default::default()
+ };
+ assert!(matches!(
+ pick_tier(&signal, PickerMode::EfficientFirst, 0.5),
+ PickOutcome::Resolved {
+ tier: Tier::Capable,
+ source: DecisionSource::Override,
+ ..
+ }
+ ));
+ }
+
+ #[test]
+ fn override_reasons_are_specific() {
+ assert_eq!(
+ override_reason(&ToolSignals {
+ repeated_failure: true,
+ ..Default::default()
+ }),
+ Some(OverrideReason::RepeatedFailure)
+ );
+ assert_eq!(
+ override_reason(&ToolSignals {
+ severity: SEVERITY_CRITICAL,
+ ..Default::default()
+ }),
+ Some(OverrideReason::CriticalError)
+ );
+ assert_eq!(
+ override_reason(&ToolSignals {
+ compacted: true,
+ ..Default::default()
+ }),
+ Some(OverrideReason::Compaction)
+ );
+ }
+
#[test]
fn one_signal_scores_below_half() {
// A single full wrong signal ≈ 0.46 confidence — just under 0.5.
@@ -825,11 +943,10 @@ mod tests {
}
#[tokio::test]
- async fn classifier_deescalates_settled_turn_to_weak() -> Result<()> {
- // Tests passed with recent production and no error → the settled-turn shortcut
- // resolves straight to a definite efficient-tier score.
+ async fn passing_tests_do_not_force_the_efficient_tier() -> Result<()> {
let signal = ToolSignals {
tests_passed: true,
+ no_error_streak: 1,
recent_write_count: 1,
severity: 0.0,
..Default::default()
@@ -839,13 +956,101 @@ mod tests {
let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), &driver)
.await?;
- match classification.0 {
- Classification::Scores(scores) => {
- assert_eq!(scores.len(), 1);
- assert_eq!(scores[0].target, "weak");
- }
- _ => panic!("expected a definite classification"),
+ assert!(matches!(classification.0, Classification::Ambiguous(_)));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn escalation_holds_capable_for_two_more_turns() -> Result<()> {
+ let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5);
+ let mut state = state_with(critical());
+ let driver = driver();
+
+ for _ in 0..3 {
+ let classification = classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ assert_eq!(classification.0.argmax(false)?.unwrap().target, "strong");
+ state.tool_signals = Some(ToolSignals::default());
}
+
+ let classification = classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ assert!(classification.0.argmax(false)?.is_none());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn passing_tests_clear_the_capable_hold() -> Result<()> {
+ let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5);
+ let mut state = state_with(critical());
+ let driver = driver();
+ classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ state.tool_signals = Some(ToolSignals {
+ tests_passed: true,
+ no_error_streak: 1,
+ severity: HARD_SEVERITY as f32,
+ ..Default::default()
+ });
+
+ let classification = classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ assert!(classification.0.argmax(false)?.is_none());
+ assert!(!state.extra.contains_key(CAPABLE_HOLD_KEY));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn capable_hold_is_isolated_per_agent() -> Result<()> {
+ let classifier = StageClassifier::new(PickerMode::EfficientFirst, 0.5);
+ let mut state = state_with(critical());
+ let driver = driver();
+ let mut parent = Request {
+ metadata: Some(Metadata {
+ agent_id: Some("parent".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ classifier.score(&mut state, &mut parent, &driver).await?;
+
+ state.tool_signals = Some(ToolSignals::default());
+ let mut child = Request {
+ metadata: Some(Metadata {
+ agent_id: Some("child".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ let classification = classifier.score(&mut state, &mut child, &driver).await?;
+
+ assert!(classification.0.argmax(false)?.is_none());
+ assert!(matches!(
+ state.extra.get("capable_hold_turns:parent"),
+ Some(StateValue::Count(2))
+ ));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn zero_disables_the_capable_hold() -> Result<()> {
+ let classifier =
+ StageClassifier::new(PickerMode::EfficientFirst, 0.5).with_capable_hold_turns(0);
+ let mut state = state_with(critical());
+ let driver = driver();
+ classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ state.tool_signals = Some(ToolSignals::default());
+
+ let classification = classifier
+ .score(&mut state, &mut Request::default(), &driver)
+ .await?;
+ assert!(classification.0.argmax(false)?.is_none());
Ok(())
}
@@ -905,7 +1110,7 @@ mod tests {
#[test]
fn deescalation_note_applies_to_efficient_when_configured() {
assert_eq!(
- config(true).note_for(Tier::Efficient, DecisionSource::TestsPassed),
+ config(true).note_for(Tier::Efficient, DecisionSource::Dimensions),
Some(DEESCALATION)
);
}
@@ -914,7 +1119,7 @@ mod tests {
fn no_deescalation_note_when_unconfigured() {
let config = HandoffNoteConfig::new(ESCALATION, None, true);
assert_eq!(
- config.note_for(Tier::Efficient, DecisionSource::TestsPassed),
+ config.note_for(Tier::Efficient, DecisionSource::Dimensions),
None
);
}
@@ -972,27 +1177,28 @@ mod tests {
}
#[tokio::test]
- async fn every_turn_the_signals_drive_carries_the_note() -> Result<()> {
- // Stateless by design: the note describes this turn's signals, so a run
- // of escalated turns each carries one. Nothing tracks the previous tier.
+ async fn held_turns_do_not_repeat_the_escalation_note() -> Result<()> {
let classifier = noting_classifier(PickerMode::EfficientFirst);
let mut state = state_with(critical());
let driver = driver();
- for _ in 0..3 {
- let mut request = request();
- classifier.score(&mut state, &mut request, &driver).await?;
- assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}")));
+ let mut first = request();
+ classifier.score(&mut state, &mut first, &driver).await?;
+ assert_eq!(trailing_text(&first), Some(format!("hi|{ESCALATION}")));
+
+ for _ in 0..2 {
+ let mut held = request();
+ classifier.score(&mut state, &mut held, &driver).await?;
+ assert_eq!(trailing_text(&held), Some("hi".to_string()));
}
Ok(())
}
#[tokio::test]
- async fn a_settled_turn_carries_the_deescalation_note() -> Result<()> {
- // Tests passed with recent production resolves to weak on the settled-turn
- // shortcut, which is the hand-back the de-escalation note is for.
+ async fn a_passing_test_does_not_force_a_deescalation_note() -> Result<()> {
let signal = ToolSignals {
tests_passed: true,
+ no_error_streak: 1,
recent_write_count: 1,
..Default::default()
};
@@ -1004,7 +1210,7 @@ mod tests {
.score(&mut state, &mut request, &driver)
.await?;
- assert_eq!(trailing_text(&request), Some(format!("hi|{DEESCALATION}")));
+ assert_eq!(trailing_text(&request), Some("hi".to_string()));
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs
index 43e1bd814..f305d5135 100644
--- a/crates/libsy/src/algorithms/util/tool_signals.rs
+++ b/crates/libsy/src/algorithms/util/tool_signals.rs
@@ -12,6 +12,8 @@
#![allow(dead_code)]
+use std::path::Path;
+
use async_trait::async_trait;
use serde::Deserialize;
use serde_json::Value;
@@ -39,7 +41,7 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
),
(
"connection_refused",
- CRITICAL,
+ HARD,
&[
"connection refused",
"connectionrefusederror",
@@ -83,17 +85,14 @@ static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
],
),
// SOFT: plain non-zero exit without a recognisable exception traceback.
- (
- "exit_nonzero",
- SOFT,
- &[
- "exit code 1",
- "exit code 2",
- "exit status 1",
- "returned non-zero",
- "exited with code",
- ],
- ),
+ ("exit_nonzero", SOFT, &["returned non-zero"]),
+];
+
+static NONZERO_EXIT_PHRASES: &[&str] = &[
+ "exit code",
+ "exit status",
+ "exited with code",
+ "exited with status",
];
static EDIT_TOOL_NAMES: &[&str] = &[
@@ -132,6 +131,13 @@ static BASH_WRITE_PATTERNS: &[&str] = &[
/// interpreter is running them rather than a search looking for them.
static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("];
+static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[
+ "writefilesync(",
+ "writefile(",
+ "appendfilesync(",
+ "appendfile(",
+];
+
static BASH_EDIT_PATTERNS: &[&str] = &[
"sed -i",
"sed --in-place",
@@ -151,6 +157,31 @@ static BASH_READ_PATTERNS: &[&str] = &[
"diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ",
];
+/// Read-only shell programs seen in Codex trajectories. Matching is limited to
+/// command-segment starts so prose and arguments do not masquerade as actions.
+static BASH_READ_COMMANDS: &[&str] = &[
+ "cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc",
+ "diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath",
+ "basename", "dirname", "printenv",
+];
+
+static GIT_READ_SUBCOMMANDS: &[&str] = &[
+ "status",
+ "diff",
+ "log",
+ "show",
+ "show-ref",
+ "rev-parse",
+ "ls-files",
+ "ls-remote",
+ "ls-tree",
+ "grep",
+ "blame",
+ "merge-base",
+ "check-ignore",
+ "tag",
+];
+
static READ_TOOL_NAMES: &[&str] = &["read", "view", "read_file", "search_files"];
// Planning / scratchpad tool calls — investigative (non-producing) activity.
@@ -170,8 +201,8 @@ static BASH_TOOL_NAMES: &[&str] = &[
"exec_command", // codex
];
-// Prefer false negatives: tests_passed routes the picker to EFFICIENT, so a false
-// positive would drop tier on an unfinished task.
+// Prefer false negatives: tests_passed clears a capable hold, so a false positive
+// could hand an unfinished task back too early.
static TEST_PASS_PHRASES: &[&str] = &[
" passed",
"passed in",
@@ -296,6 +327,9 @@ pub struct ToolSignals {
/// Windowed so an error persists through the recovery turns instead of clearing
/// the instant the next result is clean.
pub severity: f32,
+ /// The same hard-or-critical failure appeared at least twice in the recent
+ /// tool-result window.
+ pub repeated_failure: bool,
/// Consecutive clean tool results back from the most recent. `0` if the last failed.
pub no_error_streak: u32,
/// Total edit-style tool calls in the request.
@@ -322,7 +356,7 @@ pub struct ToolSignals {
/// Consecutive trailing tool calls in the `Unknown` category (no Write/Edit/Read/
/// Plan match). Surfaced in the classifier state summary; not scored directly.
pub pure_bash_streak: u32,
- /// At least one of the last three tool results matched a test-pass pattern.
+ /// A tool result after the latest recent failure matched a test-pass pattern.
pub tests_passed: bool,
/// Total `ToolResult` blocks, counted per block (a message batching N
/// results contributes N) and including empty-content results.
@@ -450,16 +484,23 @@ fn classify_tool_call_with_semantics(
&& let Some(cmd) = command
{
// Write/edit redirection trumps read-like operands.
- if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
+ if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(cmd) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
return ToolSemantic::Mutate(MutationKind::Write);
}
- if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) {
+ if shell_invokes_program(cmd, "node")
+ && JAVASCRIPT_WRITE_PATTERNS
+ .iter()
+ .any(|pattern| cmd.contains(pattern))
+ {
+ return ToolSemantic::Mutate(MutationKind::Write);
+ }
+ if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_edit(cmd) {
return ToolSemantic::Mutate(MutationKind::Edit);
}
- if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) {
+ if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(cmd) {
return ToolSemantic::Observe;
}
}
@@ -474,6 +515,176 @@ fn is_builtin_tool_name(lower: &str) -> bool {
|| BASH_TOOL_NAMES.contains(&lower)
}
+/// Split a shell line at unquoted command separators. This intentionally avoids
+/// pretending to be a full shell parser; only the leading program and flags of
+/// each segment are inspected below.
+fn shell_segments(command: &str) -> impl Iterator<Item = &str> {
+ let mut chars = command.char_indices();
+ let mut start = 0usize;
+ let mut quote = None;
+ let mut escaped = false;
+ let mut finished = false;
+
+ std::iter::from_fn(move || {
+ loop {
+ for (index, character) in chars.by_ref() {
+ if escaped {
+ escaped = false;
+ } else if character == '\\' && quote != Some('\'') {
+ escaped = true;
+ } else if quote == Some(character) {
+ quote = None;
+ } else if quote.is_none() && matches!(character, '\'' | '"') {
+ quote = Some(character);
+ } else if quote.is_none() && matches!(character, '\n' | ';' | '|' | '&') {
+ let segment = command[start..index].trim();
+ start = index + character.len_utf8();
+ if !segment.is_empty() {
+ return Some(segment);
+ }
+ }
+ }
+
+ if finished {
+ return None;
+ }
+ finished = true;
+ let segment = command[start..].trim();
+ if !segment.is_empty() {
+ return Some(segment);
+ }
+ }
+ })
+}
+
+fn shell_words(segment: &str) -> std::iter::Peekable<std::str::SplitAsciiWhitespace<'_>> {
+ let mut words = segment.split_ascii_whitespace().peekable();
+
+ if words.peek().copied() == Some("env") {
+ words.next();
+ while words.peek().is_some_and(|word| word.starts_with('-')) {
+ words.next();
+ }
+ }
+ while words
+ .peek()
+ .is_some_and(|word| word.contains('=') && !word.starts_with('='))
+ {
+ words.next();
+ }
+
+ words
+}
+
+fn program_name(word: &str) -> &str {
+ Path::new(word)
+ .file_name()
+ .and_then(|name| name.to_str())
+ .unwrap_or(word)
+}
+
+fn shell_invokes_program(command: &str, expected: &str) -> bool {
+ shell_segments(command).any(|segment| {
+ shell_words(segment)
+ .next()
+ .is_some_and(|word| program_name(word) == expected)
+ })
+}
+
+fn shell_command_is_write(command: &str) -> bool {
+ shell_segments(command).any(|segment| {
+ let mut words = shell_words(segment);
+ let Some(program) = words.next().map(program_name) else {
+ return false;
+ };
+ if matches!(program, "cp" | "mkdir" | "touch" | "install") {
+ return true;
+ }
+
+ let redirects_output = words.any(|word| matches!(word, ">" | ">>"));
+ redirects_output
+ && (matches!(program, "echo" | "printf" | "git")
+ || BASH_READ_COMMANDS.contains(&program))
+ })
+}
+
+fn shell_command_is_edit(command: &str) -> bool {
+ shell_segments(command).any(|segment| {
+ let mut words = shell_words(segment);
+ let Some(program) = words.next().map(program_name) else {
+ return false;
+ };
+ let has_arg = |arg: &str| words.clone().any(|word| word == arg);
+
+ match program {
+ "mv" | "rm" => true,
+ "perl" => words
+ .take_while(|word| word.starts_with('-'))
+ .any(|option| {
+ option
+ .trim_start_matches('-')
+ .chars()
+ .any(|flag| flag == 'i')
+ }),
+ "git" => words
+ .next()
+ .is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")),
+ "gofmt" => has_arg("-w"),
+ "cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"),
+ "ruff" => {
+ let subcommand = words.clone().next();
+ (subcommand == Some("format") && !has_arg("--check"))
+ || (subcommand == Some("check") && has_arg("--fix"))
+ }
+ "prettier" => has_arg("--write"),
+ "black" => !has_arg("--check"),
+ _ => {
+ (words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write"))
+ || (words.clone().any(|word| program_name(word) == "ruff")
+ && ((has_arg("format") && !has_arg("--check"))
+ || (has_arg("check") && has_arg("--fix"))))
+ }
+ }
+ })
+}
+
+fn shell_command_is_read(command: &str) -> bool {
+ shell_segments(command).any(|segment| {
+ if segment == "env" {
+ return true;
+ }
+ let mut words = shell_words(segment);
+ let Some(program) = words.next().map(program_name) else {
+ return false;
+ };
+
+ if BASH_READ_COMMANDS.contains(&program) {
+ return true;
+ }
+ if program == "command" && words.next() == Some("-v") {
+ return true;
+ }
+ if program == "type" {
+ return true;
+ }
+ if program != "git" {
+ return false;
+ }
+
+ match words.next() {
+ Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')),
+ Some("remote") => words
+ .next()
+ .is_none_or(|arg| arg.starts_with('-') || arg == "get-url"),
+ Some("config") => words
+ .next()
+ .is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")),
+ Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand),
+ None => false,
+ }
+ })
+}
+
// ─── extraction entry point ───────────────────────────────────────────────────
/// Extract all tool-execution signals from a normalized [`Request`].
@@ -609,11 +820,17 @@ fn build_signal(
// error signal instead of the router flapping straight back to the weak tier.
let sev_start = tool_texts.len().saturating_sub(recent_window.max(1));
let mut severity = 0.0f32;
+ let mut failure_fingerprints = Vec::new();
+ let mut repeated_failure = false;
for text in &tool_texts[sev_start..] {
let (sev, _patterns) = classify_text(text);
if sev > severity {
severity = sev;
}
+ if let Some(fingerprint) = failure_fingerprint(text) {
+ repeated_failure |= failure_fingerprints.contains(&fingerprint);
+ failure_fingerprints.push(fingerprint);
+ }
}
let no_error_streak = compute_no_error_streak(&tool_texts);
@@ -682,6 +899,7 @@ fn build_signal(
ToolSignals {
severity,
+ repeated_failure,
no_error_streak,
edit_count,
write_count,
@@ -744,9 +962,171 @@ pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {
severity = severity.max(*sev);
}
}
+ if has_nonzero_exit_status(&lower) && !patterns.iter().any(|p| p == "exit_nonzero") {
+ patterns.push("exit_nonzero".to_string());
+ severity = severity.max(SOFT);
+ }
+ for (name, matched) in [
+ ("compile_error", has_compiler_diagnostic(&lower)),
+ ("runtime_exception", has_runtime_exception(&lower)),
+ ("runtime_panic", has_runtime_panic(&lower)),
+ ("patch_error", has_patch_failure(&lower)),
+ ] {
+ if matched && !patterns.iter().any(|pattern| pattern == name) {
+ patterns.push(name.to_string());
+ severity = severity.max(HARD);
+ }
+ }
(severity, patterns)
}
+/// Stable identity for a material failure. Soft non-zero exits are excluded:
+/// they are too generic to prove that an agent is repeating the same mistake.
+fn failure_fingerprint(text: &str) -> Option<String> {
+ let (severity, patterns) = classify_text(text);
+ if severity < HARD {
+ return None;
+ }
+
+ let lower = text.to_lowercase();
+ let diagnostic = lower
+ .lines()
+ .find(|line| is_failure_diagnostic(line))
+ .or_else(|| lower.lines().find(|line| !line.trim().is_empty()))
+ .unwrap_or_default();
+ let normalized = normalize_failure_text(diagnostic);
+ Some(format!("{}|{normalized}", patterns.join(",")))
+}
+
+fn is_failure_diagnostic(line: &str) -> bool {
+ let line = line.trim();
+ [
+ "error",
+ "exception",
+ "panic",
+ "failed",
+ "timed out",
+ "timeout",
+ "connection refused",
+ "cannot allocate memory",
+ "out of memory",
+ "not found",
+ ]
+ .iter()
+ .any(|marker| line.contains(marker))
+}
+
+/// Removes values that normally change between retries while retaining the
+/// diagnostic wording that distinguishes one failure from another.
+fn normalize_failure_text(text: &str) -> String {
+ let mut normalized = String::new();
+ for word in text.split_whitespace() {
+ if !normalized.is_empty() {
+ normalized.push(' ');
+ }
+ let mut in_digits = false;
+ if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") {
+ normalized.push_str("<path>");
+ continue;
+ }
+ for character in word.chars() {
+ if character.is_ascii_digit() {
+ if !in_digits {
+ normalized.push('#');
+ in_digits = true;
+ }
+ } else {
+ normalized.push(character);
+ in_digits = false;
+ }
+ }
+ }
+ normalized.chars().take(240).collect()
+}
+
+fn has_compiler_diagnostic(lower: &str) -> bool {
+ lower.lines().any(|line| {
+ let line = line.trim_start();
+ if matches!(
+ line,
+ "compilation failed" | "error: compilation failed" | "error: could not compile"
+ ) || line.starts_with("error: could not compile ")
+ {
+ return true;
+ }
+
+ let Some(rest) = line.strip_prefix("error[e") else {
+ return false;
+ };
+ let Some((code, _)) = rest.split_once("]:") else {
+ return false;
+ };
+ !code.is_empty() && code.chars().all(|character| character.is_ascii_digit())
+ })
+}
+
+fn has_runtime_exception(lower: &str) -> bool {
+ let has_exception_line = lower.lines().any(|line| {
+ let line = line.trim_start();
+ [
+ "typeerror:",
+ "referenceerror:",
+ "rangeerror:",
+ "runtimeerror:",
+ "keyerror:",
+ "attributeerror:",
+ ]
+ .iter()
+ .any(|prefix| line.starts_with(prefix))
+ });
+ has_exception_line && (lower.contains("\n at ") || lower.contains("\n at "))
+}
+
+fn has_runtime_panic(lower: &str) -> bool {
+ lower
+ .lines()
+ .any(|line| line.trim_start().starts_with("panic: runtime error:"))
+ && (lower.contains("\ngoroutine ") || lower.contains("[signal sig"))
+}
+
+fn has_patch_failure(lower: &str) -> bool {
+ lower.lines().any(|line| {
+ let line = line.trim_start();
+ line.starts_with("error: patch failed:")
+ || line.starts_with("patch failed:")
+ || line.contains(": patch does not apply")
+ || line.starts_with("invalid context")
+ })
+}
+
+/// Detects `exit_nonzero` only when a supported exit phrase is followed by a
+/// nonzero decimal status.
+///
+/// Codex includes "Process exited with code 0" on clean tool results, so exit
+/// phrases must parse their numeric status instead of matching the phrase alone.
+fn has_nonzero_exit_status(lower: &str) -> bool {
+ NONZERO_EXIT_PHRASES
+ .iter()
+ .any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase))
+}
+
+/// Matches common "exit code/status N" spellings after optional separators.
+fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {
+ let mut cursor = 0usize;
+ while let Some(rel) = lower[cursor..].find(phrase) {
+ let value_start = cursor + rel + phrase.len();
+ let rest = lower[value_start..].trim_start_matches(|c: char| {
+ c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`')
+ });
+ let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
+ if !digits.is_empty() && digits.chars().any(|d| d != '0') {
+ return true;
+ }
+ cursor = value_start;
+ }
+ false
+}
+
fn compute_no_error_streak(tool_texts: &[String]) -> u32 {
let mut streak = 0u32;
for text in tool_texts.iter().rev() {
@@ -761,7 +1141,12 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 {
fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool {
let start = tool_texts.len().saturating_sub(recent_window.max(1));
- tool_texts[start..].iter().any(|text| {
+ let recent = &tool_texts[start..];
+ let after_latest_failure = recent
+ .iter()
+ .rposition(|text| classify_text(text).0 > 0.0)
+ .map_or(recent, |index| &recent[index + 1..]);
+ after_latest_failure.iter().any(|text| {
let lower = text.to_lowercase();
TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))
&& !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))
@@ -811,7 +1196,9 @@ mod tests {
use super::*;
use crate::algorithms::util::stage::score_signal;
use serde_json::json;
- use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult};
+ use switchyard_protocol::{
+ ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult,
+ };
fn with_messages(messages: Vec<Message>) -> Request {
Request {
@@ -882,6 +1269,36 @@ mod tests {
assert_eq!(sev, CRITICAL);
}
+ #[test]
+ fn connection_refused_is_hard() {
+ let (severity, _) = classify_text("Connection refused on port 8000");
+ assert_eq!(severity, HARD);
+ }
+
+ #[test]
+ fn repeated_failure_ignores_volatile_paths_and_numbers() {
+ let request = with_messages(vec![
+ tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"),
+ tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"),
+ ]);
+ assert!(ToolSignals::from_request(&request, None).repeated_failure);
+ }
+
+ #[test]
+ fn different_failures_are_not_repeated() {
+ let request = with_messages(vec![
+ tr("error[E0308]: mismatched types"),
+ tr("error[E0509]: cannot move out"),
+ ]);
+ assert!(!ToolSignals::from_request(&request, None).repeated_failure);
+ }
+
+ #[test]
+ fn one_material_failure_is_not_repeated() {
+ let request = with_messages(vec![tr("Connection refused on port 8000")]);
+ assert!(!ToolSignals::from_request(&request, None).repeated_failure);
+ }
+
#[test]
fn severity_is_max_across_patterns() {
// exit_nonzero (SOFT) + traceback (HARD) → HARD.
@@ -889,6 +1306,64 @@ mod tests {
assert_eq!(sev, HARD);
}
+ #[test]
+ fn codex_process_exit_zero_stays_clean() {
+ let (sev, patterns) =
+ classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok");
+ assert_eq!(sev, 0.0);
+ assert!(!patterns.contains(&"exit_nonzero".to_string()));
+ }
+
+ #[test]
+ fn nonzero_exit_codes_are_soft_errors() {
+ let cases = [
+ "Process exited with code 1",
+ "Process exited with code 127",
+ "exit code: 2",
+ "exit status 3",
+ "exited with status 9",
+ ];
+ for case in cases {
+ let (sev, patterns) = classify_text(case);
+ assert_eq!(sev, SOFT, "expected soft severity for {case}");
+ assert!(patterns.contains(&"exit_nonzero".to_string()));
+ }
+ }
+
+ #[test]
+ fn partial_process_failures_are_hard_errors() {
+ let cases = [
+ (
+ "Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out",
+ "compile_error",
+ ),
+ (
+ "Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2",
+ "runtime_exception",
+ ),
+ (
+ "Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:",
+ "runtime_panic",
+ ),
+ (
+ "Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply",
+ "patch_error",
+ ),
+ ];
+ for (text, expected_pattern) in cases {
+ let (severity, patterns) = classify_text(text);
+ assert_eq!(severity, HARD, "expected hard severity for {text}");
+ assert!(patterns.iter().any(|pattern| pattern == expected_pattern));
+ }
+ }
+
+ #[test]
+ fn source_text_that_names_exceptions_stays_clean() {
+ let text =
+ "pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';";
+ assert_eq!(classify_text(text).0, 0.0);
+ }
+
#[test]
fn file_does_not_exist_is_hard() {
// Claude Code Read-tool miss. Trace-mined addition (22 true / 2 false positives).
@@ -938,6 +1413,25 @@ mod tests {
));
}
+ #[test]
+ fn tests_passed_must_follow_the_latest_failure() {
+ assert!(!detect_tests_passed(
+ &[
+ "5 passed in 0.12s".to_string(),
+ "Traceback (most recent call last):\nValueError".to_string(),
+ "edit applied".to_string(),
+ ],
+ DEFAULT_RECENT_WINDOW
+ ));
+ assert!(detect_tests_passed(
+ &[
+ "Traceback (most recent call last):\nValueError".to_string(),
+ "5 passed in 0.12s".to_string(),
+ ],
+ DEFAULT_RECENT_WINDOW
+ ));
+ }
+
#[test]
fn severity_is_windowed_over_recent_results() {
// An error two results back, then two clean results.
@@ -1129,6 +1623,17 @@ mod tests {
assert!(ToolSignals::from_request(&request, None).compacted);
}
+ #[test]
+ fn codex_compaction_metadata_stays_on_parent_route() {
+ let mut request = with_messages(vec![bash("ls")]);
+ request.metadata = Some(Metadata {
+ is_subagent: true,
+ agent_kind: Some("compact".to_string()),
+ ..Default::default()
+ });
+ assert!(!ToolSignals::from_request(&request, None).compacted);
+ }
+
#[test]
fn no_compaction_marker_stays_uncompacted() {
let request = with_messages(vec![
@@ -1338,6 +1843,102 @@ mod tests {
}
}
+ #[test]
+ fn codex_inspection_commands_classify_as_read() {
+ let cases = [
+ "sed -n '1,80p' src/lib.rs",
+ "rg -n 'needle' src",
+ "nl -ba src/lib.rs",
+ "cat package.json",
+ "jq '.scripts' package.json",
+ "git status --short",
+ "git log --oneline -5",
+ "git show HEAD:src/lib.rs",
+ "git branch --show-current",
+ "git remote -v",
+ "git config --get remote.origin.url",
+ ];
+ for command in cases {
+ assert_eq!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Observe,
+ "expected Read for {command}"
+ );
+ }
+ }
+
+ #[test]
+ fn quoted_shell_separators_do_not_create_commands() {
+ for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] {
+ assert_eq!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Observe,
+ "quoted text must not be parsed as a command: {command}"
+ );
+ }
+ }
+
+ #[test]
+ fn codex_shell_mutations_classify_as_production() {
+ let writes = [
+ "cp source.rs destination.rs",
+ "mkdir -p src/generated",
+ "touch src/generated/mod.rs",
+ "git show HEAD:file.rs > file.rs",
+ "node <<'node'\nfs.writefilesync('file.js', text)\nnode",
+ ];
+ for command in writes {
+ assert_eq!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Mutate(MutationKind::Write),
+ "expected Write for {command}"
+ );
+ }
+
+ let edits = [
+ "mv old.rs new.rs",
+ "rm obsolete.rs",
+ "gofmt -w main.go",
+ "cargo fmt",
+ "ruff check --fix src",
+ "perl -0pi -e 's/old/new/' src/lib.rs",
+ "npx prettier --write src/lib.ts",
+ "uv run ruff format src",
+ "git apply fix.patch",
+ ];
+ for command in edits {
+ assert_eq!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Mutate(MutationKind::Edit),
+ "expected Edit for {command}"
+ );
+ }
+ }
+
+ #[test]
+ fn formatter_checks_are_not_edits() {
+ for command in [
+ "cargo fmt --check",
+ "ruff format --check src",
+ "black --check src",
+ ] {
+ assert_ne!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Mutate(MutationKind::Edit),
+ "read-only formatter check must not be Edit: {command}"
+ );
+ }
+ }
+
+ #[test]
+ fn embedded_comparison_is_not_a_shell_write() {
+ let command = "node <<'node'\nif (index > 0) console.log(index)\nnode";
+ assert_eq!(
+ classify_tool_call("exec_command", Some(command)),
+ ToolSemantic::Unknown
+ );
+ }
+
#[test]
fn bash_write_precedence_over_read() {
// `cat /file > out` contains both `cat /` (read) and ` > ` (write);
diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs
index 626c150af..b3935a04c 100644
--- a/crates/switchyard-runner/src/algorithm.rs
+++ b/crates/switchyard-runner/src/algorithm.rs
@@ -495,6 +495,9 @@ pub struct StageTierConfig {
/// Exact tool-name semantics added to the built-in stage vocabulary.
#[serde(default)]
pub tool_semantics: ToolSemantics,
+ /// Requests to keep on the capable tier after an escalation.
+ #[serde(default)]
+ pub capable_hold_turns: Option<u32>,
/// Notes handed to a tier when the router switches to it.
#[serde(default)]
pub handoff_notes: Option<HandoffNoteConfig>,
@@ -1237,6 +1240,7 @@ fn build_algorithm(
confidence_threshold,
recent_turn_window,
tool_semantics,
+ capable_hold_turns,
handoff_notes,
..
} = tiers;
@@ -1247,6 +1251,9 @@ fn build_algorithm(
}
let mut config = StageRouterConfig::new(*picker, *confidence_threshold);
config.recent_window = *recent_turn_window;
+ if let Some(turns) = capable_hold_turns {
+ config.capable_hold_turns = *turns;
+ }
config.tool_semantics = tool_semantics.clone();
config.handoff_notes = handoff_notes.clone();
// The judge is called through its own target, so it is not a routing
@@ -1281,6 +1288,9 @@ fn build_algorithm(
let mut stage_config =
StageRouterConfig::new(PickerMode::EfficientFirst, stage.confidence_threshold);
stage_config.recent_window = stage.recent_turn_window;
+ if let Some(turns) = stage.capable_hold_turns {
+ stage_config.capable_hold_turns = turns;
+ }
stage_config.tool_semantics = stage.tool_semantics.clone();
stage_config.handoff_notes = stage.handoff_notes.clone();
let config = CompositeRouterConfig {
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 60a8f3444..d65c585dc 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -907,6 +907,7 @@ capable_target = "strong"
efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 1.0
+capable_hold_turns = 2
[routes.stage.tool_semantics]
observe = ["lookup_customer"]
@@ -941,6 +942,7 @@ classify_trigger = "user_turn"
capable_target = "strong"
efficient_target = "weak"
confidence_threshold = 0.5
+capable_hold_turns = 2
[routes.composed.stage.tool_semantics]
new = ["send_message"]
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index c820d21d1..e595e7cef 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -266,6 +266,7 @@ optional `handoff_notes` and `classifier` tables and for tuning.
| `picker` | Yes | — | `efficient_first`, or `capable_first` (experimental, unbenchmarked). Tier used when the signals are not confident. |
| `confidence_threshold` | Yes | — | Corroboration a decisive pick needs. In `[0, 1]`. |
| `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. |
+| `capable_hold_turns` | No | `2` | Requests kept on the capable tier after escalation. A clean test pass clears the hold early; `0` disables it. |
| `tool_semantics.observe` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as read-only investigation. |
| `tool_semantics.mutate` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as state-changing production. |
| `tool_semantics.plan` | No | `[]` | Exact ASCII case-insensitive domain tool names that count as planning or task decomposition. |
@@ -305,6 +306,7 @@ configuration. Today a classifier sets the tier a stage router falls open to whe
| `stage.efficient_target` | Yes | — | Efficient tier. |
| `stage.confidence_threshold` | Yes | — | Corroboration a decisive signal needs. In `[0, 1]`. |
| `stage.recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. |
+| `stage.capable_hold_turns` | No | `2` | Requests kept on the capable tier after escalation. A clean test pass clears the hold early; `0` disables it. |
| `stage.tool_semantics.observe` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as observation. |
| `stage.tool_semantics.mutate` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as mutation. |
| `stage.tool_semantics.plan` | No | `[]` | Additional exact ASCII case-insensitive tool names that count as planning. |
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index 6e7ff9ac1..f9f600af1 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -31,7 +31,7 @@ For each LLM call, stage-router estimates which stage the agent is in from the
The axes are **corroborative**: the signed score is `tanh`-squashed to a
confidence in `[0, 1]`, so one full signal alone scores ~`0.46` and a second
corroborating signal is what pushes it decisively past a `0.5` threshold. A
-critical-error severity is a hard override that escalates on its own. The router
+repeated failures and critical-error severity are hard overrides that escalate. The router
then routes:
- the **capable** tier for uncertain, exploratory, or error-recovery turns, and
@@ -199,6 +199,7 @@ efficient_target = "weak"
picker = "efficient_first"
confidence_threshold = 0.5
recent_turn_window = 3 # optional, defaults to 3
+capable_hold_turns = 2 # optional, defaults to 2
```
Save as `routes.toml` and start the server:
@@ -242,8 +243,8 @@ rules are outside this exact-name configuration.
Add a `[routes.stage.handoff_notes]` section to pass a contextual note to the
model the router switches to. The escalation note is sent to the capable tier on
-a signal-driven escalation; the de-escalation note is sent back to the efficient
-tier when a settled signal drops the turn there.
+a signal-driven escalation; the de-escalation note is sent when the scorer
+decisively picks the efficient tier.
```toml
[routes.stage.handoff_notes]
@@ -302,8 +303,8 @@ paths through its cascade:
| Source | When |
|---|---|
-| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. |
-| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — landed the turn on the efficient tier. |
+| `override` | A repeated failure, critical-error severity, or context-compaction marker forced the capable tier. Structured logs set `override_reason` to `repeated_failure`, `critical_error`, or `compaction`. |
+| `capable_hold` | A recent escalation kept this recovery turn on the capable tier. |
| `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. |
| `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. |
| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured; the default tier was used. |
diff --git a/examples/litellm/tests/unit/test_routing_plugin_configuration.py b/examples/litellm/tests/unit/test_routing_plugin_configuration.py
index 123fb7681..73fbbcfc4 100644
--- a/examples/litellm/tests/unit/test_routing_plugin_configuration.py
+++ b/examples/litellm/tests/unit/test_routing_plugin_configuration.py
@@ -48,7 +48,7 @@ async def test_loads_stage_prompt_and_handoff_rewrites(tmp_path: Path) -> None:
config_path.write_text(
'algorithm = "stage"\n'
'picker = "capable_first"\n'
- "confidence_threshold = 0.5\n"
+ "confidence_threshold = 0.4\n"
'escalation_note = "The efficient tier failed."\n'
'deescalation_note = "The capable tier recovered."\n'
'capable_system_prompt = "Use the capable tier."\n'