Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): honor explicit tool-result failures in stage routing (#722)
Ayush Agarwal2eb53cf1ff89
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rsindex f305d5135..ba469cfdf 100644--- a/crates/libsy/src/algorithms/util/tool_signals.rs+++ b/crates/libsy/src/algorithms/util/tool_signals.rs@@ -4,7 +4,7 @@//! Tool-result context signals extracted from the conversation history.//!//! The extractor walks normalized messages, finds tool calls and results,-//! pattern-matches their text against a curated error table, and aggregates+//! reads explicit failure flags, matches text against an error table, and aggregates//! conversation-history metrics used by [`crate::StageRouter`] and the//! advisor gate's request-side guards.//!@@ -708,7 +708,7 @@ fn extract_tool_signals_with_window_and_semantics(// in the same shape here, so the signals do not depend on knowing which one it// arrived as.let messages = &request.llm_request.messages;- let mut tool_texts: Vec<String> = Vec::new();+ let mut tool_texts: Vec<(String, bool)> = Vec::new();let mut tool_calls: Vec<ObservedToolCall> = Vec::new();let mut compacted = false;let mut tool_result_count = 0usize;@@ -735,8 +735,10 @@ fn extract_tool_signals_with_window_and_semantics(.filter_map(text_of).collect::<Vec<_>>().join("\n");- if !text.is_empty() {- tool_texts.push(text);+ let is_error = result.is_error == Some(true);+ // An explicit failure remains a signal even without text.+ if !text.is_empty() || is_error {+ tool_texts.push((text, is_error));}}// Compaction is detected anywhere in the conversation: the summary@@ -807,7 +809,7 @@ fn text_of(block: &ContentBlock) -> Option<&str> {}fn build_signal(- tool_texts: Vec<String>,+ tool_texts: Vec<(String, bool)>,tool_calls: Vec<ObservedToolCall>,turn_depth: u32,recent_window: usize,@@ -822,12 +824,14 @@ fn build_signal(let mut severity = 0.0f32;let mut failure_fingerprints = Vec::new();let mut repeated_failure = false;- for text in &tool_texts[sev_start..] {+ for (text, is_error) in &tool_texts[sev_start..] {let (sev, _patterns) = classify_text(text);+ // Explicit failure is at least hard; retain stronger text diagnostics.+ let sev = if *is_error { sev.max(HARD) } else { sev };if sev > severity {severity = sev;}- if let Some(fingerprint) = failure_fingerprint(text) {+ if let Some(fingerprint) = failure_fingerprint(text, *is_error) {repeated_failure |= failure_fingerprints.contains(&fingerprint);failure_fingerprints.push(fingerprint);}@@ -980,11 +984,11 @@ pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {(severity, patterns)}-/// Stable identity for a material failure. Soft non-zero exits are excluded:-/// they are too generic to prove that an agent is repeating the same mistake.-fn failure_fingerprint(text: &str) -> Option<String> {+/// Stable identity for a material failure. Soft non-zero exits need an explicit+/// failure flag to count as a repeated mistake.+fn failure_fingerprint(text: &str, is_error: bool) -> Option<String> {let (severity, patterns) = classify_text(text);- if severity < HARD {+ if severity < HARD && !is_error {return None;}@@ -1127,11 +1131,11 @@ fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {false}-fn compute_no_error_streak(tool_texts: &[String]) -> u32 {+fn compute_no_error_streak(tool_texts: &[(String, bool)]) -> u32 {let mut streak = 0u32;- for text in tool_texts.iter().rev() {+ for (text, is_error) in tool_texts.iter().rev() {let (sev, _) = classify_text(text);- if sev > 0.0 {+ if *is_error || sev > 0.0 {break;}streak += 1;@@ -1139,14 +1143,14 @@ fn compute_no_error_streak(tool_texts: &[String]) -> u32 {streak}-fn detect_tests_passed(tool_texts: &[String], recent_window: usize) -> bool {+fn detect_tests_passed(tool_texts: &[(String, bool)], recent_window: usize) -> bool {let start = tool_texts.len().saturating_sub(recent_window.max(1));let recent = &tool_texts[start..];let after_latest_failure = recent.iter()- .rposition(|text| classify_text(text).0 > 0.0)+ .rposition(|(text, is_error)| *is_error || classify_text(text).0 > 0.0).map_or(recent, |index| &recent[index + 1..]);- after_latest_failure.iter().any(|text| {+ after_latest_failure.iter().any(|(text, _)| {let lower = text.to_lowercase();TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))&& !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))@@ -1299,6 +1303,33 @@ mod tests {assert!(!ToolSignals::from_request(&request, None).repeated_failure);}+ /// Explicit failures count even without diagnostic text and cannot signal recovery.+ #[test]+ fn structured_tool_failures_feed_error_and_recovery_signals() {+ for text in ["Dependency unavailable", "", "5 passed in 0.12s"] {+ let mut failed = tr(text);+ let ContentBlock::ToolResult(result) = &mut failed.content[0] else {+ panic!("expected tool result");+ };+ result.is_error = Some(true);+ let mut request = with_messages(vec![tr("5 passed in 0.12s"), failed.clone()]);+ let signals = ToolSignals::from_request(&request, Some(3));+ assert_eq!(signals.severity, HARD);+ assert!(!signals.repeated_failure);+ assert_eq!(signals.no_error_streak, 0);+ assert!(!signals.tests_passed);++ request.llm_request.messages.push(failed);+ assert!(ToolSignals::from_request(&request, Some(3)).repeated_failure);+ request.llm_request.messages.push(tr("5 passed in 0.12s"));+ let recovered = ToolSignals::from_request(&request, Some(1));+ assert_eq!(recovered.severity, 0.0);+ assert!(!recovered.repeated_failure);+ assert_eq!(recovered.no_error_streak, 1);+ assert!(recovered.tests_passed);+ }+ }+#[test]fn severity_is_max_across_patterns() {// exit_nonzero (SOFT) + traceback (HARD) → HARD.@@ -1383,16 +1414,16 @@ mod tests {#[test]fn no_error_streak_all_clean() {- let texts = vec!["ok".to_string(), "all good".to_string()];+ let texts = vec![("ok".to_string(), false), ("all good".to_string(), false)];assert_eq!(compute_no_error_streak(&texts), 2);}#[test]fn no_error_streak_stops_at_error() {let texts = vec![- "Traceback (most recent call last):".to_string(),- "ok".to_string(),- "ok".to_string(),+ ("Traceback (most recent call last):".to_string(), false),+ ("ok".to_string(), false),+ ("ok".to_string(), false),];assert_eq!(compute_no_error_streak(&texts), 2);}@@ -1400,7 +1431,7 @@ mod tests {#[test]fn tests_passed_detects_pytest_output() {assert!(detect_tests_passed(- &["====== 5 passed in 0.12s ======".to_string()],+ &[("====== 5 passed in 0.12s ======".to_string(), false)],DEFAULT_RECENT_WINDOW));}@@ -1408,7 +1439,7 @@ mod tests {#[test]fn tests_passed_ignores_partial_failures() {assert!(!detect_tests_passed(- &["2 failed, 5 passed in 0.56s".to_string()],+ &[("2 failed, 5 passed in 0.56s".to_string(), false)],DEFAULT_RECENT_WINDOW));}@@ -1417,16 +1448,22 @@ mod tests {fn tests_passed_must_follow_the_latest_failure() {assert!(!detect_tests_passed(&[- "5 passed in 0.12s".to_string(),- "Traceback (most recent call last):\nValueError".to_string(),- "edit applied".to_string(),+ ("5 passed in 0.12s".to_string(), false),+ (+ "Traceback (most recent call last):\nValueError".to_string(),+ false+ ),+ ("edit applied".to_string(), false),],DEFAULT_RECENT_WINDOW));assert!(detect_tests_passed(&[- "Traceback (most recent call last):\nValueError".to_string(),- "5 passed in 0.12s".to_string(),+ (+ "Traceback (most recent call last):\nValueError".to_string(),+ false+ ),+ ("5 passed in 0.12s".to_string(), false),],DEFAULT_RECENT_WINDOW));@@ -1679,7 +1716,7 @@ mod tests {fn tests_passed_detects_pytest_with_failure_block() {// Mixed pytest run: 2 failed + 5 passed → NOT considered tests_passed.assert!(!detect_tests_passed(- &["2 failed, 5 passed in 0.56s".to_string()],+ &[("2 failed, 5 passed in 0.56s".to_string(), false)],DEFAULT_RECENT_WINDOW));}@@ -1689,7 +1726,10 @@ mod tests {// Cargo's clean-run summary contains "0 failed" — must not trip the// failure list (regression: previously substring-matched "failed").assert!(detect_tests_passed(- &["running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string()],+ &[(+ "running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string(),+ false+ )],DEFAULT_RECENT_WINDOW));}@@ -1698,7 +1738,10 @@ mod tests {fn tests_passed_rejects_cargo_real_failure() {// Cargo's actual-failure summary: nonzero count before "failed".assert!(!detect_tests_passed(- &["running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string()],+ &[(+ "running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string(),+ false+ )],DEFAULT_RECENT_WINDOW));}@@ -1707,7 +1750,10 @@ mod tests {fn tests_passed_accepts_go_clean_summary() {// Go test's clean-run "0 errors" must not trip (regression).assert!(detect_tests_passed(- &["ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string()],+ &[(+ "ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string(),+ false+ )],DEFAULT_RECENT_WINDOW));}@@ -1716,7 +1762,7 @@ mod tests {fn tests_passed_accepts_pytest_zero_errors() {// Pytest long-form: "0 errors in 0.3s" on a clean run.assert!(detect_tests_passed(- &["5 passed, 0 errors in 0.30s".to_string()],+ &[("5 passed, 0 errors in 0.30s".to_string(), false)],DEFAULT_RECENT_WINDOW));}@@ -1724,7 +1770,7 @@ mod tests {#[test]fn tests_passed_detects_diy_checkmark() {assert!(detect_tests_passed(- &["✓ all checks passed".to_string()],+ &[("✓ all checks passed".to_string(), false)],DEFAULT_RECENT_WINDOW));}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex b87ead7c5..66f372761 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -539,7 +539,7 @@ async fn upstream_buffered_responses("model/invalid-error" => response["error"] = json!("invalid error details"),"model/invalid-code" => response["error"]["code"] = json!(42),"model/empty-code" => response["error"]["code"] = json!(""),- "model/fallback" => {+ "model/fallback" | "model/efficient" => {response["status"] = json!("completed");response["error"] = Value::Null;response["output"] = json!([{@@ -1646,6 +1646,81 @@ confidence_threshold = 0.5Ok(())}+/// Anthropic failure flags must affect routing before translating to Responses.+#[tokio::test]+async fn stage_route_honors_anthropic_tool_result_errors() -> TestResult {+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.upstream]+format = "openai_responses"+base_url = "{}/buffered"+[targets.capable]+id = "model/fallback"+llm_client = "upstream"+[targets.efficient]+id = "model/efficient"+llm_client = "upstream"+[routes.stage]+id = "switchyard/stage-structured-failure"+type = "stage_router"+capable_target = "capable"+efficient_target = "efficient"+picker = "efficient_first"+confidence_threshold = 0.5+capable_hold_turns = 0+recent_turn_window = 3+context_window = 131072+tool_calling = true+reasoning = true+"#,+ upstream.base_url.trim_end_matches("/v1")+ ))?);+ let neutral = "Synthetic dependency unavailable; retry with the recovery path.";+ for (is_error, text, expected) in [+ (false, neutral, "model/efficient"),+ (true, neutral, "model/fallback"),+ (+ false,+ "fatal runtime error: out of memory",+ "model/fallback",+ ),+ ] {+ let mut messages = vec![json!({+ "role": "user", "content": "Run the synthetic dependency probe."+ })];+ for id in ["toolu_probe_1", "toolu_probe_2"] {+ messages.push(json!({"role": "assistant", "content": [{+ "type": "tool_use", "id": id,+ "name": "synthetic_dependency_probe", "input": {}+ }]}));+ messages.push(json!({"role": "user", "content": [{+ "type": "tool_result", "tool_use_id": id,+ "content": text, "is_error": is_error+ }]}));+ }+ let body = json!({+ "model": "switchyard/stage-structured-failure",+ "max_tokens": 16, "messages": messages+ });+ for _ in 0..3 {+ let response = send(&app, "POST", "/v1/messages", Some(body.clone())).await?;+ assert_eq!(response.status, StatusCode::OK);+ assert_eq!(+ response.json()?["model"],+ expected,+ "is_error={is_error}, text={text}"+ );+ assert_eq!(+ upstream.models().await.last().map(String::as_str),+ Some(expected)+ );+ }+ }+ Ok(())+}+#[tokio::test]async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult {let upstream = MockUpstream::start().await?;