Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(stage-router): recognize structured tool failures (#949)
ryan-lempka2bd647e8b906
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rsindex e5b0fc038..eeed831be 100644--- a/crates/libsy/src/algorithms/util/tool_signals.rs+++ b/crates/libsy/src/algorithms/util/tool_signals.rs@@ -746,6 +746,7 @@ fn extract_tool_signals_with_window_and_semantics(let mut tool_calls: Vec<ObservedToolCall> = Vec::new();// IDs whose latest call is a retrieval tool.let mut retrieval_calls: HashSet<&str> = HashSet::new();+ let mut shell_read_calls: HashSet<&str> = HashSet::new();let mut compacted = false;let mut tool_result_count = 0usize;let mut assistant_turn_count = 0usize;@@ -784,6 +785,19 @@ fn extract_tool_signals_with_window_and_semantics(} else {retrieval_calls.remove(call.id.as_str());}+ if BASH_TOOL_NAMES.contains(&name.to_lowercase().as_str())+ && command.as_deref().is_some_and(|command| {+ !command.trim().is_empty()+ && shell_segments(command).all(|segment| {+ classify_tool_call(name, Some(segment))+ == ToolSemantic::Observe+ })+ })+ {+ shell_read_calls.insert(call.id.as_str());+ } else {+ shell_read_calls.remove(call.id.as_str());+ }}tool_calls.push(ObservedToolCall {name: call.name.clone(),@@ -801,8 +815,18 @@ fn extract_tool_signals_with_window_and_semantics(.collect::<Vec<_>>().join("\n");let is_error = result.is_error == Some(true);- let is_retrieval_result =- !is_error && retrieval_calls.contains(result.tool_call_id.as_str());+ // Shell reads can return bare JSON from a file. Hermes wraps+ // terminal output with output and exit_code fields.+ let is_shell_read_json = shell_read_calls+ .contains(result.tool_call_id.as_str())+ && serde_json::from_str::<Value>(&text).is_ok_and(|value| {+ value.is_object()+ && !(value["output"].is_string()+ && value.get("exit_code").is_some())+ });+ let is_retrieval_result = !is_error+ && (retrieval_calls.contains(result.tool_call_id.as_str())+ || is_shell_read_json);// An explicit failure remains a signal even without text.if !text.is_empty() || is_error {// Read and search results show file contents, not the outcome@@ -1095,7 +1119,7 @@ fn content_to_text(content: Option<&Value>) -> Option<String> {}}-/// Match `text` against the error pattern table.+/// Match tool text and structured result fields against error patterns.////// Returns `(max_severity, matched_pattern_names)`.pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {@@ -1108,11 +1132,25 @@ pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {severity = severity.max(*sev);}}- if has_nonzero_exit_status(&lower) && !patterns.iter().any(|p| p == "exit_nonzero") {+ // Hermes can append a loop warning after the JSON result.+ let result = serde_json::Deserializer::from_str(text)+ .into_iter::<Value>()+ .next()+ .and_then(|result| result.ok())+ .unwrap_or_default();+ let nonzero_exit = result["exit_code"].as_i64().is_some_and(|code| code != 0);+ let tool_error = result["success"].as_bool() == Some(false)+ || result["error"]+ .as_str()+ .is_some_and(|error| !error.trim().is_empty());+ if (nonzero_exit || has_nonzero_exit_status(&lower))+ && !patterns.iter().any(|p| p == "exit_nonzero")+ {patterns.push("exit_nonzero".to_string());severity = severity.max(SOFT);}for (name, matched) in [+ ("tool_error", tool_error),("compile_error", has_compiler_diagnostic(&lower)),("runtime_exception", has_runtime_exception(&lower)),("runtime_panic", has_runtime_panic(&lower)),@@ -1403,6 +1441,34 @@ mod tests {assert!(patterns.is_empty());}+ #[test]+ fn structured_tool_failures_affect_recovery_signals() {+ for (text, severity) in [+ (r#"{"output":"","exit_code":7,"error":null}"#, SOFT),+ (r#"{"output":"","exit_code":-1,"error":null}"#, SOFT),+ (r#"{"success":false,"error":"No matching text"}"#, HARD),+ (r#"{"success":false,"error":null}"#, HARD),+ (+ r#"{"error":"Overwrite refused","stale_write_blocked":true}"#,+ HARD,+ ),+ (r#"{"output":"out of memory","exit_code":1}"#, CRITICAL),+ (r#"{"output":"done","exit_code":0,"error":null}"#, 0.0),+ (r#"{"success":true,"error":" "}"#, 0.0),+ (r#"{"output":"running","exit_code":null,"error":null}"#, 0.0),+ (r#"{"output":{"success":false},"exit_code":0}"#, 0.0),+ ] {+ let warned = format!("{text}\n\n[Tool loop warning: repeated identical call]");+ let request = with_messages(vec![tr("5 passed in 0.12s"), tr(text), tr(&warned)]);+ let signal = ToolSignals::from_request(&request, None);+ let clean = severity == 0.0;+ assert_eq!(signal.severity, severity, "{text}");+ assert_eq!(signal.no_error_streak, if clean { 3 } else { 0 }, "{text}");+ assert_eq!(signal.repeated_failure, severity >= HARD, "{text}");+ assert_eq!(signal.tests_passed, clean, "{text}");+ }+ }+#[test]fn traceback_is_hard() {let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError");@@ -1637,7 +1703,10 @@ mod tests {call("a", "Bash", json!({"command": "pytest"})),result("a", "Traceback (most recent call last):\nValueError"),call("b", "Read", json!({"file_path": "notes.md"})),- result("b", "the worker ran out of memory"),+ result(+ "b",+ r#"{"success":false,"error":"out of memory","exit_code":7}"#,+ ),call("c", "Grep", json!({"pattern": "passed"})),result("c", "CHANGELOG.md: all tests passed"),]),@@ -1647,6 +1716,30 @@ mod tests {assert_eq!(signal.severity, HARD);assert!(!signal.tests_passed);assert_eq!(signal.tool_result_count, 3);++ for (command, text, severity) in [+ ("cat config.json", r#"{"error":"fixture data"}"#, 0.0),+ (+ "cat missing.json",+ r#"{"output":"","exit_code":7,"error":null}"#,+ SOFT,+ ),+ (+ "cat config.json && python check.py",+ r#"{"error":"check rejected"}"#,+ HARD,+ ),+ ] {+ let request = with_messages(vec![+ call("shell", "Bash", json!({"command": command})),+ result("shell", text),+ ]);+ assert_eq!(+ ToolSignals::from_request(&request, None).severity,+ severity,+ "{text}"+ );+ }}#[test]