Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(stage): recognize failed Responses patch and shell outputs (#789)
Graham Kingbfcd023cb679
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rsindex ba469cfdf..3cecf06a3 100644--- a/crates/libsy/src/algorithms/util/tool_signals.rs+++ b/crates/libsy/src/algorithms/util/tool_signals.rs@@ -17,7 +17,7 @@ use std::path::Path;use async_trait::async_trait;use serde::Deserialize;use serde_json::Value;-use switchyard_protocol::{ContentBlock, Request, Role};+use switchyard_protocol::{ContentBlock, Request, Role, WireFormat};use crate::{LibsyError, Result};@@ -704,9 +704,7 @@ fn extract_tool_signals_with_window_and_semantics(recent_window: usize,semantics: &ToolSemantics,) -> ToolSignals {- // Read the decoded conversation, not the raw body: every inbound format lands- // in the same shape here, so the signals do not depend on knowing which one it- // arrived as.+ // Read the decoded conversation, including preserved built-in tool outputs.let messages = &request.llm_request.messages;let mut tool_texts: Vec<(String, bool)> = Vec::new();let mut tool_calls: Vec<ObservedToolCall> = Vec::new();@@ -741,6 +739,60 @@ fn extract_tool_signals_with_window_and_semantics(tool_texts.push((text, is_error));}}+ // Built-in tool history stays opaque so it can be replayed unchanged.+ ContentBlock::Unknown { provider, raw }+ if provider.as_str() == WireFormat::OpenAiResponses.as_str()+ && raw.get("type").and_then(Value::as_str)+ == Some("apply_patch_call_output") =>+ {+ tool_result_count += 1;+ let text = raw+ .get("output")+ .and_then(Value::as_str)+ .unwrap_or_default();+ let is_error = raw.get("status").and_then(Value::as_str) == Some("failed");+ tool_texts.push((text.to_owned(), is_error));+ }+ ContentBlock::Unknown { provider, raw }+ if provider.as_str() == WireFormat::OpenAiResponses.as_str()+ && raw.get("type").and_then(Value::as_str) == Some("shell_call_output") =>+ {+ tool_result_count += 1;+ let mut texts = Vec::new();+ let mut is_error = false;+ if let Some(outputs) = raw.get("output").and_then(Value::as_array) {+ for output in outputs {+ for field in ["stdout", "stderr"] {+ if let Some(text) = output.get(field).and_then(Value::as_str)+ && !text.is_empty()+ {+ texts.push(text);+ }+ }+ if let Some(outcome) = output.get("outcome") {+ is_error |= match outcome.get("type").and_then(Value::as_str) {+ Some("timeout") => true,+ Some("exit")+ if outcome+ .get("exit_code")+ .and_then(Value::as_i64)+ .is_some_and(|code| code != 0) =>+ {+ // Keep plain nonzero exits SOFT, including empty output.+ texts.push("returned non-zero");+ output+ .get("stderr")+ .and_then(Value::as_str)+ .is_some_and(|text| !text.trim().is_empty())+ }+ _ => false,+ };+ }+ }+ }+ let text = texts.join("\n");+ tool_texts.push((text, is_error));+ }// Compaction is detected anywhere in the conversation: the summary// stays in the prefix on every later turn, so this self-latches// once it fires.@@ -1511,6 +1563,128 @@ mod tests {assert_eq!(sig.write_count, 1);}+ #[test]+ fn responses_builtin_tool_failures_escalate() {+ use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier};++ let mut cases = Vec::new();+ for (status, output) in [+ (+ "failed",+ "Synthetic dependency unavailable; retry with the recovery path.",+ ),+ (+ "completed",+ "Synthetic dependency unavailable; retry with the recovery path.",+ ),+ ("failed", ""),+ ] {+ cases.push((+ json!({+ "type": "apply_patch_call_output",+ "status": status,+ "output": output,+ }),+ if status == "failed" { HARD } else { 0.0 },+ ));+ }+ for (outcome, stdout, stderr, severity) in [+ (json!({"type": "exit", "exit_code": 1}), "", "", SOFT),+ (json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT),+ (+ json!({"type": "exit", "exit_code": 1}),+ "",+ "command failed",+ HARD,+ ),+ (json!({"type": "timeout"}), "", "", HARD),+ (json!({"type": "exit", "exit_code": 0}), "done", "", 0.0),+ (+ json!({"type": "exit", "exit_code": 0}),+ "Traceback (most recent call last):",+ "",+ HARD,+ ),+ (+ json!({"type": "exit", "exit_code": 0}),+ "",+ "Traceback (most recent call last):",+ HARD,+ ),+ ] {+ cases.push((+ json!({+ "type": "shell_call_output",+ "output": [+ {"stdout": stdout, "stderr": stderr, "outcome": outcome},+ {"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}+ ],+ }),+ severity,+ ));+ }+ for (raw, severity) in cases {+ let is_error = severity >= HARD;+ let mut request = with_messages(+ ["call_1", "call_2"]+ .into_iter()+ .map(|call_id| {+ let mut raw = raw.clone();+ raw["call_id"] = json!(call_id);+ Message {+ role: Role::User,+ content: vec![ContentBlock::Unknown {+ provider: WireFormat::OpenAiResponses.into(),+ raw,+ }],+ }+ })+ .collect(),+ );+ let signal = ToolSignals::from_request(&request, Some(3));+ assert_eq!(signal.severity, severity, "{raw}");+ assert_eq!(signal.repeated_failure, is_error, "{raw}");+ assert_eq!(signal.tool_result_count, 2);+ assert_eq!(+ matches!(+ pick_tier(&signal, PickerMode::EfficientFirst, 0.5),+ PickOutcome::Resolved {+ tier: Tier::Capable,+ ..+ }+ ),+ is_error,+ "{raw}"+ );++ let mut success = match raw["type"].as_str() {+ Some("apply_patch_call_output") => json!({+ "type": "apply_patch_call_output", "status": "completed", "output": ""+ }),+ Some("shell_call_output") => json!({+ "type": "shell_call_output",+ "output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}]+ }),+ _ => unreachable!(),+ };+ for index in 0..3 {+ success["call_id"] = json!(format!("success_{index}"));+ request.llm_request.messages.push(Message {+ role: Role::User,+ content: vec![ContentBlock::Unknown {+ provider: WireFormat::OpenAiResponses.into(),+ raw: success.clone(),+ }],+ });+ }+ let recovered = ToolSignals::from_request(&request, Some(3));+ assert_eq!(recovered.severity, 0.0, "{raw}");+ assert!(!recovered.repeated_failure, "{raw}");+ assert!(recovered.no_error_streak >= 3, "{raw}");+ assert_eq!(recovered.tool_result_count, 5);+ }+ }+#[test]fn conversation_counts_are_per_block_and_role_aware() {// A batched user message (Anthropic shape) contributes one count per