Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): keep tool-result media and stored tool continuations across Responses (#713)
eric-liu-nvidia0ccb89e59507
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex b7429ffd1..55239be30 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -121,6 +121,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).### Fixed+- **Cross-format tool results keep image and file content** — an Anthropic+ `tool_result` carrying image or document blocks reached a Responses target+ as text only, and an image-only result became an empty `output`. In the other+ direction a Responses `function_call_output` whose `output` was an array of+ `input_text`, `input_image`, and `input_file` parts reached an Anthropic+ target as one JSON string. Both directions now carry typed text, image, and+ file blocks; plain-text results are unchanged.+- **Stored Responses tool continuations stay on the selected model** — a+ `function_call_output` sent with `previous_response_id`, where the matching+ `function_call` lives in provider state, was decoded as ordinary user text. A+ route with `classify_trigger = "user_turn"` then judged the turn again and+ could switch models mid tool loop, and a Responses upstream received a user+ message instead of the tool output. The output now stays a tool result, and+ a stored `custom_tool_call_output` keeps its type when re-encoded. An output+ with no matching call and no `previous_response_id` still degrades to+ readable user text.- **Return HTTP 502 for failed Responses generations** — a provider's HTTP 200response with `status: "failed"` could appear as an empty successful answer.Switchyard now counts the call as an error and tries another model when thediff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 8c6a3c078..07c763c9d 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -3,7 +3,7 @@//! Buffered codec for OpenAI Responses request and response JSON.-use std::collections::{HashMap, HashSet};+use std::collections::HashMap;use serde_json::{Map, Value, json};@@ -91,8 +91,17 @@ impl FormatCodec for OpenAiResponsesCodec {}],});}+ // With `previous_response_id`, the provider holds the earlier turns, so a tool output+ // may answer a call that is not in this body.+ let stored_state = body+ .get("previous_response_id")+ .and_then(Value::as_str)+ .is_some_and(|id| !id.is_empty());+ let mut custom_call_outputs = Vec::new();let (messages, instructions) = decode_responses_input(body.get("input").unwrap_or(&Value::String(String::new())),+ stored_state,+ &mut custom_call_outputs,&mut diagnostics,policy,)?;@@ -143,6 +152,10 @@ impl FormatCodec for OpenAiResponsesCodec {);crate::codex_namespaces::attach_tool_namespaces(&mut request.extensions, tool_namespaces);crate::codex_custom_tools::attach_custom_tools(&mut request.extensions, custom_tools);+ crate::codex_custom_tools::attach_custom_call_outputs(+ &mut request.extensions,+ custom_call_outputs,+ );crate::codex_custom_tools::attach_additional_tools(&mut request.extensions,additional_tools,@@ -193,6 +206,7 @@ impl FormatCodec for OpenAiResponsesCodec {_policy,crate::codex_namespaces::tool_namespaces(&request.extensions),&crate::codex_custom_tools::custom_tool_names(&request.extensions),+ crate::codex_custom_tools::custom_call_outputs(&request.extensions),)?,);if let Some(additional) = crate::codex_custom_tools::additional_tools(&request.extensions) {@@ -404,6 +418,8 @@ impl FormatCodec for OpenAiResponsesCodec {/// item from flushing pending reasoning or disturbing tool-call grouping.fn decode_responses_input(value: &Value,+ stored_state: bool,+ custom_call_outputs: &mut Vec<String>,diagnostics: &mut Vec<TranslationDiagnostic>,policy: &TranslationPolicy,) -> Result<(Vec<Message>, Vec<InstructionBlock>)> {@@ -545,16 +561,51 @@ fn decode_responses_input(arguments: item.get("arguments").cloned().unwrap_or_else(|| json!({})),});}- Some("function_call_output") | Some("custom_tool_call_output") => {+ kind @ (Some("function_call_output") | Some("custom_tool_call_output")) => {let tool_call_id = item.get("call_id").and_then(Value::as_str).unwrap_or_default().to_string();- let output_text = item.get("output").map(json_string).unwrap_or_default();+ // A structured output keeps its typed text, image, and file parts;+ // any other shape rides through as one text block.+ let content = match item.get("output") {+ Some(output @ Value::Array(_)) => decode_responses_content(output),+ output => vec![ContentBlock::Text {+ text: output.map(json_string).unwrap_or_default(),+ }],+ };+ // An output whose call is not in this body answers a call the provider+ // holds behind `previous_response_id`; it stays a tool result so routing+ // sees a tool continuation, not a new user turn. Without stored state+ // the request is malformed, and the output becomes readable user text.+ let answers_pending_call = pending_tool_calls+ .iter()+ .any(|call| call.id == tool_call_id);+ if !answers_pending_call && !stored_state {+ let output_text = text_from_blocks(&content, " ");+ let message = Message::text(+ Role::User,+ format!("Tool result {tool_call_id}: {output_text}"),+ );+ push_responses_non_tool_message(+ &mut messages,+ &mut pending_tool_calls,+ &mut pending_tool_outputs,+ &mut deferred_messages,+ &mut pending_reasoning,+ message,+ );+ continue;+ }+ // The encoder types an output by the call it answers; a stored custom+ // output has no call in this body, so its type is carried separately.+ if !answers_pending_call && kind == Some("custom_tool_call_output") {+ custom_call_outputs.push(tool_call_id.clone());+ }pending_tool_outputs.push(ToolResult {tool_call_id,- content: vec![ContentBlock::Text { text: output_text }],+ content,is_error: None,});}@@ -721,11 +772,6 @@ fn flush_responses_tool_block(deferred_messages: &mut Vec<Message>,pending_reasoning: &mut Vec<ContentBlock>,) {- let tool_call_ids = pending_tool_calls- .iter()- .map(|call| call.id.clone())- .collect::<HashSet<_>>();-if !pending_tool_calls.is_empty() {let mut content = std::mem::take(pending_reasoning);content.extend(@@ -740,19 +786,10 @@ fn flush_responses_tool_block(}for output in std::mem::take(pending_tool_outputs) {- if tool_call_ids.contains(&output.tool_call_id) {- messages.push(Message {- role: Role::User,- content: vec![ContentBlock::ToolResult(output)],- });- } else {- let tool_call_id = output.tool_call_id;- let output_text = text_from_blocks(&output.content, " ");- messages.push(Message::text(- Role::User,- format!("Tool result {tool_call_id}: {output_text}"),- ));- }+ messages.push(Message {+ role: Role::User,+ content: vec![ContentBlock::ToolResult(output)],+ });}messages.append(deferred_messages);@@ -1135,11 +1172,12 @@ fn encode_responses_input(policy: &TranslationPolicy,namespaces: Option<&Map<String, Value>>,custom_tools: &std::collections::HashSet<String>,+ mut custom_call_ids: std::collections::HashSet<String>,) -> Result<Value> {- // Call ids of freeform tool calls emitted by this pass. A tool result is typed by the call it- // answers, and Responses history always lists the call before its output, so recording ids- // as calls are encoded is enough to type the outputs that follow.- let mut custom_call_ids: std::collections::HashSet<String> = std::collections::HashSet::new();+ // Call ids of freeform tool calls emitted by this pass, seeded with the ids of stored-state+ // custom outputs. A tool result is typed by the call it answers, and Responses history+ // always lists the call before its output, so recording ids as calls are encoded is enough+ // to type the outputs that follow.if messages.len() == 1&& matches!(messages[0].role, Role::User)&& messages[0].content.len() == 1@@ -1184,15 +1222,19 @@ fn encode_responses_input(ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_))}) {- encoded.extend(content.iter().filter_map(|block| {- encode_responses_special_input(+ for block in &content {+ if let Some(item) = encode_responses_special_input(block,namespaces,&call_names,custom_tools,&mut custom_call_ids,- )- }));+ diagnostics,+ policy,+ )? {+ encoded.push(item);+ }+ }continue;}let mut visible_content = Vec::new();@@ -1205,7 +1247,9 @@ fn encode_responses_input(&call_names,custom_tools,&mut custom_call_ids,- ) {+ diagnostics,+ policy,+ )? {encoded.push(item);emitted_special = true;} else if !matches!(block, ContentBlock::Reasoning { .. }) {@@ -1292,8 +1336,10 @@ fn encode_responses_special_input(call_names: &HashMap<&str, &str>,custom_tools: &std::collections::HashSet<String>,custom_call_ids: &mut std::collections::HashSet<String>,-) -> Option<Value> {- match block {+ diagnostics: &mut Vec<TranslationDiagnostic>,+ policy: &TranslationPolicy,+) -> Result<Option<Value>> {+ Ok(match block {ContentBlock::Reasoning {text,signature: None,@@ -1340,7 +1386,7 @@ fn encode_responses_special_input("function_call_output"},"call_id": result.tool_call_id,- "output": text_from_blocks(&result.content, " "),+ "output": encode_responses_tool_output(&result.content, diagnostics, policy)?,});// Carry the paired call's name, un-qualified to match the emitted// function_call, for upstreams that resolve outputs by name.@@ -1355,7 +1401,7 @@ fn encode_responses_special_input(Some(item)}_ => None,- }+ })}// Encodes reasoning in the shape accepted for Responses input history. Response@@ -1485,6 +1531,25 @@ fn encode_responses_content(Ok(Value::Array(blocks))}+// Encodes tool-result content as a `function_call_output.output`: a string when it is+// text only, otherwise the typed `input_text` / `input_image` / `input_file` parts.+fn encode_responses_tool_output(+ content: &[ContentBlock],+ diagnostics: &mut Vec<TranslationDiagnostic>,+ policy: &TranslationPolicy,+) -> Result<Value> {+ let text_only = content.iter().all(|block| {+ matches!(+ block,+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }+ )+ });+ if text_only {+ return Ok(Value::String(text_from_blocks(content, " ")));+ }+ encode_responses_content(content, diagnostics, policy)+}+fn responses_image_part(source: &ImageSource) -> Option<Value> {let ImagePayload { url, detail } = image_payload(source)?;let mut part = json!({"type": "input_image"});diff --git a/crates/switchyard-translation/src/codex_custom_tools.rs b/crates/switchyard-translation/src/codex_custom_tools.rsindex 7d15d56bc..924e4ea31 100644--- a/crates/switchyard-translation/src/codex_custom_tools.rs+++ b/crates/switchyard-translation/src/codex_custom_tools.rs@@ -46,6 +46,35 @@ pub fn additional_tools(extensions: &ProviderExtensions) -> Option<&Vec<Value>>.and_then(Value::as_array)}+/// Request-extension key holding the `call_id`s of `custom_tool_call_output` items whose+/// call is not in the request, so the outputs keep their type when re-encoded.+pub const CUSTOM_CALL_OUTPUTS_KEY: &str = "switchyard_codex_custom_call_outputs";++/// Stores the call ids of stored-state custom tool outputs, when there are any.+pub fn attach_custom_call_outputs(extensions: &mut ProviderExtensions, call_ids: Vec<String>) {+ if !call_ids.is_empty() {+ extensions.fields.insert(+ CUSTOM_CALL_OUTPUTS_KEY.to_string(),+ Value::Array(call_ids.into_iter().map(Value::String).collect()),+ );+ }+}++/// Reads the stored-state custom tool output call ids back off a request's extensions.+pub fn custom_call_outputs(extensions: &ProviderExtensions) -> HashSet<String> {+ extensions+ .fields+ .get(CUSTOM_CALL_OUTPUTS_KEY)+ .and_then(Value::as_array)+ .map(|ids| {+ ids.iter()+ .filter_map(Value::as_str)+ .map(str::to_owned)+ .collect()+ })+ .unwrap_or_default()+}+/// Argument name used to carry a custom tool's freeform input through the IR.pub const INPUT_ARGUMENT: &str = "input";diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 2591fd65b..71c2ff740 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -8,8 +8,8 @@ pub mod common;use pretty_assertions::assert_eq;use serde_json::{Value, json};use switchyard_translation::{- FormatId, LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,- prepare_request_for_target, sanitize_anthropic_tool_use_id,+ ContentBlock, FormatId, LossyConversionPolicy, TranslationEngine, TranslationPolicy,+ WireFormat, prepare_request_for_target, sanitize_anthropic_tool_use_id,};use common::{REASONING_MODEL, normalized_policy, shell_tool_call};@@ -3014,3 +3014,151 @@ fn anthropic_base64_media_translates_to_responses() -> TestResult {);Ok(())}++// Verifies tool-result image and file content survives translation in both directions between+// Anthropic Messages and OpenAI Responses.+#[test]+fn tool_result_media_survives_anthropic_and_responses_translation() -> TestResult {+ let engine = TranslationEngine::default();+ let image = json!({+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}+ });+ let document = json!({+ "type": "document",+ "title": "report.pdf",+ "source": {"type": "base64", "media_type": "application/pdf", "data": "ZG9jdW1lbnQ="}+ });+ let anthropic_request = |content: Value| {+ json!({+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 32,+ "messages": [+ {"role": "assistant", "content": [+ {"type": "tool_use", "id": "toolu_1", "name": "capture", "input": {}}+ ]},+ {"role": "user", "content": [+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": content}+ ]}+ ]+ })+ };+ let to_responses = |body: &Value| -> TestResult<Value> {+ let output = engine+ .translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiResponses,+ body,+ &TranslationPolicy::default(),+ )?+ .body;+ Ok(output["input"][1]["output"].clone())+ };++ // Anthropic -> Responses: typed parts, an image-only result, and unchanged plain text.+ assert_eq!(+ to_responses(&anthropic_request(json!([+ {"type": "text", "text": "media attached"},+ image,+ document+ ])))?,+ json!([+ {"type": "input_text", "text": "media attached"},+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="},+ {"type": "input_file", "file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}+ ])+ );+ assert_eq!(+ to_responses(&anthropic_request(json!([image])))?,+ json!([{"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="}])+ );+ assert_eq!(+ to_responses(&anthropic_request(json!("plain")))?,+ json!("plain")+ );++ // Responses -> Anthropic: typed blocks instead of one JSON string.+ let body = json!({+ "model": "gpt-5.2",+ "input": [+ {"type": "function_call", "call_id": "call_1", "name": "capture", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "call_1", "output": [+ {"type": "input_text", "text": "media attached"},+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U=", "detail": "low"},+ {"type": "input_file", "file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}+ ]}+ ]+ });+ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::AnthropicMessages,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;+ let result = &output["messages"][1]["content"][0];+ assert_eq!(result["type"], "tool_result");+ assert_eq!(result["tool_use_id"], "call_1");+ assert_eq!(+ result["content"],+ json!([+ {"type": "text", "text": "media attached"},+ {+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}+ },+ {+ "type": "document",+ "source": {"type": "base64", "data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}+ }+ ])+ );+ Ok(())+}++// Verifies Responses tool outputs answering calls held behind `previous_response_id` stay+// tool results, so a user-turn classifier does not treat them as a new user message, and+// re-encode with their original item types.+#[test]+fn responses_stored_tool_outputs_stay_tool_results() -> TestResult {+ let engine = TranslationEngine::default();+ let outputs = json!([+ {"type": "function_call_output", "call_id": "call_weather", "output": "sunny"},+ {"type": "custom_tool_call_output", "call_id": "call_shell", "output": "README.md"}+ ]);+ let body = json!({+ "model": "gpt-5.2",+ "previous_response_id": "resp_1",+ "input": outputs+ });++ let request = engine+ .decode_request(+ WireFormat::OpenAiResponses,+ &body,+ &TranslationPolicy::default(),+ )?+ .request;+ let tool_call_ids = request+ .messages+ .iter()+ .flat_map(|message| &message.content)+ .map(|block| match block {+ ContentBlock::ToolResult(result) => Ok(result.tool_call_id.as_str()),+ other => Err(format!("expected a tool result, got {other:?}")),+ })+ .collect::<Result<Vec<_>, _>>()?;+ assert_eq!(tool_call_ids, ["call_weather", "call_shell"]);++ let output = engine+ .translate_request(+ WireFormat::OpenAiResponses,+ WireFormat::OpenAiResponses,+ &body,+ &normalized_policy(),+ )?+ .body;+ assert_eq!(output["input"], outputs);+ Ok(())+}