Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): keep tool-result media and stored tool continuations across Responses (#713)

eric-liu-nvidia0ccb89e59507
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7429ffd1..55239be30 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -121,6 +121,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Fixed
+- **Cross-format tool results keep image and file content** — an Anthropic
+ `tool_result` carrying image or document blocks reached a Responses target
+ as text only, and an image-only result became an empty `output`. In the other
+ direction a Responses `function_call_output` whose `output` was an array of
+ `input_text`, `input_image`, and `input_file` parts reached an Anthropic
+ target as one JSON string. Both directions now carry typed text, image, and
+ file blocks; plain-text results are unchanged.
+- **Stored Responses tool continuations stay on the selected model** — a
+ `function_call_output` sent with `previous_response_id`, where the matching
+ `function_call` lives in provider state, was decoded as ordinary user text. A
+ route with `classify_trigger = "user_turn"` then judged the turn again and
+ could switch models mid tool loop, and a Responses upstream received a user
+ message instead of the tool output. The output now stays a tool result, and
+ a stored `custom_tool_call_output` keeps its type when re-encoded. An output
+ with no matching call and no `previous_response_id` still degrades to
+ readable user text.
- **Return HTTP 502 for failed Responses generations** — a provider's HTTP 200
response with `status: "failed"` could appear as an empty successful answer.
Switchyard now counts the call as an error and tries another model when the
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 8c6a3c078..07c763c9d 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -3,7 +3,7 @@
//! Buffered codec for OpenAI Responses request and response JSON.
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
use serde_json::{Map, Value, json};
@@ -91,8 +91,17 @@ impl FormatCodec for OpenAiResponsesCodec {
}],
});
}
+ // With `previous_response_id`, the provider holds the earlier turns, so a tool output
+ // may answer a call that is not in this body.
+ let stored_state = body
+ .get("previous_response_id")
+ .and_then(Value::as_str)
+ .is_some_and(|id| !id.is_empty());
+ let mut custom_call_outputs = Vec::new();
let (messages, instructions) = decode_responses_input(
body.get("input").unwrap_or(&Value::String(String::new())),
+ stored_state,
+ &mut custom_call_outputs,
&mut diagnostics,
policy,
)?;
@@ -143,6 +152,10 @@ impl FormatCodec for OpenAiResponsesCodec {
);
crate::codex_namespaces::attach_tool_namespaces(&mut request.extensions, tool_namespaces);
crate::codex_custom_tools::attach_custom_tools(&mut request.extensions, custom_tools);
+ crate::codex_custom_tools::attach_custom_call_outputs(
+ &mut request.extensions,
+ custom_call_outputs,
+ );
crate::codex_custom_tools::attach_additional_tools(
&mut request.extensions,
additional_tools,
@@ -193,6 +206,7 @@ impl FormatCodec for OpenAiResponsesCodec {
_policy,
crate::codex_namespaces::tool_namespaces(&request.extensions),
&crate::codex_custom_tools::custom_tool_names(&request.extensions),
+ crate::codex_custom_tools::custom_call_outputs(&request.extensions),
)?,
);
if let Some(additional) = crate::codex_custom_tools::additional_tools(&request.extensions) {
@@ -404,6 +418,8 @@ impl FormatCodec for OpenAiResponsesCodec {
/// item from flushing pending reasoning or disturbing tool-call grouping.
fn decode_responses_input(
value: &Value,
+ stored_state: bool,
+ custom_call_outputs: &mut Vec<String>,
diagnostics: &mut Vec<TranslationDiagnostic>,
policy: &TranslationPolicy,
) -> Result<(Vec<Message>, Vec<InstructionBlock>)> {
@@ -545,16 +561,51 @@ fn decode_responses_input(
arguments: item.get("arguments").cloned().unwrap_or_else(|| json!({})),
});
}
- Some("function_call_output") | Some("custom_tool_call_output") => {
+ kind @ (Some("function_call_output") | Some("custom_tool_call_output")) => {
let tool_call_id = item
.get("call_id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string();
- let output_text = item.get("output").map(json_string).unwrap_or_default();
+ // A structured output keeps its typed text, image, and file parts;
+ // any other shape rides through as one text block.
+ let content = match item.get("output") {
+ Some(output @ Value::Array(_)) => decode_responses_content(output),
+ output => vec![ContentBlock::Text {
+ text: output.map(json_string).unwrap_or_default(),
+ }],
+ };
+ // An output whose call is not in this body answers a call the provider
+ // holds behind `previous_response_id`; it stays a tool result so routing
+ // sees a tool continuation, not a new user turn. Without stored state
+ // the request is malformed, and the output becomes readable user text.
+ let answers_pending_call = pending_tool_calls
+ .iter()
+ .any(|call| call.id == tool_call_id);
+ if !answers_pending_call && !stored_state {
+ let output_text = text_from_blocks(&content, " ");
+ let message = Message::text(
+ Role::User,
+ format!("Tool result {tool_call_id}: {output_text}"),
+ );
+ push_responses_non_tool_message(
+ &mut messages,
+ &mut pending_tool_calls,
+ &mut pending_tool_outputs,
+ &mut deferred_messages,
+ &mut pending_reasoning,
+ message,
+ );
+ continue;
+ }
+ // The encoder types an output by the call it answers; a stored custom
+ // output has no call in this body, so its type is carried separately.
+ if !answers_pending_call && kind == Some("custom_tool_call_output") {
+ custom_call_outputs.push(tool_call_id.clone());
+ }
pending_tool_outputs.push(ToolResult {
tool_call_id,
- content: vec![ContentBlock::Text { text: output_text }],
+ content,
is_error: None,
});
}
@@ -721,11 +772,6 @@ fn flush_responses_tool_block(
deferred_messages: &mut Vec<Message>,
pending_reasoning: &mut Vec<ContentBlock>,
) {
- let tool_call_ids = pending_tool_calls
- .iter()
- .map(|call| call.id.clone())
- .collect::<HashSet<_>>();
-
if !pending_tool_calls.is_empty() {
let mut content = std::mem::take(pending_reasoning);
content.extend(
@@ -740,19 +786,10 @@ fn flush_responses_tool_block(
}
for output in std::mem::take(pending_tool_outputs) {
- if tool_call_ids.contains(&output.tool_call_id) {
- messages.push(Message {
- role: Role::User,
- content: vec![ContentBlock::ToolResult(output)],
- });
- } else {
- let tool_call_id = output.tool_call_id;
- let output_text = text_from_blocks(&output.content, " ");
- messages.push(Message::text(
- Role::User,
- format!("Tool result {tool_call_id}: {output_text}"),
- ));
- }
+ messages.push(Message {
+ role: Role::User,
+ content: vec![ContentBlock::ToolResult(output)],
+ });
}
messages.append(deferred_messages);
@@ -1135,11 +1172,12 @@ fn encode_responses_input(
policy: &TranslationPolicy,
namespaces: Option<&Map<String, Value>>,
custom_tools: &std::collections::HashSet<String>,
+ mut custom_call_ids: std::collections::HashSet<String>,
) -> Result<Value> {
- // Call ids of freeform tool calls emitted by this pass. A tool result is typed by the call it
- // answers, and Responses history always lists the call before its output, so recording ids
- // as calls are encoded is enough to type the outputs that follow.
- let mut custom_call_ids: std::collections::HashSet<String> = std::collections::HashSet::new();
+ // Call ids of freeform tool calls emitted by this pass, seeded with the ids of stored-state
+ // custom outputs. A tool result is typed by the call it answers, and Responses history
+ // always lists the call before its output, so recording ids as calls are encoded is enough
+ // to type the outputs that follow.
if messages.len() == 1
&& matches!(messages[0].role, Role::User)
&& messages[0].content.len() == 1
@@ -1184,15 +1222,19 @@ fn encode_responses_input(
ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_)
)
}) {
- encoded.extend(content.iter().filter_map(|block| {
- encode_responses_special_input(
+ for block in &content {
+ if let Some(item) = encode_responses_special_input(
block,
namespaces,
&call_names,
custom_tools,
&mut custom_call_ids,
- )
- }));
+ diagnostics,
+ policy,
+ )? {
+ encoded.push(item);
+ }
+ }
continue;
}
let mut visible_content = Vec::new();
@@ -1205,7 +1247,9 @@ fn encode_responses_input(
&call_names,
custom_tools,
&mut custom_call_ids,
- ) {
+ diagnostics,
+ policy,
+ )? {
encoded.push(item);
emitted_special = true;
} else if !matches!(block, ContentBlock::Reasoning { .. }) {
@@ -1292,8 +1336,10 @@ fn encode_responses_special_input(
call_names: &HashMap<&str, &str>,
custom_tools: &std::collections::HashSet<String>,
custom_call_ids: &mut std::collections::HashSet<String>,
-) -> Option<Value> {
- match block {
+ diagnostics: &mut Vec<TranslationDiagnostic>,
+ policy: &TranslationPolicy,
+) -> Result<Option<Value>> {
+ Ok(match block {
ContentBlock::Reasoning {
text,
signature: None,
@@ -1340,7 +1386,7 @@ fn encode_responses_special_input(
"function_call_output"
},
"call_id": result.tool_call_id,
- "output": text_from_blocks(&result.content, " "),
+ "output": encode_responses_tool_output(&result.content, diagnostics, policy)?,
});
// Carry the paired call's name, un-qualified to match the emitted
// function_call, for upstreams that resolve outputs by name.
@@ -1355,7 +1401,7 @@ fn encode_responses_special_input(
Some(item)
}
_ => None,
- }
+ })
}
// Encodes reasoning in the shape accepted for Responses input history. Response
@@ -1485,6 +1531,25 @@ fn encode_responses_content(
Ok(Value::Array(blocks))
}
+// Encodes tool-result content as a `function_call_output.output`: a string when it is
+// text only, otherwise the typed `input_text` / `input_image` / `input_file` parts.
+fn encode_responses_tool_output(
+ content: &[ContentBlock],
+ diagnostics: &mut Vec<TranslationDiagnostic>,
+ policy: &TranslationPolicy,
+) -> Result<Value> {
+ let text_only = content.iter().all(|block| {
+ matches!(
+ block,
+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }
+ )
+ });
+ if text_only {
+ return Ok(Value::String(text_from_blocks(content, " ")));
+ }
+ encode_responses_content(content, diagnostics, policy)
+}
+
fn responses_image_part(source: &ImageSource) -> Option<Value> {
let ImagePayload { url, detail } = image_payload(source)?;
let mut part = json!({"type": "input_image"});
diff --git a/crates/switchyard-translation/src/codex_custom_tools.rs b/crates/switchyard-translation/src/codex_custom_tools.rs
index 7d15d56bc..924e4ea31 100644
--- a/crates/switchyard-translation/src/codex_custom_tools.rs
+++ b/crates/switchyard-translation/src/codex_custom_tools.rs
@@ -46,6 +46,35 @@ pub fn additional_tools(extensions: &ProviderExtensions) -> Option<&Vec<Value>>
.and_then(Value::as_array)
}
+/// Request-extension key holding the `call_id`s of `custom_tool_call_output` items whose
+/// call is not in the request, so the outputs keep their type when re-encoded.
+pub const CUSTOM_CALL_OUTPUTS_KEY: &str = "switchyard_codex_custom_call_outputs";
+
+/// Stores the call ids of stored-state custom tool outputs, when there are any.
+pub fn attach_custom_call_outputs(extensions: &mut ProviderExtensions, call_ids: Vec<String>) {
+ if !call_ids.is_empty() {
+ extensions.fields.insert(
+ CUSTOM_CALL_OUTPUTS_KEY.to_string(),
+ Value::Array(call_ids.into_iter().map(Value::String).collect()),
+ );
+ }
+}
+
+/// Reads the stored-state custom tool output call ids back off a request's extensions.
+pub fn custom_call_outputs(extensions: &ProviderExtensions) -> HashSet<String> {
+ extensions
+ .fields
+ .get(CUSTOM_CALL_OUTPUTS_KEY)
+ .and_then(Value::as_array)
+ .map(|ids| {
+ ids.iter()
+ .filter_map(Value::as_str)
+ .map(str::to_owned)
+ .collect()
+ })
+ .unwrap_or_default()
+}
+
/// Argument name used to carry a custom tool's freeform input through the IR.
pub const INPUT_ARGUMENT: &str = "input";
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 2591fd65b..71c2ff740 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -8,8 +8,8 @@ pub mod common;
use pretty_assertions::assert_eq;
use serde_json::{Value, json};
use switchyard_translation::{
- FormatId, LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,
- prepare_request_for_target, sanitize_anthropic_tool_use_id,
+ ContentBlock, FormatId, LossyConversionPolicy, TranslationEngine, TranslationPolicy,
+ WireFormat, prepare_request_for_target, sanitize_anthropic_tool_use_id,
};
use common::{REASONING_MODEL, normalized_policy, shell_tool_call};
@@ -3014,3 +3014,151 @@ fn anthropic_base64_media_translates_to_responses() -> TestResult {
);
Ok(())
}
+
+// Verifies tool-result image and file content survives translation in both directions between
+// Anthropic Messages and OpenAI Responses.
+#[test]
+fn tool_result_media_survives_anthropic_and_responses_translation() -> TestResult {
+ let engine = TranslationEngine::default();
+ let image = json!({
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}
+ });
+ let document = json!({
+ "type": "document",
+ "title": "report.pdf",
+ "source": {"type": "base64", "media_type": "application/pdf", "data": "ZG9jdW1lbnQ="}
+ });
+ let anthropic_request = |content: Value| {
+ json!({
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 32,
+ "messages": [
+ {"role": "assistant", "content": [
+ {"type": "tool_use", "id": "toolu_1", "name": "capture", "input": {}}
+ ]},
+ {"role": "user", "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_1", "content": content}
+ ]}
+ ]
+ })
+ };
+ let to_responses = |body: &Value| -> TestResult<Value> {
+ let output = engine
+ .translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiResponses,
+ body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+ Ok(output["input"][1]["output"].clone())
+ };
+
+ // Anthropic -> Responses: typed parts, an image-only result, and unchanged plain text.
+ assert_eq!(
+ to_responses(&anthropic_request(json!([
+ {"type": "text", "text": "media attached"},
+ image,
+ document
+ ])))?,
+ json!([
+ {"type": "input_text", "text": "media attached"},
+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="},
+ {"type": "input_file", "file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}
+ ])
+ );
+ assert_eq!(
+ to_responses(&anthropic_request(json!([image])))?,
+ json!([{"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="}])
+ );
+ assert_eq!(
+ to_responses(&anthropic_request(json!("plain")))?,
+ json!("plain")
+ );
+
+ // Responses -> Anthropic: typed blocks instead of one JSON string.
+ let body = json!({
+ "model": "gpt-5.2",
+ "input": [
+ {"type": "function_call", "call_id": "call_1", "name": "capture", "arguments": "{}"},
+ {"type": "function_call_output", "call_id": "call_1", "output": [
+ {"type": "input_text", "text": "media attached"},
+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U=", "detail": "low"},
+ {"type": "input_file", "file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}
+ ]}
+ ]
+ });
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::AnthropicMessages,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+ let result = &output["messages"][1]["content"][0];
+ assert_eq!(result["type"], "tool_result");
+ assert_eq!(result["tool_use_id"], "call_1");
+ assert_eq!(
+ result["content"],
+ json!([
+ {"type": "text", "text": "media attached"},
+ {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}
+ },
+ {
+ "type": "document",
+ "source": {"type": "base64", "data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}
+ }
+ ])
+ );
+ Ok(())
+}
+
+// Verifies Responses tool outputs answering calls held behind `previous_response_id` stay
+// tool results, so a user-turn classifier does not treat them as a new user message, and
+// re-encode with their original item types.
+#[test]
+fn responses_stored_tool_outputs_stay_tool_results() -> TestResult {
+ let engine = TranslationEngine::default();
+ let outputs = json!([
+ {"type": "function_call_output", "call_id": "call_weather", "output": "sunny"},
+ {"type": "custom_tool_call_output", "call_id": "call_shell", "output": "README.md"}
+ ]);
+ let body = json!({
+ "model": "gpt-5.2",
+ "previous_response_id": "resp_1",
+ "input": outputs
+ });
+
+ let request = engine
+ .decode_request(
+ WireFormat::OpenAiResponses,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .request;
+ let tool_call_ids = request
+ .messages
+ .iter()
+ .flat_map(|message| &message.content)
+ .map(|block| match block {
+ ContentBlock::ToolResult(result) => Ok(result.tool_call_id.as_str()),
+ other => Err(format!("expected a tool result, got {other:?}")),
+ })
+ .collect::<Result<Vec<_>, _>>()?;
+ assert_eq!(tool_call_ids, ["call_weather", "call_shell"]);
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiResponses,
+ &body,
+ &normalized_policy(),
+ )?
+ .body;
+ assert_eq!(output["input"], outputs);
+ Ok(())
+}