Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve multimodal tool results (#389)

Elias Shiehc68946658ad8
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index 7a8e79c9e..fe5573039 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -554,20 +554,18 @@ fn decode_anthropic_content_block(
content: decode_tool_result_content(block.get("content").unwrap_or(&Value::Null)),
is_error: block.get("is_error").and_then(Value::as_bool),
})],
- Some("image") => {
- let source = block
- .get("source")
- .cloned()
- .map(ImageSource::Raw)
- .unwrap_or_else(|| ImageSource::Raw(Value::Object(block.clone())));
- vec![ContentBlock::Image { source }]
- }
+ Some("image") => vec![ContentBlock::Image {
+ source: ImageSource::Raw(Value::Object(block.clone())),
+ }],
Some("input_image") | Some("image_url") => decode_image_source(block)
.map(|source| vec![ContentBlock::Image { source }])
.unwrap_or_default(),
Some("input_file") | Some("file") => vec![ContentBlock::File {
source: decode_file_source(block),
}],
+ Some("document") => vec![ContentBlock::File {
+ source: decode_anthropic_file_source(block),
+ }],
_ => vec![ContentBlock::Unknown {
provider: WireFormat::AnthropicMessages.into(),
raw: Value::Object(block.clone()),
@@ -575,30 +573,36 @@ fn decode_anthropic_content_block(
})
}
-// Converts Anthropic tool-result content into text-like IR blocks.
+// Preserves supported Anthropic tool-result blocks in the neutral IR.
fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {
match value {
Value::String(text) => vec![ContentBlock::Text { text: text.clone() }],
Value::Array(blocks) => {
- let mut text = Vec::new();
+ let mut content = Vec::new();
for block in blocks {
if let Some(block) = block.as_object() {
- if block.get("type").and_then(Value::as_str) == Some("text") {
- text.push(
- block
+ match block.get("type").and_then(Value::as_str) {
+ Some("text") => content.push(ContentBlock::Text {
+ text: block
.get("text")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
- );
- } else {
- text.push(json_string(&Value::Object(block.clone())));
+ }),
+ Some("image") => content.push(ContentBlock::Image {
+ source: ImageSource::Raw(Value::Object(block.clone())),
+ }),
+ Some("document") => content.push(ContentBlock::File {
+ source: decode_anthropic_file_source(block),
+ }),
+ _ => content.push(ContentBlock::Unknown {
+ provider: WireFormat::AnthropicMessages.into(),
+ raw: Value::Object(block.clone()),
+ }),
}
}
}
- vec![ContentBlock::Text {
- text: text.join(" "),
- }]
+ content
}
Value::Null => vec![ContentBlock::Text {
text: String::new(),
@@ -609,6 +613,11 @@ fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {
}
}
+// Keeps Anthropic document fields together for same-format re-encoding.
+fn decode_anthropic_file_source(block: &Map<String, Value>) -> FileSource {
+ FileSource::Raw(Value::Object(block.clone()))
+}
+
// Decodes Anthropic tool definitions into normalized tool definitions.
fn decode_anthropic_tools(value: Option<&Value>) -> Vec<ToolDefinition> {
value
@@ -815,11 +824,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
"name": call.name,
"input": anthropic_tool_input(&call.arguments),
})],
- ContentBlock::ToolResult(result) => vec![json!({
- "type": "tool_result",
- "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),
- "content": text_from_blocks(&result.content, " "),
- })],
+ ContentBlock::ToolResult(result) => {
+ let content = if result.content.iter().all(|block| {
+ matches!(
+ block,
+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }
+ )
+ }) {
+ Value::String(text_from_blocks(&result.content, " "))
+ } else {
+ Value::Array(
+ result
+ .content
+ .iter()
+ .flat_map(encode_one_anthropic_tool_result_block)
+ .collect(),
+ )
+ };
+ vec![json!({
+ "type": "tool_result",
+ "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),
+ "content": content,
+ })]
+ }
ContentBlock::Image { source } => vec![match source {
ImageSource::Url { url, .. } => {
json!({"type": "image", "source": {"type": "url", "url": url}})
@@ -880,6 +907,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
}
}
+// Encodes only provider-safe block shapes inside Anthropic tool results.
+fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {
+ match block {
+ ContentBlock::Text { .. }
+ | ContentBlock::Refusal { .. }
+ | ContentBlock::Image { .. }
+ | ContentBlock::File { .. } => encode_one_anthropic_block(block),
+ ContentBlock::Unknown { provider, raw }
+ if provider.as_str() == WireFormat::AnthropicMessages.as_str() =>
+ {
+ vec![raw.clone()]
+ }
+ ContentBlock::Unknown { raw, .. } => {
+ vec![json!({"type": "text", "text": json_string(raw)})]
+ }
+ ContentBlock::Reasoning { .. }
+ | ContentBlock::Audio { .. }
+ | ContentBlock::Video { .. }
+ | ContentBlock::ToolCall(_)
+ | ContentBlock::ToolResult(_) => Vec::new(),
+ }
+}
+
// Anthropic requires `tool_use.input` to be object-shaped, while OpenAI and
// Responses commonly carry function-call arguments as JSON strings.
fn anthropic_tool_input(arguments: &Value) -> Value {
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index dbe77ca2a..32d0de1a2 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -700,18 +700,30 @@ fn encode_message_with_tool_results_to_openai(
for block in &message.content {
if let ContentBlock::ToolResult(result) = block {
- push_pending_openai_message(
- &mut out,
- message.role,
- &mut pending_content,
- diagnostics,
- policy,
- )?;
out.push(json!({
"role": "tool",
"tool_call_id": result.tool_call_id,
"content": text_from_blocks(&result.content, " "),
}));
+ let non_text = result
+ .content
+ .iter()
+ .filter(|block| {
+ !matches!(
+ block,
+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }
+ )
+ })
+ .cloned()
+ .collect::<Vec<_>>();
+ if !non_text.is_empty() {
+ push_lossy(
+ diagnostics,
+ policy,
+ "OpenAI Chat tool messages only support text; non-text tool-result content was moved to a user message",
+ )?;
+ pending_content.extend(non_text);
+ }
} else {
pending_content.push(block.clone());
}
@@ -956,6 +968,18 @@ fn openai_image_part(source: &ImageSource) -> Option<Value> {
// Recognizes common raw image shapes emitted by Anthropic and Responses.
fn openai_raw_image_part(raw: &Value) -> Option<Value> {
let object = raw.as_object()?;
+ let object = if object.get("type").and_then(Value::as_str) == Some("image") {
+ let source = object.get("source").and_then(Value::as_object)?;
+ if !matches!(
+ source.get("type").and_then(Value::as_str),
+ Some("base64" | "url")
+ ) {
+ return None;
+ }
+ source
+ } else {
+ object
+ };
if let Some(url) = object.get("url").and_then(Value::as_str) {
return Some(json!({"type": "image_url", "image_url": {"url": url}}));
}
@@ -999,8 +1023,26 @@ fn openai_file_part(source: &FileSource) -> Option<Value> {
}
Some(json!({"type": "file", "file": file}))
}
- FileSource::Raw(_) => None,
+ FileSource::Raw(raw) => openai_raw_file_part(raw),
+ }
+}
+
+// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.
+fn openai_raw_file_part(raw: &Value) -> Option<Value> {
+ let block = raw.as_object()?;
+ if block.get("type").and_then(Value::as_str) != Some("document") {
+ return None;
+ }
+ let source = block.get("source").and_then(Value::as_object)?;
+ if source.get("type").and_then(Value::as_str) != Some("base64") {
+ return None;
+ }
+ let data = source.get("data").and_then(Value::as_str)?;
+ let mut file = json!({"file_data": data});
+ if let Some(title) = block.get("title").and_then(Value::as_str) {
+ file["filename"] = Value::String(title.to_string());
}
+ Some(json!({"type": "file", "file": file}))
}
// Converts file sources to deterministic text fallback content.
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 71c765d01..e14d38e9b 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -5,7 +5,9 @@
use pretty_assertions::assert_eq;
use serde_json::{Value, json};
-use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};
+use switchyard_translation::{
+ LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,
+};
type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
@@ -310,6 +312,211 @@ fn anthropic_tool_result_followup_text_splits_to_openai_messages() -> TestResult
Ok(())
}
+// Verifies Anthropic multimodal blocks retain provider fields inside tool results.
+#[test]
+fn anthropic_tool_result_multimodal_blocks_round_trip_complete() -> TestResult {
+ let engine = TranslationEngine::default();
+ let policy = TranslationPolicy {
+ preservation: switchyard_translation::PreservationPolicy::Disabled,
+ ..TranslationPolicy::default()
+ };
+ let document = json!({
+ "type": "document",
+ "title": "report.pdf",
+ "context": "Quarterly results",
+ "citations": {"enabled": true},
+ "source": {
+ "type": "base64",
+ "media_type": "application/pdf",
+ "data": "ZG9jdW1lbnQ="
+ }
+ });
+ let image = json!({
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "aW1hZ2U="
+ }
+ });
+ let body = json!({
+ "model": "claude-sonnet-4-20250514",
+ "messages": [{
+ "role": "user",
+ "content": [{
+ "type": "tool_result",
+ "tool_use_id": "toolu_document",
+ "content": [
+ {"type": "text", "text": "content ready"},
+ image.clone(),
+ document.clone()
+ ]
+ }]
+ }],
+ "max_tokens": 1024
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::AnthropicMessages,
+ &body,
+ &policy,
+ )?
+ .body;
+
+ assert_eq!(
+ output["messages"][0]["content"][0]["content"],
+ json!([
+ {"type": "text", "text": "content ready"},
+ image,
+ document
+ ])
+ );
+ Ok(())
+}
+
+// Verifies provider-managed Anthropic file IDs are not reused as OpenAI file IDs.
+#[test]
+fn anthropic_tool_result_file_id_does_not_become_openai_file_id() -> TestResult {
+ let engine = TranslationEngine::default();
+ let document = json!({
+ "type": "document",
+ "source": {
+ "type": "file",
+ "file_id": "file_anthropic_123"
+ }
+ });
+ let body = json!({
+ "model": "claude-sonnet-4-20250514",
+ "messages": [{
+ "role": "user",
+ "content": [{
+ "type": "tool_result",
+ "tool_use_id": "toolu_document",
+ "content": [document.clone()]
+ }]
+ }],
+ "max_tokens": 1024
+ });
+
+ let translated = engine.translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?;
+
+ assert_eq!(translated.body["messages"][1]["content"][0]["type"], "text");
+ let recovered: Value = serde_json::from_str(
+ translated.body["messages"][1]["content"][0]["text"]
+ .as_str()
+ .ok_or("file fallback should be text")?,
+ )?;
+ assert_eq!(recovered, document);
+ assert!(translated.diagnostics.iter().any(|diagnostic| {
+ diagnostic.message == "OpenAI Chat codec could not map file content"
+ }));
+ Ok(())
+}
+
+// Verifies parallel tool results preserve ordering and obey strict conversion policy.
+#[test]
+fn anthropic_parallel_multimodal_tool_results_preserve_order_and_policy() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "claude-sonnet-4-20250514",
+ "messages": [{
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_image",
+ "content": [
+ {"type": "text", "text": "image ready"},
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "aW1hZ2U="
+ }
+ }
+ ]
+ },
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_document",
+ "content": [
+ {"type": "text", "text": "document ready"},
+ {
+ "type": "document",
+ "title": "report.pdf",
+ "source": {
+ "type": "base64",
+ "media_type": "application/pdf",
+ "data": "ZG9jdW1lbnQ="
+ }
+ }
+ ]
+ }
+ ]
+ }],
+ "max_tokens": 1024
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(
+ output["messages"],
+ json!([
+ {"role": "tool", "tool_call_id": "toolu_image", "content": "image ready"},
+ {
+ "role": "tool",
+ "tool_call_id": "toolu_document",
+ "content": "document ready"
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="}
+ },
+ {
+ "type": "file",
+ "file": {"file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}
+ }
+ ]
+ }
+ ])
+ );
+ let policy = TranslationPolicy {
+ lossy_conversion_policy: LossyConversionPolicy::Reject,
+ ..TranslationPolicy::default()
+ };
+
+ let error = match engine.translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiChat,
+ &body,
+ &policy,
+ ) {
+ Ok(_) => panic!("multimodal tool result should be rejected by strict policy"),
+ Err(error) => error,
+ };
+
+ assert_eq!(error.kind(), "LossyConversion");
+ Ok(())
+}
+
// Verifies structured Anthropic system blocks remain separated in OpenAI system text.
#[test]
fn anthropic_structured_system_blocks_preserve_boundaries_for_openai_chat() -> TestResult {