Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): preserve multimodal tool results (#389)
Elias Shiehc68946658ad8
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rsindex 7a8e79c9e..fe5573039 100644--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs@@ -554,20 +554,18 @@ fn decode_anthropic_content_block(content: decode_tool_result_content(block.get("content").unwrap_or(&Value::Null)),is_error: block.get("is_error").and_then(Value::as_bool),})],- Some("image") => {- let source = block- .get("source")- .cloned()- .map(ImageSource::Raw)- .unwrap_or_else(|| ImageSource::Raw(Value::Object(block.clone())));- vec![ContentBlock::Image { source }]- }+ Some("image") => vec![ContentBlock::Image {+ source: ImageSource::Raw(Value::Object(block.clone())),+ }],Some("input_image") | Some("image_url") => decode_image_source(block).map(|source| vec![ContentBlock::Image { source }]).unwrap_or_default(),Some("input_file") | Some("file") => vec![ContentBlock::File {source: decode_file_source(block),}],+ Some("document") => vec![ContentBlock::File {+ source: decode_anthropic_file_source(block),+ }],_ => vec![ContentBlock::Unknown {provider: WireFormat::AnthropicMessages.into(),raw: Value::Object(block.clone()),@@ -575,30 +573,36 @@ fn decode_anthropic_content_block(})}-// Converts Anthropic tool-result content into text-like IR blocks.+// Preserves supported Anthropic tool-result blocks in the neutral IR.fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {match value {Value::String(text) => vec![ContentBlock::Text { text: text.clone() }],Value::Array(blocks) => {- let mut text = Vec::new();+ let mut content = Vec::new();for block in blocks {if let Some(block) = block.as_object() {- if block.get("type").and_then(Value::as_str) == Some("text") {- text.push(- block+ match block.get("type").and_then(Value::as_str) {+ Some("text") => content.push(ContentBlock::Text {+ text: block.get("text").and_then(Value::as_str).unwrap_or_default().to_string(),- );- } else {- text.push(json_string(&Value::Object(block.clone())));+ }),+ Some("image") => content.push(ContentBlock::Image {+ source: ImageSource::Raw(Value::Object(block.clone())),+ }),+ Some("document") => content.push(ContentBlock::File {+ source: decode_anthropic_file_source(block),+ }),+ _ => content.push(ContentBlock::Unknown {+ provider: WireFormat::AnthropicMessages.into(),+ raw: Value::Object(block.clone()),+ }),}}}- vec![ContentBlock::Text {- text: text.join(" "),- }]+ content}Value::Null => vec![ContentBlock::Text {text: String::new(),@@ -609,6 +613,11 @@ fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {}}+// Keeps Anthropic document fields together for same-format re-encoding.+fn decode_anthropic_file_source(block: &Map<String, Value>) -> FileSource {+ FileSource::Raw(Value::Object(block.clone()))+}+// Decodes Anthropic tool definitions into normalized tool definitions.fn decode_anthropic_tools(value: Option<&Value>) -> Vec<ToolDefinition> {value@@ -815,11 +824,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {"name": call.name,"input": anthropic_tool_input(&call.arguments),})],- ContentBlock::ToolResult(result) => vec![json!({- "type": "tool_result",- "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),- "content": text_from_blocks(&result.content, " "),- })],+ ContentBlock::ToolResult(result) => {+ let content = if result.content.iter().all(|block| {+ matches!(+ block,+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }+ )+ }) {+ Value::String(text_from_blocks(&result.content, " "))+ } else {+ Value::Array(+ result+ .content+ .iter()+ .flat_map(encode_one_anthropic_tool_result_block)+ .collect(),+ )+ };+ vec![json!({+ "type": "tool_result",+ "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),+ "content": content,+ })]+ }ContentBlock::Image { source } => vec![match source {ImageSource::Url { url, .. } => {json!({"type": "image", "source": {"type": "url", "url": url}})@@ -880,6 +907,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {}}+// Encodes only provider-safe block shapes inside Anthropic tool results.+fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {+ match block {+ ContentBlock::Text { .. }+ | ContentBlock::Refusal { .. }+ | ContentBlock::Image { .. }+ | ContentBlock::File { .. } => encode_one_anthropic_block(block),+ ContentBlock::Unknown { provider, raw }+ if provider.as_str() == WireFormat::AnthropicMessages.as_str() =>+ {+ vec![raw.clone()]+ }+ ContentBlock::Unknown { raw, .. } => {+ vec![json!({"type": "text", "text": json_string(raw)})]+ }+ ContentBlock::Reasoning { .. }+ | ContentBlock::Audio { .. }+ | ContentBlock::Video { .. }+ | ContentBlock::ToolCall(_)+ | ContentBlock::ToolResult(_) => Vec::new(),+ }+}+// Anthropic requires `tool_use.input` to be object-shaped, while OpenAI and// Responses commonly carry function-call arguments as JSON strings.fn anthropic_tool_input(arguments: &Value) -> Value {diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex dbe77ca2a..32d0de1a2 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -700,18 +700,30 @@ fn encode_message_with_tool_results_to_openai(for block in &message.content {if let ContentBlock::ToolResult(result) = block {- push_pending_openai_message(- &mut out,- message.role,- &mut pending_content,- diagnostics,- policy,- )?;out.push(json!({"role": "tool","tool_call_id": result.tool_call_id,"content": text_from_blocks(&result.content, " "),}));+ let non_text = result+ .content+ .iter()+ .filter(|block| {+ !matches!(+ block,+ ContentBlock::Text { .. } | ContentBlock::Refusal { .. }+ )+ })+ .cloned()+ .collect::<Vec<_>>();+ if !non_text.is_empty() {+ push_lossy(+ diagnostics,+ policy,+ "OpenAI Chat tool messages only support text; non-text tool-result content was moved to a user message",+ )?;+ pending_content.extend(non_text);+ }} else {pending_content.push(block.clone());}@@ -956,6 +968,18 @@ fn openai_image_part(source: &ImageSource) -> Option<Value> {// Recognizes common raw image shapes emitted by Anthropic and Responses.fn openai_raw_image_part(raw: &Value) -> Option<Value> {let object = raw.as_object()?;+ let object = if object.get("type").and_then(Value::as_str) == Some("image") {+ let source = object.get("source").and_then(Value::as_object)?;+ if !matches!(+ source.get("type").and_then(Value::as_str),+ Some("base64" | "url")+ ) {+ return None;+ }+ source+ } else {+ object+ };if let Some(url) = object.get("url").and_then(Value::as_str) {return Some(json!({"type": "image_url", "image_url": {"url": url}}));}@@ -999,8 +1023,26 @@ fn openai_file_part(source: &FileSource) -> Option<Value> {}Some(json!({"type": "file", "file": file}))}- FileSource::Raw(_) => None,+ FileSource::Raw(raw) => openai_raw_file_part(raw),+ }+}++// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.+fn openai_raw_file_part(raw: &Value) -> Option<Value> {+ let block = raw.as_object()?;+ if block.get("type").and_then(Value::as_str) != Some("document") {+ return None;+ }+ let source = block.get("source").and_then(Value::as_object)?;+ if source.get("type").and_then(Value::as_str) != Some("base64") {+ return None;+ }+ let data = source.get("data").and_then(Value::as_str)?;+ let mut file = json!({"file_data": data});+ if let Some(title) = block.get("title").and_then(Value::as_str) {+ file["filename"] = Value::String(title.to_string());}+ Some(json!({"type": "file", "file": file}))}// Converts file sources to deterministic text fallback content.diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 71c765d01..e14d38e9b 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -5,7 +5,9 @@use pretty_assertions::assert_eq;use serde_json::{Value, json};-use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat};+use switchyard_translation::{+ LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,+};type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;@@ -310,6 +312,211 @@ fn anthropic_tool_result_followup_text_splits_to_openai_messages() -> TestResultOk(())}+// Verifies Anthropic multimodal blocks retain provider fields inside tool results.+#[test]+fn anthropic_tool_result_multimodal_blocks_round_trip_complete() -> TestResult {+ let engine = TranslationEngine::default();+ let policy = TranslationPolicy {+ preservation: switchyard_translation::PreservationPolicy::Disabled,+ ..TranslationPolicy::default()+ };+ let document = json!({+ "type": "document",+ "title": "report.pdf",+ "context": "Quarterly results",+ "citations": {"enabled": true},+ "source": {+ "type": "base64",+ "media_type": "application/pdf",+ "data": "ZG9jdW1lbnQ="+ }+ });+ let image = json!({+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "aW1hZ2U="+ }+ });+ let body = json!({+ "model": "claude-sonnet-4-20250514",+ "messages": [{+ "role": "user",+ "content": [{+ "type": "tool_result",+ "tool_use_id": "toolu_document",+ "content": [+ {"type": "text", "text": "content ready"},+ image.clone(),+ document.clone()+ ]+ }]+ }],+ "max_tokens": 1024+ });++ let output = engine+ .translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::AnthropicMessages,+ &body,+ &policy,+ )?+ .body;++ assert_eq!(+ output["messages"][0]["content"][0]["content"],+ json!([+ {"type": "text", "text": "content ready"},+ image,+ document+ ])+ );+ Ok(())+}++// Verifies provider-managed Anthropic file IDs are not reused as OpenAI file IDs.+#[test]+fn anthropic_tool_result_file_id_does_not_become_openai_file_id() -> TestResult {+ let engine = TranslationEngine::default();+ let document = json!({+ "type": "document",+ "source": {+ "type": "file",+ "file_id": "file_anthropic_123"+ }+ });+ let body = json!({+ "model": "claude-sonnet-4-20250514",+ "messages": [{+ "role": "user",+ "content": [{+ "type": "tool_result",+ "tool_use_id": "toolu_document",+ "content": [document.clone()]+ }]+ }],+ "max_tokens": 1024+ });++ let translated = engine.translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?;++ assert_eq!(translated.body["messages"][1]["content"][0]["type"], "text");+ let recovered: Value = serde_json::from_str(+ translated.body["messages"][1]["content"][0]["text"]+ .as_str()+ .ok_or("file fallback should be text")?,+ )?;+ assert_eq!(recovered, document);+ assert!(translated.diagnostics.iter().any(|diagnostic| {+ diagnostic.message == "OpenAI Chat codec could not map file content"+ }));+ Ok(())+}++// Verifies parallel tool results preserve ordering and obey strict conversion policy.+#[test]+fn anthropic_parallel_multimodal_tool_results_preserve_order_and_policy() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "claude-sonnet-4-20250514",+ "messages": [{+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_image",+ "content": [+ {"type": "text", "text": "image ready"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "aW1hZ2U="+ }+ }+ ]+ },+ {+ "type": "tool_result",+ "tool_use_id": "toolu_document",+ "content": [+ {"type": "text", "text": "document ready"},+ {+ "type": "document",+ "title": "report.pdf",+ "source": {+ "type": "base64",+ "media_type": "application/pdf",+ "data": "ZG9jdW1lbnQ="+ }+ }+ ]+ }+ ]+ }],+ "max_tokens": 1024+ });++ let output = engine+ .translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiChat,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(+ output["messages"],+ json!([+ {"role": "tool", "tool_call_id": "toolu_image", "content": "image ready"},+ {+ "role": "tool",+ "tool_call_id": "toolu_document",+ "content": "document ready"+ },+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="}+ },+ {+ "type": "file",+ "file": {"file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}+ }+ ]+ }+ ])+ );+ let policy = TranslationPolicy {+ lossy_conversion_policy: LossyConversionPolicy::Reject,+ ..TranslationPolicy::default()+ };++ let error = match engine.translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiChat,+ &body,+ &policy,+ ) {+ Ok(_) => panic!("multimodal tool result should be rejected by strict policy"),+ Err(error) => error,+ };++ assert_eq!(error.kind(), "LossyConversion");+ Ok(())+}+// Verifies structured Anthropic system blocks remain separated in OpenAI system text.#[test]fn anthropic_structured_system_blocks_preserve_boundaries_for_openai_chat() -> TestResult {