Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): preserve media inputs during request re-encoding (#765)
Graham King0a32f56bcbb7
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rsindex a0f1be95c..9661c7674 100644--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs@@ -30,6 +30,7 @@ use crate::util::{json_string, push_lossy, reject_responses_builtin_tool_item, stable_id, string_value,validate_request_capabilities,};+use base64::{Engine as _, engine::general_purpose::STANDARD};/// Format codec for Anthropic Messages payloads.pub struct AnthropicMessagesCodec;@@ -391,6 +392,7 @@ impl FormatCodec for AnthropicMessagesCodec {let output = response.first_output();let content = output.map(|output| encode_anthropic_content(&output.content))+ .transpose()?.unwrap_or_else(|| vec![json!({"type": "text", "text": ""})]);let normalized_stop_reason = output.and_then(|output| output.stop_reason);let body = json!({@@ -866,6 +868,7 @@ fn encode_anthropic_content_with_policy() -> Result<Vec<Value>> {let mut blocks = Vec::new();for block in content {+ crate::codecs::openai_media::validate_media(block, WireFormat::AnthropicMessages)?;match block {ContentBlock::Unknown { provider, raw } => {reject_responses_builtin_tool_item(provider, raw, WireFormat::AnthropicMessages)?;@@ -876,7 +879,7 @@ fn encode_anthropic_content_with_policy()?;blocks.push(json!({"type": "text", "text": json_string(raw)}));}- other => blocks.extend(encode_one_anthropic_block(other)),+ other => blocks.extend(encode_one_anthropic_block(other)?),}}if blocks.is_empty() {@@ -885,37 +888,91 @@ fn encode_anthropic_content_with_policy(Ok(blocks)}+fn encode_anthropic_file(source: &FileSource) -> Result<Value> {+ match source {+ FileSource::FileData { data, filename } => {+ let (media_type, data) = split_base64_data_uri(data)+ .unwrap_or_else(|| (document_media_type(filename.as_deref()), data.as_str()));+ let source = match media_type {+ "text/plain" => {+ let bytes =+ STANDARD+ .decode(data)+ .map_err(|_| TranslationError::InvalidValue {+ path: "file_data".into(),+ message: "invalid base64 text document".into(),+ })?;+ let text =+ String::from_utf8(bytes).map_err(|_| TranslationError::InvalidValue {+ path: "file_data".into(),+ message: "text document must be UTF-8".into(),+ })?;+ json!({"type": "text", "media_type": media_type, "data": text})+ }+ "application/pdf" => {+ json!({"type": "base64", "media_type": media_type, "data": data})+ }+ _ => {+ return Err(TranslationError::LossyConversion(+ "Anthropic requires PDF or plain-text documents".into(),+ ));+ }+ };+ let mut block = json!({"type": "document", "source": source});+ if let Some(filename) = filename {+ block["title"] = filename.clone().into();+ }+ Ok(block)+ }+ FileSource::Raw(raw) if raw.get("type").and_then(Value::as_str) == Some("input_file") => {+ let url = raw.get("file_url").and_then(Value::as_str).ok_or_else(|| {+ TranslationError::LossyConversion("unsupported Anthropic document source".into())+ })?;+ let mut block = json!({"type": "document", "source": {"type": "url", "url": url}});+ if let Some(filename) = raw.get("filename") {+ block["title"] = filename.clone();+ }+ Ok(block)+ }+ FileSource::Raw(raw) => Ok(raw.clone()),+ FileSource::FileId(file_id) => Ok(json!({+ "type": "document", "source": {"type": "file", "file_id": file_id}+ })),+ }+}+// Encodes content without producing diagnostics for response paths.-fn encode_anthropic_content(content: &[ContentBlock]) -> Vec<Value> {- let mut blocks = content- .iter()- .flat_map(encode_one_anthropic_response_block)- .collect::<Vec<_>>();+fn encode_anthropic_content(content: &[ContentBlock]) -> Result<Vec<Value>> {+ let mut blocks = Vec::new();+ for block in content {+ crate::codecs::openai_media::validate_media(block, WireFormat::AnthropicMessages)?;+ blocks.extend(encode_one_anthropic_response_block(block)?);+ }if blocks.is_empty() {blocks.push(json!({"type": "text", "text": ""}));}- blocks+ Ok(blocks)}// Encodes response content, where synthetic reasoning may be shown to clients.-fn encode_one_anthropic_response_block(block: &ContentBlock) -> Vec<Value> {+fn encode_one_anthropic_response_block(block: &ContentBlock) -> Result<Vec<Value>> {match block {ContentBlock::Reasoning {text,signature: None,..- } => vec![json!({+ } => Ok(vec![json!({"type": "thinking","thinking": text,"signature": "",- })],+ })]),other => encode_one_anthropic_block(other),}}// Encodes a single normalized content block into Anthropic block JSON.-fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {- match block {+fn encode_one_anthropic_block(block: &ContentBlock) -> Result<Vec<Value>> {+ Ok(match block {ContentBlock::Text { text } | ContentBlock::Refusal { text } => {vec![json!({"type": "text", "text": text})]}@@ -944,13 +1001,11 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {}) {Value::String(text_from_blocks(&result.content, " "))} else {- Value::Array(- result- .content- .iter()- .flat_map(encode_one_anthropic_tool_result_block)- .collect(),- )+ let mut content = Vec::new();+ for block in &result.content {+ content.extend(encode_one_anthropic_tool_result_block(block)?);+ }+ Value::Array(content)};let mut item = json!({"type": "tool_result",@@ -980,26 +1035,7 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {}),ImageSource::Raw(raw) => raw.clone(),}],- ContentBlock::File { source } => vec![match source {- FileSource::FileId(file_id) => {- json!({"type": "document", "source": {"type": "file", "file_id": file_id}})- }- // OpenAI carries `file_data` as raw base64 or a data URI. Anthropic wants raw- // base64 with a media type, and takes the name as the block `title`.- FileSource::FileData { data, filename } => {- let (media_type, data) = split_base64_data_uri(data)- .unwrap_or_else(|| (document_media_type(filename.as_deref()), data.as_str()));- let mut block = json!({- "type": "document",- "source": {"type": "base64", "media_type": media_type, "data": data},- });- if let Some(filename) = filename {- block["title"] = Value::String(filename.clone());- }- block- }- FileSource::Raw(raw) => raw.clone(),- }],+ ContentBlock::File { source } => vec![encode_anthropic_file(source)?],ContentBlock::Audio { source } => vec![match source {MediaSource::Url { url, media_type } => {json!({"type": "audio", "source": {"type": "url", "url": url, "media_type": media_type}})@@ -1029,11 +1065,11 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {MediaSource::Raw(raw) => raw.clone(),}],ContentBlock::Unknown { raw, .. } => vec![raw.clone()],- }+ })}// Encodes only provider-safe block shapes inside Anthropic tool results.-fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {+fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Result<Vec<Value>> {match block {ContentBlock::Text { .. }| ContentBlock::Refusal { .. }@@ -1042,16 +1078,16 @@ fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {ContentBlock::Unknown { provider, raw }if provider.as_str() == WireFormat::AnthropicMessages.as_str() =>{- vec![raw.clone()]+ Ok(vec![raw.clone()])}ContentBlock::Unknown { raw, .. } => {- vec![json!({"type": "text", "text": json_string(raw)})]+ Ok(vec![json!({"type": "text", "text": json_string(raw)})])}ContentBlock::Reasoning { .. }| ContentBlock::Audio { .. }| ContentBlock::Video { .. }| ContentBlock::ToolCall(_)- | ContentBlock::ToolResult(_) => Vec::new(),+ | ContentBlock::ToolResult(_) => Ok(Vec::new()),}}diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex d68c9acbd..a864814a6 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -546,6 +546,9 @@ pub(crate) fn decode_openai_content(content.push(ContentBlock::Image { source });}}+ Some("input_audio") => content.push(ContentBlock::Audio {+ source: MediaSource::Raw(Value::Object(block.clone())),+ }),Some("file") | Some("input_file") => {content.push(ContentBlock::File {source: decode_file_source(block),@@ -606,7 +609,7 @@ pub(crate) fn decode_image_source(block: &Map<String, Value>) -> Option<ImageSou.map(ToOwned::to_owned),});}- None+ Some(ImageSource::Raw(Value::Object(block.clone())))}/// Decodes OpenAI file block shapes into normalized file sources.@@ -1026,6 +1029,7 @@ pub(crate) fn encode_openai_content(}let mut blocks = Vec::new();for block in content {+ crate::codecs::openai_media::validate_media(block, WireFormat::OpenAiChat)?;match block {ContentBlock::Text { text } => blocks.push(json!({"type": "text", "text": text})),ContentBlock::Refusal { text } => blocks.push(json!({"type": "text", "text": text})),@@ -1052,12 +1056,7 @@ pub(crate) fn encode_openai_content(}},ContentBlock::Audio { source } => {- push_lossy(- diagnostics,- policy,- "OpenAI Chat codec does not have a stable audio request mapping yet",- )?;- blocks.push(openai_text_part(&media_source_text(source)));+ blocks.push(crate::codecs::openai_media::audio_part(source)?);}ContentBlock::Video { source } => {push_lossy(diff --git a/crates/switchyard-translation/src/codecs/openai_media.rs b/crates/switchyard-translation/src/codecs/openai_media.rsindex cd959abfc..403c27320 100644--- a/crates/switchyard-translation/src/codecs/openai_media.rs+++ b/crates/switchyard-translation/src/codecs/openai_media.rs@@ -1,11 +1,14 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Image and file payloads shared by the OpenAI Chat and Responses codecs.+//! Media payloads shared by the OpenAI Chat and Responses codecs.+use base64::{Engine as _, engine::general_purpose::STANDARD};use serde_json::{Map, Value, json};-use crate::llm::{FileSource, ImageSource};+use crate::error::{Result, TranslationError};+use crate::format::WireFormat;+use crate::llm::{ContentBlock, FileSource, ImageSource, MediaSource};use crate::util::json_string;pub(super) struct ImagePayload {@@ -107,18 +110,101 @@ fn file_data_payload(data: &str, filename: Option<&str>) -> Map<String, Value> {// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.fn raw_file_payload(raw: &Value) -> Option<Map<String, Value>> {let block = raw.as_object()?;+ if block.get("type").and_then(Value::as_str) == Some("input_file") {+ let mut payload = block.clone();+ payload.remove("type");+ return Some(payload);+ }if block.get("type").and_then(Value::as_str) != Some("document") {return None;}let source = block.get("source").and_then(Value::as_object)?;- if source.get("type").and_then(Value::as_str) != Some("base64") {- return None;+ let filename = block.get("title").and_then(Value::as_str);+ match source.get("type").and_then(Value::as_str)? {+ "url" => {+ let mut payload = Map::new();+ payload.insert("file_url".into(), source.get("url")?.clone());+ if let Some(filename) = filename {+ payload.insert("filename".into(), filename.into());+ }+ Some(payload)+ }+ "text" => Some(file_data_payload(+ &format!(+ "data:text/plain;base64,{}",+ STANDARD.encode(source.get("data")?.as_str()?)+ ),+ filename,+ )),+ "base64" => Some(file_data_payload(source.get("data")?.as_str()?, filename)),+ _ => None,+ }+}++pub(super) fn audio_part(source: &MediaSource) -> Result<Value> {+ match source {+ MediaSource::Raw(raw)+ if raw.get("type").and_then(Value::as_str) == Some("input_audio")+ && raw["input_audio"]["data"].is_string()+ && matches!(raw["input_audio"]["format"].as_str(), Some("wav" | "mp3")) =>+ {+ Ok(raw.clone())+ }+ MediaSource::Base64 {+ media_type: Some(media_type),+ data,+ } => {+ let format = match media_type.as_str() {+ "audio/wav" => "wav",+ "audio/mpeg" => "mp3",+ _ => {+ return Err(TranslationError::LossyConversion(+ "unsupported input audio media type".into(),+ ));+ }+ };+ Ok(json!({"type": "input_audio", "input_audio": {"data": data, "format": format}}))+ }+ MediaSource::Url { .. } => Err(TranslationError::LossyConversion(+ "OpenAI input audio requires base64 data, not a URL; provide MediaSource::Base64 instead".into(),+ )),+ _ => Err(TranslationError::LossyConversion(+ "unsupported input audio source".into(),+ )),+ }+}++// File IDs belong to their provider; a target cannot resolve another provider's image ID.+pub(super) fn validate_media(block: &ContentBlock, target: WireFormat) -> Result<()> {+ let is_unsupported = match block {+ ContentBlock::Image {+ source: ImageSource::Raw(raw),+ } => raw.get("file_id").is_some() && target != WireFormat::OpenAiResponses,+ ContentBlock::Audio { .. } => target == WireFormat::AnthropicMessages,+ ContentBlock::File {+ source: FileSource::FileId(_),+ } => target == WireFormat::AnthropicMessages,+ ContentBlock::File {+ source: FileSource::Raw(raw),+ } if target == WireFormat::OpenAiChat => {+ raw.get("file_url").is_some()+ || (raw.get("type").and_then(Value::as_str) == Some("document")+ && raw["source"]["type"].as_str() == Some("url"))+ }+ ContentBlock::ToolResult(result) => {+ for block in &result.content {+ validate_media(block, target)?;+ }+ false+ }+ _ => false,+ };+ if is_unsupported {+ return Err(TranslationError::LossyConversion(format!(+ "input media cannot be represented in {target}"+ )));}- let data = source.get("data").and_then(Value::as_str)?;- Some(file_data_payload(- data,- block.get("title").and_then(Value::as_str),- ))+ Ok(())}pub(super) fn file_source_text(source: &FileSource) -> String {diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex f078d469e..ef3f078ed 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -897,6 +897,9 @@ fn decode_responses_content(value: &Value) -> Vec<ContentBlock> {out.push(ContentBlock::Image { source });}}+ Some("input_audio") => out.push(ContentBlock::Audio {+ source: MediaSource::Raw(Value::Object(block.clone())),+ }),Some("input_file") => out.push(ContentBlock::File {source: decode_file_source(block),}),@@ -1527,18 +1530,9 @@ fn encode_responses_content(blocks.push(json!({"type": "input_text", "text": image_source_text(source)}));}},- ContentBlock::Audio { source } => blocks.push(match source {- MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),- MediaSource::Url { url, media_type } => json!({- "type": "input_audio",- "audio_url": url,- "media_type": media_type,- }),- MediaSource::Base64 { media_type, data } => json!({- "type": "input_audio",- "audio": {"media_type": media_type, "data": data},- }),- }),+ ContentBlock::Audio { source } => {+ blocks.push(crate::codecs::openai_media::audio_part(source)?);+ }ContentBlock::Video { source } => blocks.push(match source {MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),MediaSource::Url { url, media_type } => json!({@@ -1598,6 +1592,12 @@ fn encode_responses_tool_output(}fn responses_image_part(source: &ImageSource) -> Option<Value> {+ if let ImageSource::Raw(raw) = source+ && raw.get("type").and_then(Value::as_str) == Some("input_image")+ && raw.get("file_id").and_then(Value::as_str).is_some()+ {+ return Some(raw.clone());+ }let ImagePayload { url, detail } = image_payload(source)?;let mut part = json!({"type": "input_image"});part["image_url"] = Value::String(url);diff --git a/crates/switchyard-translation/tests/lossless_roundtrip.rs b/crates/switchyard-translation/tests/lossless_roundtrip.rsindex c9375b481..db99a71fd 100644--- a/crates/switchyard-translation/tests/lossless_roundtrip.rs+++ b/crates/switchyard-translation/tests/lossless_roundtrip.rs@@ -304,7 +304,7 @@ fn request_fixture(format: WireFormat) -> Value {},{"type": "file",- "file": {"file_id": "file_123"}+ "file": {"file_data": "aGVsbG8=", "filename": "notes.txt"}},{"type": "vendor_block",@@ -470,7 +470,8 @@ fn request_fixture(format: WireFormat) -> Value {},{"type": "input_file",- "file": {"file_id": "file_123"}+ "file_data": "aGVsbG8=",+ "filename": "notes.txt"},{"type": "vendor_block",diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 9cb06ea40..01d35cd12 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -16,6 +16,116 @@ use common::{REASONING_MODEL, normalized_policy, shell_tool_call};type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;+#[test]+fn request_media_survives_reencoding_or_is_rejected() -> TestResult {+ use WireFormat::{+ AnthropicMessages as Anthropic, OpenAiChat as Chat, OpenAiResponses as Responses,+ };+ let engine = TranslationEngine::default();+ let untitled = json!({+ "type": "document",+ "source": {"type": "text", "media_type": "text/plain", "data": "hello"}+ });+ let body = json!({"messages": [{"role": "user", "content": [untitled]}], "max_tokens": 16});+ for target in [Chat, Responses] {+ let translated =+ engine.translate_request(Anthropic, target, &body, &normalized_policy())?;+ let restored =+ engine.translate_request(target, Anthropic, &translated.body, &normalized_policy())?;+ assert_eq!(restored.body["messages"][0]["content"][0], untitled);+ }+ let image = json!({"type": "input_image", "file_id": "file_image", "detail": "auto"});+ let audio =+ json!({"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}});+ let file = json!({"type": "input_file", "file_url": "https://example.com/report.pdf", "filename": "report.pdf"});+ let document = json!({"type": "document", "source": {"type": "url", "url": "https://example.com/report.pdf"}, "title": "report.pdf"});+ let text_file = json!({"type": "input_file", "file_data": "aGVsbG8=", "filename": "notes.txt"});+ let text_document = json!({"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "hello"}, "title": "notes.txt"});+ for (source, target, block, expected) in [+ (Responses, Responses, image.clone(), Some(image.clone())),+ (Responses, Chat, image.clone(), None),+ (Responses, Anthropic, image, None),+ (+ Responses,+ Anthropic,+ json!({"type": "input_file", "file_id": "file_openai"}),+ None,+ ),+ (+ Chat,+ Anthropic,+ json!({"type": "file", "file": {"file_id": "file_openai"}}),+ None,+ ),+ (+ Anthropic,+ Anthropic,+ json!({"type": "document", "source": {"type": "file", "file_id": "file_anthropic"}}),+ Some(+ json!({"type": "document", "source": {"type": "file", "file_id": "file_anthropic"}}),+ ),+ ),+ (Chat, Chat, audio.clone(), Some(audio.clone())),+ (Chat, Responses, audio.clone(), Some(audio.clone())),+ (Responses, Chat, audio.clone(), Some(audio.clone())),+ (Responses, Responses, audio.clone(), Some(audio.clone())),+ (Responses, Anthropic, audio, None),+ (Responses, Responses, file.clone(), Some(file.clone())),+ (Responses, Chat, file.clone(), None),+ (Responses, Anthropic, file.clone(), Some(document.clone())),+ (Anthropic, Chat, document.clone(), None),+ (Anthropic, Responses, document, Some(file)),+ (+ Responses,+ Anthropic,+ text_file.clone(),+ Some(text_document.clone()),+ ),+ (+ Responses,+ Anthropic,+ json!({"type": "input_file", "file_data": "data:text/plain;base64,aGVsbG8=", "filename": "notes.txt"}),+ Some(text_document.clone()),+ ),+ (+ Anthropic,+ Responses,+ text_document,+ Some(+ json!({"type": "input_file", "file_data": "data:text/plain;base64,aGVsbG8=", "filename": "notes.txt"}),+ ),+ ),+ ] {+ let body = if source == Responses {+ json!({"input": [{"role": "user", "content": [block]}]})+ } else {+ json!({"messages": [{"role": "user", "content": [block]}], "max_tokens": 16})+ };+ let result = engine.translate_request(source, target, &body, &normalized_policy());+ if let Some(expected) = expected {+ let output = result?.body;+ let messages = if target == Responses {+ &output["input"]+ } else {+ &output["messages"]+ };+ assert_eq!(+ messages[0]["content"][0], expected,+ "{source:?} -> {target:?}"+ );+ } else {+ assert!(+ matches!(+ result,+ Err(switchyard_translation::TranslationError::LossyConversion(_))+ ),+ "{source:?} -> {target:?} must reject unsupported media"+ );+ }+ }+ Ok(())+}+#[test]fn abuse_identity_survives_request_translation() -> TestResult {let engine = TranslationEngine::default();