Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): preserve media inputs during request re-encoding (#765)

Graham King0a32f56bcbb7
diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
index a0f1be95c..9661c7674 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs
@@ -30,6 +30,7 @@ use crate::util::{
json_string, push_lossy, reject_responses_builtin_tool_item, stable_id, string_value,
validate_request_capabilities,
};
+use base64::{Engine as _, engine::general_purpose::STANDARD};
/// Format codec for Anthropic Messages payloads.
pub struct AnthropicMessagesCodec;
@@ -391,6 +392,7 @@ impl FormatCodec for AnthropicMessagesCodec {
let output = response.first_output();
let content = output
.map(|output| encode_anthropic_content(&output.content))
+ .transpose()?
.unwrap_or_else(|| vec![json!({"type": "text", "text": ""})]);
let normalized_stop_reason = output.and_then(|output| output.stop_reason);
let body = json!({
@@ -866,6 +868,7 @@ fn encode_anthropic_content_with_policy(
) -> Result<Vec<Value>> {
let mut blocks = Vec::new();
for block in content {
+ crate::codecs::openai_media::validate_media(block, WireFormat::AnthropicMessages)?;
match block {
ContentBlock::Unknown { provider, raw } => {
reject_responses_builtin_tool_item(provider, raw, WireFormat::AnthropicMessages)?;
@@ -876,7 +879,7 @@ fn encode_anthropic_content_with_policy(
)?;
blocks.push(json!({"type": "text", "text": json_string(raw)}));
}
- other => blocks.extend(encode_one_anthropic_block(other)),
+ other => blocks.extend(encode_one_anthropic_block(other)?),
}
}
if blocks.is_empty() {
@@ -885,37 +888,91 @@ fn encode_anthropic_content_with_policy(
Ok(blocks)
}
+fn encode_anthropic_file(source: &FileSource) -> Result<Value> {
+ match source {
+ FileSource::FileData { data, filename } => {
+ let (media_type, data) = split_base64_data_uri(data)
+ .unwrap_or_else(|| (document_media_type(filename.as_deref()), data.as_str()));
+ let source = match media_type {
+ "text/plain" => {
+ let bytes =
+ STANDARD
+ .decode(data)
+ .map_err(|_| TranslationError::InvalidValue {
+ path: "file_data".into(),
+ message: "invalid base64 text document".into(),
+ })?;
+ let text =
+ String::from_utf8(bytes).map_err(|_| TranslationError::InvalidValue {
+ path: "file_data".into(),
+ message: "text document must be UTF-8".into(),
+ })?;
+ json!({"type": "text", "media_type": media_type, "data": text})
+ }
+ "application/pdf" => {
+ json!({"type": "base64", "media_type": media_type, "data": data})
+ }
+ _ => {
+ return Err(TranslationError::LossyConversion(
+ "Anthropic requires PDF or plain-text documents".into(),
+ ));
+ }
+ };
+ let mut block = json!({"type": "document", "source": source});
+ if let Some(filename) = filename {
+ block["title"] = filename.clone().into();
+ }
+ Ok(block)
+ }
+ FileSource::Raw(raw) if raw.get("type").and_then(Value::as_str) == Some("input_file") => {
+ let url = raw.get("file_url").and_then(Value::as_str).ok_or_else(|| {
+ TranslationError::LossyConversion("unsupported Anthropic document source".into())
+ })?;
+ let mut block = json!({"type": "document", "source": {"type": "url", "url": url}});
+ if let Some(filename) = raw.get("filename") {
+ block["title"] = filename.clone();
+ }
+ Ok(block)
+ }
+ FileSource::Raw(raw) => Ok(raw.clone()),
+ FileSource::FileId(file_id) => Ok(json!({
+ "type": "document", "source": {"type": "file", "file_id": file_id}
+ })),
+ }
+}
+
// Encodes content without producing diagnostics for response paths.
-fn encode_anthropic_content(content: &[ContentBlock]) -> Vec<Value> {
- let mut blocks = content
- .iter()
- .flat_map(encode_one_anthropic_response_block)
- .collect::<Vec<_>>();
+fn encode_anthropic_content(content: &[ContentBlock]) -> Result<Vec<Value>> {
+ let mut blocks = Vec::new();
+ for block in content {
+ crate::codecs::openai_media::validate_media(block, WireFormat::AnthropicMessages)?;
+ blocks.extend(encode_one_anthropic_response_block(block)?);
+ }
if blocks.is_empty() {
blocks.push(json!({"type": "text", "text": ""}));
}
- blocks
+ Ok(blocks)
}
// Encodes response content, where synthetic reasoning may be shown to clients.
-fn encode_one_anthropic_response_block(block: &ContentBlock) -> Vec<Value> {
+fn encode_one_anthropic_response_block(block: &ContentBlock) -> Result<Vec<Value>> {
match block {
ContentBlock::Reasoning {
text,
signature: None,
..
- } => vec![json!({
+ } => Ok(vec![json!({
"type": "thinking",
"thinking": text,
"signature": "",
- })],
+ })]),
other => encode_one_anthropic_block(other),
}
}
// Encodes a single normalized content block into Anthropic block JSON.
-fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
- match block {
+fn encode_one_anthropic_block(block: &ContentBlock) -> Result<Vec<Value>> {
+ Ok(match block {
ContentBlock::Text { text } | ContentBlock::Refusal { text } => {
vec![json!({"type": "text", "text": text})]
}
@@ -944,13 +1001,11 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
}) {
Value::String(text_from_blocks(&result.content, " "))
} else {
- Value::Array(
- result
- .content
- .iter()
- .flat_map(encode_one_anthropic_tool_result_block)
- .collect(),
- )
+ let mut content = Vec::new();
+ for block in &result.content {
+ content.extend(encode_one_anthropic_tool_result_block(block)?);
+ }
+ Value::Array(content)
};
let mut item = json!({
"type": "tool_result",
@@ -980,26 +1035,7 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
}),
ImageSource::Raw(raw) => raw.clone(),
}],
- ContentBlock::File { source } => vec![match source {
- FileSource::FileId(file_id) => {
- json!({"type": "document", "source": {"type": "file", "file_id": file_id}})
- }
- // OpenAI carries `file_data` as raw base64 or a data URI. Anthropic wants raw
- // base64 with a media type, and takes the name as the block `title`.
- FileSource::FileData { data, filename } => {
- let (media_type, data) = split_base64_data_uri(data)
- .unwrap_or_else(|| (document_media_type(filename.as_deref()), data.as_str()));
- let mut block = json!({
- "type": "document",
- "source": {"type": "base64", "media_type": media_type, "data": data},
- });
- if let Some(filename) = filename {
- block["title"] = Value::String(filename.clone());
- }
- block
- }
- FileSource::Raw(raw) => raw.clone(),
- }],
+ ContentBlock::File { source } => vec![encode_anthropic_file(source)?],
ContentBlock::Audio { source } => vec![match source {
MediaSource::Url { url, media_type } => {
json!({"type": "audio", "source": {"type": "url", "url": url, "media_type": media_type}})
@@ -1029,11 +1065,11 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
MediaSource::Raw(raw) => raw.clone(),
}],
ContentBlock::Unknown { raw, .. } => vec![raw.clone()],
- }
+ })
}
// Encodes only provider-safe block shapes inside Anthropic tool results.
-fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {
+fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Result<Vec<Value>> {
match block {
ContentBlock::Text { .. }
| ContentBlock::Refusal { .. }
@@ -1042,16 +1078,16 @@ fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {
ContentBlock::Unknown { provider, raw }
if provider.as_str() == WireFormat::AnthropicMessages.as_str() =>
{
- vec![raw.clone()]
+ Ok(vec![raw.clone()])
}
ContentBlock::Unknown { raw, .. } => {
- vec![json!({"type": "text", "text": json_string(raw)})]
+ Ok(vec![json!({"type": "text", "text": json_string(raw)})])
}
ContentBlock::Reasoning { .. }
| ContentBlock::Audio { .. }
| ContentBlock::Video { .. }
| ContentBlock::ToolCall(_)
- | ContentBlock::ToolResult(_) => Vec::new(),
+ | ContentBlock::ToolResult(_) => Ok(Vec::new()),
}
}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index d68c9acbd..a864814a6 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -546,6 +546,9 @@ pub(crate) fn decode_openai_content(
content.push(ContentBlock::Image { source });
}
}
+ Some("input_audio") => content.push(ContentBlock::Audio {
+ source: MediaSource::Raw(Value::Object(block.clone())),
+ }),
Some("file") | Some("input_file") => {
content.push(ContentBlock::File {
source: decode_file_source(block),
@@ -606,7 +609,7 @@ pub(crate) fn decode_image_source(block: &Map<String, Value>) -> Option<ImageSou
.map(ToOwned::to_owned),
});
}
- None
+ Some(ImageSource::Raw(Value::Object(block.clone())))
}
/// Decodes OpenAI file block shapes into normalized file sources.
@@ -1026,6 +1029,7 @@ pub(crate) fn encode_openai_content(
}
let mut blocks = Vec::new();
for block in content {
+ crate::codecs::openai_media::validate_media(block, WireFormat::OpenAiChat)?;
match block {
ContentBlock::Text { text } => blocks.push(json!({"type": "text", "text": text})),
ContentBlock::Refusal { text } => blocks.push(json!({"type": "text", "text": text})),
@@ -1052,12 +1056,7 @@ pub(crate) fn encode_openai_content(
}
},
ContentBlock::Audio { source } => {
- push_lossy(
- diagnostics,
- policy,
- "OpenAI Chat codec does not have a stable audio request mapping yet",
- )?;
- blocks.push(openai_text_part(&media_source_text(source)));
+ blocks.push(crate::codecs::openai_media::audio_part(source)?);
}
ContentBlock::Video { source } => {
push_lossy(
diff --git a/crates/switchyard-translation/src/codecs/openai_media.rs b/crates/switchyard-translation/src/codecs/openai_media.rs
index cd959abfc..403c27320 100644
--- a/crates/switchyard-translation/src/codecs/openai_media.rs
+++ b/crates/switchyard-translation/src/codecs/openai_media.rs
@@ -1,11 +1,14 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Image and file payloads shared by the OpenAI Chat and Responses codecs.
+//! Media payloads shared by the OpenAI Chat and Responses codecs.
+use base64::{Engine as _, engine::general_purpose::STANDARD};
use serde_json::{Map, Value, json};
-use crate::llm::{FileSource, ImageSource};
+use crate::error::{Result, TranslationError};
+use crate::format::WireFormat;
+use crate::llm::{ContentBlock, FileSource, ImageSource, MediaSource};
use crate::util::json_string;
pub(super) struct ImagePayload {
@@ -107,18 +110,101 @@ fn file_data_payload(data: &str, filename: Option<&str>) -> Map<String, Value> {
// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.
fn raw_file_payload(raw: &Value) -> Option<Map<String, Value>> {
let block = raw.as_object()?;
+ if block.get("type").and_then(Value::as_str) == Some("input_file") {
+ let mut payload = block.clone();
+ payload.remove("type");
+ return Some(payload);
+ }
if block.get("type").and_then(Value::as_str) != Some("document") {
return None;
}
let source = block.get("source").and_then(Value::as_object)?;
- if source.get("type").and_then(Value::as_str) != Some("base64") {
- return None;
+ let filename = block.get("title").and_then(Value::as_str);
+ match source.get("type").and_then(Value::as_str)? {
+ "url" => {
+ let mut payload = Map::new();
+ payload.insert("file_url".into(), source.get("url")?.clone());
+ if let Some(filename) = filename {
+ payload.insert("filename".into(), filename.into());
+ }
+ Some(payload)
+ }
+ "text" => Some(file_data_payload(
+ &format!(
+ "data:text/plain;base64,{}",
+ STANDARD.encode(source.get("data")?.as_str()?)
+ ),
+ filename,
+ )),
+ "base64" => Some(file_data_payload(source.get("data")?.as_str()?, filename)),
+ _ => None,
+ }
+}
+
+pub(super) fn audio_part(source: &MediaSource) -> Result<Value> {
+ match source {
+ MediaSource::Raw(raw)
+ if raw.get("type").and_then(Value::as_str) == Some("input_audio")
+ && raw["input_audio"]["data"].is_string()
+ && matches!(raw["input_audio"]["format"].as_str(), Some("wav" | "mp3")) =>
+ {
+ Ok(raw.clone())
+ }
+ MediaSource::Base64 {
+ media_type: Some(media_type),
+ data,
+ } => {
+ let format = match media_type.as_str() {
+ "audio/wav" => "wav",
+ "audio/mpeg" => "mp3",
+ _ => {
+ return Err(TranslationError::LossyConversion(
+ "unsupported input audio media type".into(),
+ ));
+ }
+ };
+ Ok(json!({"type": "input_audio", "input_audio": {"data": data, "format": format}}))
+ }
+ MediaSource::Url { .. } => Err(TranslationError::LossyConversion(
+ "OpenAI input audio requires base64 data, not a URL; provide MediaSource::Base64 instead".into(),
+ )),
+ _ => Err(TranslationError::LossyConversion(
+ "unsupported input audio source".into(),
+ )),
+ }
+}
+
+// File IDs belong to their provider; a target cannot resolve another provider's image ID.
+pub(super) fn validate_media(block: &ContentBlock, target: WireFormat) -> Result<()> {
+ let is_unsupported = match block {
+ ContentBlock::Image {
+ source: ImageSource::Raw(raw),
+ } => raw.get("file_id").is_some() && target != WireFormat::OpenAiResponses,
+ ContentBlock::Audio { .. } => target == WireFormat::AnthropicMessages,
+ ContentBlock::File {
+ source: FileSource::FileId(_),
+ } => target == WireFormat::AnthropicMessages,
+ ContentBlock::File {
+ source: FileSource::Raw(raw),
+ } if target == WireFormat::OpenAiChat => {
+ raw.get("file_url").is_some()
+ || (raw.get("type").and_then(Value::as_str) == Some("document")
+ && raw["source"]["type"].as_str() == Some("url"))
+ }
+ ContentBlock::ToolResult(result) => {
+ for block in &result.content {
+ validate_media(block, target)?;
+ }
+ false
+ }
+ _ => false,
+ };
+ if is_unsupported {
+ return Err(TranslationError::LossyConversion(format!(
+ "input media cannot be represented in {target}"
+ )));
}
- let data = source.get("data").and_then(Value::as_str)?;
- Some(file_data_payload(
- data,
- block.get("title").and_then(Value::as_str),
- ))
+ Ok(())
}
pub(super) fn file_source_text(source: &FileSource) -> String {
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index f078d469e..ef3f078ed 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -897,6 +897,9 @@ fn decode_responses_content(value: &Value) -> Vec<ContentBlock> {
out.push(ContentBlock::Image { source });
}
}
+ Some("input_audio") => out.push(ContentBlock::Audio {
+ source: MediaSource::Raw(Value::Object(block.clone())),
+ }),
Some("input_file") => out.push(ContentBlock::File {
source: decode_file_source(block),
}),
@@ -1527,18 +1530,9 @@ fn encode_responses_content(
blocks.push(json!({"type": "input_text", "text": image_source_text(source)}));
}
},
- ContentBlock::Audio { source } => blocks.push(match source {
- MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),
- MediaSource::Url { url, media_type } => json!({
- "type": "input_audio",
- "audio_url": url,
- "media_type": media_type,
- }),
- MediaSource::Base64 { media_type, data } => json!({
- "type": "input_audio",
- "audio": {"media_type": media_type, "data": data},
- }),
- }),
+ ContentBlock::Audio { source } => {
+ blocks.push(crate::codecs::openai_media::audio_part(source)?);
+ }
ContentBlock::Video { source } => blocks.push(match source {
MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),
MediaSource::Url { url, media_type } => json!({
@@ -1598,6 +1592,12 @@ fn encode_responses_tool_output(
}
fn responses_image_part(source: &ImageSource) -> Option<Value> {
+ if let ImageSource::Raw(raw) = source
+ && raw.get("type").and_then(Value::as_str) == Some("input_image")
+ && raw.get("file_id").and_then(Value::as_str).is_some()
+ {
+ return Some(raw.clone());
+ }
let ImagePayload { url, detail } = image_payload(source)?;
let mut part = json!({"type": "input_image"});
part["image_url"] = Value::String(url);
diff --git a/crates/switchyard-translation/tests/lossless_roundtrip.rs b/crates/switchyard-translation/tests/lossless_roundtrip.rs
index c9375b481..db99a71fd 100644
--- a/crates/switchyard-translation/tests/lossless_roundtrip.rs
+++ b/crates/switchyard-translation/tests/lossless_roundtrip.rs
@@ -304,7 +304,7 @@ fn request_fixture(format: WireFormat) -> Value {
},
{
"type": "file",
- "file": {"file_id": "file_123"}
+ "file": {"file_data": "aGVsbG8=", "filename": "notes.txt"}
},
{
"type": "vendor_block",
@@ -470,7 +470,8 @@ fn request_fixture(format: WireFormat) -> Value {
},
{
"type": "input_file",
- "file": {"file_id": "file_123"}
+ "file_data": "aGVsbG8=",
+ "filename": "notes.txt"
},
{
"type": "vendor_block",
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 9cb06ea40..01d35cd12 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -16,6 +16,116 @@ use common::{REASONING_MODEL, normalized_policy, shell_tool_call};
type TestResult<T = ()> = std::result::Result<T, Box<dyn std::error::Error + Send + Sync>>;
+#[test]
+fn request_media_survives_reencoding_or_is_rejected() -> TestResult {
+ use WireFormat::{
+ AnthropicMessages as Anthropic, OpenAiChat as Chat, OpenAiResponses as Responses,
+ };
+ let engine = TranslationEngine::default();
+ let untitled = json!({
+ "type": "document",
+ "source": {"type": "text", "media_type": "text/plain", "data": "hello"}
+ });
+ let body = json!({"messages": [{"role": "user", "content": [untitled]}], "max_tokens": 16});
+ for target in [Chat, Responses] {
+ let translated =
+ engine.translate_request(Anthropic, target, &body, &normalized_policy())?;
+ let restored =
+ engine.translate_request(target, Anthropic, &translated.body, &normalized_policy())?;
+ assert_eq!(restored.body["messages"][0]["content"][0], untitled);
+ }
+ let image = json!({"type": "input_image", "file_id": "file_image", "detail": "auto"});
+ let audio =
+ json!({"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}});
+ let file = json!({"type": "input_file", "file_url": "https://example.com/report.pdf", "filename": "report.pdf"});
+ let document = json!({"type": "document", "source": {"type": "url", "url": "https://example.com/report.pdf"}, "title": "report.pdf"});
+ let text_file = json!({"type": "input_file", "file_data": "aGVsbG8=", "filename": "notes.txt"});
+ let text_document = json!({"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "hello"}, "title": "notes.txt"});
+ for (source, target, block, expected) in [
+ (Responses, Responses, image.clone(), Some(image.clone())),
+ (Responses, Chat, image.clone(), None),
+ (Responses, Anthropic, image, None),
+ (
+ Responses,
+ Anthropic,
+ json!({"type": "input_file", "file_id": "file_openai"}),
+ None,
+ ),
+ (
+ Chat,
+ Anthropic,
+ json!({"type": "file", "file": {"file_id": "file_openai"}}),
+ None,
+ ),
+ (
+ Anthropic,
+ Anthropic,
+ json!({"type": "document", "source": {"type": "file", "file_id": "file_anthropic"}}),
+ Some(
+ json!({"type": "document", "source": {"type": "file", "file_id": "file_anthropic"}}),
+ ),
+ ),
+ (Chat, Chat, audio.clone(), Some(audio.clone())),
+ (Chat, Responses, audio.clone(), Some(audio.clone())),
+ (Responses, Chat, audio.clone(), Some(audio.clone())),
+ (Responses, Responses, audio.clone(), Some(audio.clone())),
+ (Responses, Anthropic, audio, None),
+ (Responses, Responses, file.clone(), Some(file.clone())),
+ (Responses, Chat, file.clone(), None),
+ (Responses, Anthropic, file.clone(), Some(document.clone())),
+ (Anthropic, Chat, document.clone(), None),
+ (Anthropic, Responses, document, Some(file)),
+ (
+ Responses,
+ Anthropic,
+ text_file.clone(),
+ Some(text_document.clone()),
+ ),
+ (
+ Responses,
+ Anthropic,
+ json!({"type": "input_file", "file_data": "data:text/plain;base64,aGVsbG8=", "filename": "notes.txt"}),
+ Some(text_document.clone()),
+ ),
+ (
+ Anthropic,
+ Responses,
+ text_document,
+ Some(
+ json!({"type": "input_file", "file_data": "data:text/plain;base64,aGVsbG8=", "filename": "notes.txt"}),
+ ),
+ ),
+ ] {
+ let body = if source == Responses {
+ json!({"input": [{"role": "user", "content": [block]}]})
+ } else {
+ json!({"messages": [{"role": "user", "content": [block]}], "max_tokens": 16})
+ };
+ let result = engine.translate_request(source, target, &body, &normalized_policy());
+ if let Some(expected) = expected {
+ let output = result?.body;
+ let messages = if target == Responses {
+ &output["input"]
+ } else {
+ &output["messages"]
+ };
+ assert_eq!(
+ messages[0]["content"][0], expected,
+ "{source:?} -> {target:?}"
+ );
+ } else {
+ assert!(
+ matches!(
+ result,
+ Err(switchyard_translation::TranslationError::LossyConversion(_))
+ ),
+ "{source:?} -> {target:?} must reject unsupported media"
+ );
+ }
+ }
+ Ok(())
+}
+
#[test]
fn abuse_identity_survives_request_translation() -> TestResult {
let engine = TranslationEngine::default();