Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: encode Responses image and file inputs as wire shapes, not IR enums (#530)

Jeremy Schoemakerf27abd664cb8
diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rs
index 78d3a007c..16b1bda80 100644
--- a/crates/switchyard-translation/src/codecs/mod.rs
+++ b/crates/switchyard-translation/src/codecs/mod.rs
@@ -6,6 +6,7 @@
pub mod anthropic;
pub(crate) mod common;
pub mod openai_chat;
+mod openai_media;
pub mod responses;
pub mod stream;
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
index 988cd9768..bdbd4b60a 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs
@@ -9,6 +9,9 @@ use crate::codecs::common::{
first_nonempty_string, is_known_role_name, provider_extensions, reasoning_text_from_blocks,
reasoning_text_from_details, text_from_blocks,
};
+use crate::codecs::openai_media::{
+ ImagePayload, file_payload, file_source_text, image_payload, image_source_text,
+};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
};
@@ -1066,116 +1069,22 @@ fn openai_text_part(text: &str) -> Value {
json!({"type": "text", "text": text})
}
-// Maps IR image sources to OpenAI Chat image content parts when possible.
fn openai_image_part(source: &ImageSource) -> Option<Value> {
- match source {
- ImageSource::Url { url, detail } => {
- let mut image_url = json!({"url": url});
- if let Some(detail) = detail {
- image_url["detail"] = Value::String(detail.clone());
- }
- Some(json!({"type": "image_url", "image_url": image_url}))
- }
- ImageSource::Base64 { media_type, data } => media_type.as_ref().map(|media_type| {
- json!({
- "type": "image_url",
- "image_url": {"url": format!("data:{media_type};base64,{data}")},
- })
- }),
- ImageSource::Raw(raw) => openai_raw_image_part(raw),
+ let ImagePayload { url, detail } = image_payload(source)?;
+ let mut image_url = json!({});
+ image_url["url"] = Value::String(url);
+ if let Some(detail) = detail {
+ image_url["detail"] = Value::String(detail);
}
+ let mut part = json!({"type": "image_url"});
+ part["image_url"] = image_url;
+ Some(part)
}
-// Recognizes common raw image shapes emitted by Anthropic and Responses.
-fn openai_raw_image_part(raw: &Value) -> Option<Value> {
- let object = raw.as_object()?;
- let object = if object.get("type").and_then(Value::as_str) == Some("image") {
- let source = object.get("source").and_then(Value::as_object)?;
- if !matches!(
- source.get("type").and_then(Value::as_str),
- Some("base64" | "url")
- ) {
- return None;
- }
- source
- } else {
- object
- };
- if let Some(url) = object.get("url").and_then(Value::as_str) {
- return Some(json!({"type": "image_url", "image_url": {"url": url}}));
- }
- if let Some(url) = object.get("image_url").and_then(Value::as_str) {
- return Some(json!({"type": "image_url", "image_url": {"url": url}}));
- }
- let data = object.get("data").and_then(Value::as_str)?;
- let media_type = object
- .get("media_type")
- .and_then(Value::as_str)
- .unwrap_or("application/octet-stream");
- Some(json!({
- "type": "image_url",
- "image_url": {"url": format!("data:{media_type};base64,{data}")},
- }))
-}
-
-// Converts image sources to deterministic text fallback content.
-fn image_source_text(source: &ImageSource) -> String {
- match source {
- ImageSource::Url { url, detail } => json_string(&json!({
- "url": url,
- "detail": detail,
- })),
- ImageSource::Base64 { media_type, data } => json_string(&json!({
- "media_type": media_type,
- "data": data,
- })),
- ImageSource::Raw(raw) => json_string(raw),
- }
-}
-
-// Maps IR file sources to OpenAI Chat file content parts when possible.
fn openai_file_part(source: &FileSource) -> Option<Value> {
- match source {
- FileSource::FileId(file_id) => Some(json!({"type": "file", "file": {"file_id": file_id}})),
- FileSource::FileData { data, filename } => {
- let mut file = json!({"file_data": data});
- if let Some(filename) = filename {
- file["filename"] = Value::String(filename.clone());
- }
- Some(json!({"type": "file", "file": file}))
- }
- FileSource::Raw(raw) => openai_raw_file_part(raw),
- }
-}
-
-// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.
-fn openai_raw_file_part(raw: &Value) -> Option<Value> {
- let block = raw.as_object()?;
- if block.get("type").and_then(Value::as_str) != Some("document") {
- return None;
- }
- let source = block.get("source").and_then(Value::as_object)?;
- if source.get("type").and_then(Value::as_str) != Some("base64") {
- return None;
- }
- let data = source.get("data").and_then(Value::as_str)?;
- let mut file = json!({"file_data": data});
- if let Some(title) = block.get("title").and_then(Value::as_str) {
- file["filename"] = Value::String(title.to_string());
- }
- Some(json!({"type": "file", "file": file}))
-}
-
-// Converts file sources to deterministic text fallback content.
-fn file_source_text(source: &FileSource) -> String {
- match source {
- FileSource::FileId(file_id) => json_string(&json!({"file_id": file_id})),
- FileSource::FileData { data, filename } => json_string(&json!({
- "file_data": data,
- "filename": filename,
- })),
- FileSource::Raw(raw) => json_string(raw),
- }
+ let mut part = json!({"type": "file"});
+ part["file"] = Value::Object(file_payload(source)?);
+ Some(part)
}
// Converts unsupported media sources to deterministic text fallback content.
diff --git a/crates/switchyard-translation/src/codecs/openai_media.rs b/crates/switchyard-translation/src/codecs/openai_media.rs
new file mode 100644
index 000000000..cd959abfc
--- /dev/null
+++ b/crates/switchyard-translation/src/codecs/openai_media.rs
@@ -0,0 +1,133 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Image and file payloads shared by the OpenAI Chat and Responses codecs.
+
+use serde_json::{Map, Value, json};
+
+use crate::llm::{FileSource, ImageSource};
+use crate::util::json_string;
+
+pub(super) struct ImagePayload {
+ pub(super) url: String,
+ pub(super) detail: Option<String>,
+}
+
+pub(super) fn image_payload(source: &ImageSource) -> Option<ImagePayload> {
+ match source {
+ ImageSource::Url { url, detail } => Some(ImagePayload {
+ url: url.clone(),
+ detail: detail.clone(),
+ }),
+ ImageSource::Base64 { media_type, data } => {
+ media_type.as_ref().map(|media_type| ImagePayload {
+ url: format!("data:{media_type};base64,{data}"),
+ detail: None,
+ })
+ }
+ ImageSource::Raw(raw) => raw_image_payload(raw),
+ }
+}
+
+// Recognizes common raw image shapes emitted by Anthropic and Responses.
+fn raw_image_payload(raw: &Value) -> Option<ImagePayload> {
+ let object = raw.as_object()?;
+ let object = if object.get("type").and_then(Value::as_str) == Some("image") {
+ let source = object.get("source").and_then(Value::as_object)?;
+ if !matches!(
+ source.get("type").and_then(Value::as_str),
+ Some("base64" | "url")
+ ) {
+ return None;
+ }
+ source
+ } else {
+ object
+ };
+ if let Some(url) = object.get("url").and_then(Value::as_str) {
+ return Some(ImagePayload {
+ url: url.to_string(),
+ detail: None,
+ });
+ }
+ if let Some(url) = object.get("image_url").and_then(Value::as_str) {
+ return Some(ImagePayload {
+ url: url.to_string(),
+ detail: None,
+ });
+ }
+ let data = object.get("data").and_then(Value::as_str)?;
+ let media_type = object
+ .get("media_type")
+ .and_then(Value::as_str)
+ .unwrap_or("application/octet-stream");
+ Some(ImagePayload {
+ url: format!("data:{media_type};base64,{data}"),
+ detail: None,
+ })
+}
+
+pub(super) fn image_source_text(source: &ImageSource) -> String {
+ match source {
+ ImageSource::Url { url, detail } => json_string(&json!({
+ "url": url,
+ "detail": detail,
+ })),
+ ImageSource::Base64 { media_type, data } => json_string(&json!({
+ "media_type": media_type,
+ "data": data,
+ })),
+ ImageSource::Raw(raw) => json_string(raw),
+ }
+}
+
+pub(super) fn file_payload(source: &FileSource) -> Option<Map<String, Value>> {
+ match source {
+ FileSource::FileId(file_id) => {
+ let mut payload = Map::new();
+ payload.insert("file_id".to_string(), Value::String(file_id.to_string()));
+ Some(payload)
+ }
+ FileSource::FileData { data, filename } => {
+ Some(file_data_payload(data, filename.as_deref()))
+ }
+ FileSource::Raw(raw) => raw_file_payload(raw),
+ }
+}
+
+fn file_data_payload(data: &str, filename: Option<&str>) -> Map<String, Value> {
+ let mut payload = Map::new();
+ payload.insert("file_data".to_string(), Value::String(data.to_string()));
+ if let Some(filename) = filename {
+ payload.insert("filename".to_string(), Value::String(filename.to_string()));
+ }
+ payload
+}
+
+// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.
+fn raw_file_payload(raw: &Value) -> Option<Map<String, Value>> {
+ let block = raw.as_object()?;
+ if block.get("type").and_then(Value::as_str) != Some("document") {
+ return None;
+ }
+ let source = block.get("source").and_then(Value::as_object)?;
+ if source.get("type").and_then(Value::as_str) != Some("base64") {
+ return None;
+ }
+ let data = source.get("data").and_then(Value::as_str)?;
+ Some(file_data_payload(
+ data,
+ block.get("title").and_then(Value::as_str),
+ ))
+}
+
+pub(super) fn file_source_text(source: &FileSource) -> String {
+ match source {
+ FileSource::FileId(file_id) => json_string(&json!({"file_id": file_id})),
+ FileSource::FileData { data, filename } => json_string(&json!({
+ "file_data": data,
+ "filename": filename,
+ })),
+ FileSource::Raw(raw) => json_string(raw),
+ }
+}
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index 3db153209..c0a247857 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -12,6 +12,9 @@ use crate::codecs::common::{
is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks,
};
use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
+use crate::codecs::openai_media::{
+ ImagePayload, file_payload, file_source_text, image_payload, image_source_text,
+};
use crate::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
};
@@ -19,9 +22,9 @@ use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::{FormatId, WireFormat};
use crate::llm::{
- AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, MediaSource, Message, OutputParams,
- ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,
- ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,
+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,
+ SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,
};
use crate::policy::{DeterministicIdPolicy, TranslationPolicy};
use crate::util::{
@@ -1407,9 +1410,17 @@ fn encode_responses_content(
ContentBlock::Refusal { text } => {
blocks.push(json!({"type": "refusal", "refusal": text}));
}
- ContentBlock::Image { source } => {
- blocks.push(json!({"type": "input_image", "image_url": source}));
- }
+ ContentBlock::Image { source } => match responses_image_part(source) {
+ Some(part) => blocks.push(part),
+ None => {
+ push_lossy(
+ diagnostics,
+ policy,
+ "Responses codec could not map image content",
+ )?;
+ blocks.push(json!({"type": "input_text", "text": image_source_text(source)}));
+ }
+ },
ContentBlock::Audio { source } => blocks.push(match source {
MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),
MediaSource::Url { url, media_type } => json!({
@@ -1434,9 +1445,17 @@ fn encode_responses_content(
"video": {"media_type": media_type, "data": data},
}),
}),
- ContentBlock::File { source } => {
- blocks.push(json!({"type": "input_file", "file": source}));
- }
+ ContentBlock::File { source } => match responses_file_part(source) {
+ Some(part) => blocks.push(part),
+ None => {
+ push_lossy(
+ diagnostics,
+ policy,
+ "Responses codec could not map file content",
+ )?;
+ blocks.push(json!({"type": "input_text", "text": file_source_text(source)}));
+ }
+ },
ContentBlock::Unknown { raw, .. } => {
push_lossy(
diagnostics,
@@ -1453,6 +1472,22 @@ fn encode_responses_content(
Ok(Value::Array(blocks))
}
+fn responses_image_part(source: &ImageSource) -> Option<Value> {
+ let ImagePayload { url, detail } = image_payload(source)?;
+ let mut part = json!({"type": "input_image"});
+ part["image_url"] = Value::String(url);
+ if let Some(detail) = detail {
+ part["detail"] = Value::String(detail);
+ }
+ Some(part)
+}
+
+fn responses_file_part(source: &FileSource) -> Option<Value> {
+ let mut part = file_payload(source)?;
+ part.insert("type".to_string(), Value::String("input_file".to_string()));
+ Some(Value::Object(part))
+}
+
// Encodes normalized tool definitions into Responses tool JSON.
// Encodes normalized tools as Responses entries.
//
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index 0c3565cb1..0129a4971 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -2884,3 +2884,112 @@ fn responses_parallel_custom_tool_calls_pair_with_their_outputs() -> TestResult
);
Ok(())
}
+
+// Verifies Chat image and file parts encode as wire-valid Responses input parts,
+// not serialized IR enums (image_url must be a string and file fields must be flat).
+#[test]
+fn openai_chat_image_and_file_parts_translate_to_valid_responses_input() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-4o",
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Describe these."},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,YQ==", "detail": "high"}
+ },
+ {"type": "file", "file": {"file_id": "file_123"}},
+ {
+ "type": "file",
+ "file": {"file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}
+ }
+ ]
+ }]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(
+ output["input"][0]["content"],
+ json!([
+ {"type": "input_text", "text": "Describe these."},
+ {
+ "type": "input_image",
+ "image_url": "data:image/png;base64,YQ==",
+ "detail": "high"
+ },
+ {"type": "input_file", "file_id": "file_123"},
+ {
+ "type": "input_file",
+ "file_data": "ZG9jdW1lbnQ=",
+ "filename": "report.pdf"
+ }
+ ])
+ );
+ Ok(())
+}
+
+// Verifies Anthropic base64 media becomes valid Responses image and file parts.
+#[test]
+fn anthropic_base64_media_translates_to_responses() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "claude-sonnet-4-20250514",
+ "max_tokens": 64,
+ "messages": [{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What is this?"},
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "aW1hZ2U="
+ }
+ },
+ {
+ "type": "document",
+ "title": "report.pdf",
+ "source": {
+ "type": "base64",
+ "media_type": "application/pdf",
+ "data": "ZG9jdW1lbnQ="
+ }
+ }
+ ]
+ }]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::AnthropicMessages,
+ WireFormat::OpenAiResponses,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(
+ output["input"][0]["content"],
+ json!([
+ {"type": "input_text", "text": "What is this?"},
+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="},
+ {
+ "type": "input_file",
+ "file_data": "ZG9jdW1lbnQ=",
+ "filename": "report.pdf"
+ }
+ ])
+ );
+ Ok(())
+}