Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: encode Responses image and file inputs as wire shapes, not IR enums (#530)
Jeremy Schoemakerf27abd664cb8
diff --git a/crates/switchyard-translation/src/codecs/mod.rs b/crates/switchyard-translation/src/codecs/mod.rsindex 78d3a007c..16b1bda80 100644--- a/crates/switchyard-translation/src/codecs/mod.rs+++ b/crates/switchyard-translation/src/codecs/mod.rs@@ -6,6 +6,7 @@pub mod anthropic;pub(crate) mod common;pub mod openai_chat;+mod openai_media;pub mod responses;pub mod stream;diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rsindex 988cd9768..bdbd4b60a 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs@@ -9,6 +9,9 @@ use crate::codecs::common::{first_nonempty_string, is_known_role_name, provider_extensions, reasoning_text_from_blocks,reasoning_text_from_details, text_from_blocks,};+use crate::codecs::openai_media::{+ ImagePayload, file_payload, file_source_text, image_payload, image_source_text,+};use crate::codecs::{DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,};@@ -1066,116 +1069,22 @@ fn openai_text_part(text: &str) -> Value {json!({"type": "text", "text": text})}-// Maps IR image sources to OpenAI Chat image content parts when possible.fn openai_image_part(source: &ImageSource) -> Option<Value> {- match source {- ImageSource::Url { url, detail } => {- let mut image_url = json!({"url": url});- if let Some(detail) = detail {- image_url["detail"] = Value::String(detail.clone());- }- Some(json!({"type": "image_url", "image_url": image_url}))- }- ImageSource::Base64 { media_type, data } => media_type.as_ref().map(|media_type| {- json!({- "type": "image_url",- "image_url": {"url": format!("data:{media_type};base64,{data}")},- })- }),- ImageSource::Raw(raw) => openai_raw_image_part(raw),+ let ImagePayload { url, detail } = image_payload(source)?;+ let mut image_url = json!({});+ image_url["url"] = Value::String(url);+ if let Some(detail) = detail {+ image_url["detail"] = Value::String(detail);}+ let mut part = json!({"type": "image_url"});+ part["image_url"] = image_url;+ Some(part)}-// Recognizes common raw image shapes emitted by Anthropic and Responses.-fn openai_raw_image_part(raw: &Value) -> Option<Value> {- let object = raw.as_object()?;- let object = if object.get("type").and_then(Value::as_str) == Some("image") {- let source = object.get("source").and_then(Value::as_object)?;- if !matches!(- source.get("type").and_then(Value::as_str),- Some("base64" | "url")- ) {- return None;- }- source- } else {- object- };- if let Some(url) = object.get("url").and_then(Value::as_str) {- return Some(json!({"type": "image_url", "image_url": {"url": url}}));- }- if let Some(url) = object.get("image_url").and_then(Value::as_str) {- return Some(json!({"type": "image_url", "image_url": {"url": url}}));- }- let data = object.get("data").and_then(Value::as_str)?;- let media_type = object- .get("media_type")- .and_then(Value::as_str)- .unwrap_or("application/octet-stream");- Some(json!({- "type": "image_url",- "image_url": {"url": format!("data:{media_type};base64,{data}")},- }))-}--// Converts image sources to deterministic text fallback content.-fn image_source_text(source: &ImageSource) -> String {- match source {- ImageSource::Url { url, detail } => json_string(&json!({- "url": url,- "detail": detail,- })),- ImageSource::Base64 { media_type, data } => json_string(&json!({- "media_type": media_type,- "data": data,- })),- ImageSource::Raw(raw) => json_string(raw),- }-}--// Maps IR file sources to OpenAI Chat file content parts when possible.fn openai_file_part(source: &FileSource) -> Option<Value> {- match source {- FileSource::FileId(file_id) => Some(json!({"type": "file", "file": {"file_id": file_id}})),- FileSource::FileData { data, filename } => {- let mut file = json!({"file_data": data});- if let Some(filename) = filename {- file["filename"] = Value::String(filename.clone());- }- Some(json!({"type": "file", "file": file}))- }- FileSource::Raw(raw) => openai_raw_file_part(raw),- }-}--// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.-fn openai_raw_file_part(raw: &Value) -> Option<Value> {- let block = raw.as_object()?;- if block.get("type").and_then(Value::as_str) != Some("document") {- return None;- }- let source = block.get("source").and_then(Value::as_object)?;- if source.get("type").and_then(Value::as_str) != Some("base64") {- return None;- }- let data = source.get("data").and_then(Value::as_str)?;- let mut file = json!({"file_data": data});- if let Some(title) = block.get("title").and_then(Value::as_str) {- file["filename"] = Value::String(title.to_string());- }- Some(json!({"type": "file", "file": file}))-}--// Converts file sources to deterministic text fallback content.-fn file_source_text(source: &FileSource) -> String {- match source {- FileSource::FileId(file_id) => json_string(&json!({"file_id": file_id})),- FileSource::FileData { data, filename } => json_string(&json!({- "file_data": data,- "filename": filename,- })),- FileSource::Raw(raw) => json_string(raw),- }+ let mut part = json!({"type": "file"});+ part["file"] = Value::Object(file_payload(source)?);+ Some(part)}// Converts unsupported media sources to deterministic text fallback content.diff --git a/crates/switchyard-translation/src/codecs/openai_media.rs b/crates/switchyard-translation/src/codecs/openai_media.rsnew file mode 100644index 000000000..cd959abfc--- /dev/null+++ b/crates/switchyard-translation/src/codecs/openai_media.rs@@ -0,0 +1,133 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Image and file payloads shared by the OpenAI Chat and Responses codecs.++use serde_json::{Map, Value, json};++use crate::llm::{FileSource, ImageSource};+use crate::util::json_string;++pub(super) struct ImagePayload {+ pub(super) url: String,+ pub(super) detail: Option<String>,+}++pub(super) fn image_payload(source: &ImageSource) -> Option<ImagePayload> {+ match source {+ ImageSource::Url { url, detail } => Some(ImagePayload {+ url: url.clone(),+ detail: detail.clone(),+ }),+ ImageSource::Base64 { media_type, data } => {+ media_type.as_ref().map(|media_type| ImagePayload {+ url: format!("data:{media_type};base64,{data}"),+ detail: None,+ })+ }+ ImageSource::Raw(raw) => raw_image_payload(raw),+ }+}++// Recognizes common raw image shapes emitted by Anthropic and Responses.+fn raw_image_payload(raw: &Value) -> Option<ImagePayload> {+ let object = raw.as_object()?;+ let object = if object.get("type").and_then(Value::as_str) == Some("image") {+ let source = object.get("source").and_then(Value::as_object)?;+ if !matches!(+ source.get("type").and_then(Value::as_str),+ Some("base64" | "url")+ ) {+ return None;+ }+ source+ } else {+ object+ };+ if let Some(url) = object.get("url").and_then(Value::as_str) {+ return Some(ImagePayload {+ url: url.to_string(),+ detail: None,+ });+ }+ if let Some(url) = object.get("image_url").and_then(Value::as_str) {+ return Some(ImagePayload {+ url: url.to_string(),+ detail: None,+ });+ }+ let data = object.get("data").and_then(Value::as_str)?;+ let media_type = object+ .get("media_type")+ .and_then(Value::as_str)+ .unwrap_or("application/octet-stream");+ Some(ImagePayload {+ url: format!("data:{media_type};base64,{data}"),+ detail: None,+ })+}++pub(super) fn image_source_text(source: &ImageSource) -> String {+ match source {+ ImageSource::Url { url, detail } => json_string(&json!({+ "url": url,+ "detail": detail,+ })),+ ImageSource::Base64 { media_type, data } => json_string(&json!({+ "media_type": media_type,+ "data": data,+ })),+ ImageSource::Raw(raw) => json_string(raw),+ }+}++pub(super) fn file_payload(source: &FileSource) -> Option<Map<String, Value>> {+ match source {+ FileSource::FileId(file_id) => {+ let mut payload = Map::new();+ payload.insert("file_id".to_string(), Value::String(file_id.to_string()));+ Some(payload)+ }+ FileSource::FileData { data, filename } => {+ Some(file_data_payload(data, filename.as_deref()))+ }+ FileSource::Raw(raw) => raw_file_payload(raw),+ }+}++fn file_data_payload(data: &str, filename: Option<&str>) -> Map<String, Value> {+ let mut payload = Map::new();+ payload.insert("file_data".to_string(), Value::String(data.to_string()));+ if let Some(filename) = filename {+ payload.insert("filename".to_string(), Value::String(filename.to_string()));+ }+ payload+}++// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.+fn raw_file_payload(raw: &Value) -> Option<Map<String, Value>> {+ let block = raw.as_object()?;+ if block.get("type").and_then(Value::as_str) != Some("document") {+ return None;+ }+ let source = block.get("source").and_then(Value::as_object)?;+ if source.get("type").and_then(Value::as_str) != Some("base64") {+ return None;+ }+ let data = source.get("data").and_then(Value::as_str)?;+ Some(file_data_payload(+ data,+ block.get("title").and_then(Value::as_str),+ ))+}++pub(super) fn file_source_text(source: &FileSource) -> String {+ match source {+ FileSource::FileId(file_id) => json_string(&json!({"file_id": file_id})),+ FileSource::FileData { data, filename } => json_string(&json!({+ "file_data": data,+ "filename": filename,+ })),+ FileSource::Raw(raw) => json_string(raw),+ }+}diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rsindex 3db153209..c0a247857 100644--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs@@ -12,6 +12,9 @@ use crate::codecs::common::{is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks,};use crate::codecs::openai_chat::{decode_file_source, decode_image_source};+use crate::codecs::openai_media::{+ ImagePayload, file_payload, file_source_text, image_payload, image_source_text,+};use crate::codecs::{DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,};@@ -19,9 +22,9 @@ use crate::diagnostic::TranslationDiagnostic;use crate::error::{Result, TranslationError};use crate::format::{FormatId, WireFormat};use crate::llm::{- AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, MediaSource, Message, OutputParams,- ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason,- ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,+ AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest,+ MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role,+ SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage,};use crate::policy::{DeterministicIdPolicy, TranslationPolicy};use crate::util::{@@ -1407,9 +1410,17 @@ fn encode_responses_content(ContentBlock::Refusal { text } => {blocks.push(json!({"type": "refusal", "refusal": text}));}- ContentBlock::Image { source } => {- blocks.push(json!({"type": "input_image", "image_url": source}));- }+ ContentBlock::Image { source } => match responses_image_part(source) {+ Some(part) => blocks.push(part),+ None => {+ push_lossy(+ diagnostics,+ policy,+ "Responses codec could not map image content",+ )?;+ blocks.push(json!({"type": "input_text", "text": image_source_text(source)}));+ }+ },ContentBlock::Audio { source } => blocks.push(match source {MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}),MediaSource::Url { url, media_type } => json!({@@ -1434,9 +1445,17 @@ fn encode_responses_content("video": {"media_type": media_type, "data": data},}),}),- ContentBlock::File { source } => {- blocks.push(json!({"type": "input_file", "file": source}));- }+ ContentBlock::File { source } => match responses_file_part(source) {+ Some(part) => blocks.push(part),+ None => {+ push_lossy(+ diagnostics,+ policy,+ "Responses codec could not map file content",+ )?;+ blocks.push(json!({"type": "input_text", "text": file_source_text(source)}));+ }+ },ContentBlock::Unknown { raw, .. } => {push_lossy(diagnostics,@@ -1453,6 +1472,22 @@ fn encode_responses_content(Ok(Value::Array(blocks))}+fn responses_image_part(source: &ImageSource) -> Option<Value> {+ let ImagePayload { url, detail } = image_payload(source)?;+ let mut part = json!({"type": "input_image"});+ part["image_url"] = Value::String(url);+ if let Some(detail) = detail {+ part["detail"] = Value::String(detail);+ }+ Some(part)+}++fn responses_file_part(source: &FileSource) -> Option<Value> {+ let mut part = file_payload(source)?;+ part.insert("type".to_string(), Value::String("input_file".to_string()));+ Some(Value::Object(part))+}+// Encodes normalized tool definitions into Responses tool JSON.// Encodes normalized tools as Responses entries.//diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rsindex 0c3565cb1..0129a4971 100644--- a/crates/switchyard-translation/tests/request_translation.rs+++ b/crates/switchyard-translation/tests/request_translation.rs@@ -2884,3 +2884,112 @@ fn responses_parallel_custom_tool_calls_pair_with_their_outputs() -> TestResult);Ok(())}++// Verifies Chat image and file parts encode as wire-valid Responses input parts,+// not serialized IR enums (image_url must be a string and file fields must be flat).+#[test]+fn openai_chat_image_and_file_parts_translate_to_valid_responses_input() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "gpt-4o",+ "messages": [{+ "role": "user",+ "content": [+ {"type": "text", "text": "Describe these."},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,YQ==", "detail": "high"}+ },+ {"type": "file", "file": {"file_id": "file_123"}},+ {+ "type": "file",+ "file": {"file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"}+ }+ ]+ }]+ });++ let output = engine+ .translate_request(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(+ output["input"][0]["content"],+ json!([+ {"type": "input_text", "text": "Describe these."},+ {+ "type": "input_image",+ "image_url": "data:image/png;base64,YQ==",+ "detail": "high"+ },+ {"type": "input_file", "file_id": "file_123"},+ {+ "type": "input_file",+ "file_data": "ZG9jdW1lbnQ=",+ "filename": "report.pdf"+ }+ ])+ );+ Ok(())+}++// Verifies Anthropic base64 media becomes valid Responses image and file parts.+#[test]+fn anthropic_base64_media_translates_to_responses() -> TestResult {+ let engine = TranslationEngine::default();+ let body = json!({+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 64,+ "messages": [{+ "role": "user",+ "content": [+ {"type": "text", "text": "What is this?"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "aW1hZ2U="+ }+ },+ {+ "type": "document",+ "title": "report.pdf",+ "source": {+ "type": "base64",+ "media_type": "application/pdf",+ "data": "ZG9jdW1lbnQ="+ }+ }+ ]+ }]+ });++ let output = engine+ .translate_request(+ WireFormat::AnthropicMessages,+ WireFormat::OpenAiResponses,+ &body,+ &TranslationPolicy::default(),+ )?+ .body;++ assert_eq!(+ output["input"][0]["content"],+ json!([+ {"type": "input_text", "text": "What is this?"},+ {"type": "input_image", "image_url": "data:image/png;base64,aW1hZ2U="},+ {+ "type": "input_file",+ "file_data": "ZG9jdW1lbnQ=",+ "filename": "report.pdf"+ }+ ])+ );+ Ok(())+}