Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): harden Responses compatibility (#416)

Alex Steinerac7a16b31381
diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs
index a31fda0a8..db5ccab95 100644
--- a/crates/switchyard-translation/src/codecs/responses/buffered.rs
+++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs
@@ -327,8 +327,22 @@ fn decode_responses_input(
)?;
continue;
};
- match item.get("type").and_then(Value::as_str) {
- Some("message") => {
+ let item_type = match item.get("type") {
+ Some(Value::String(item_type)) => Some(item_type.as_str()),
+ Some(_) => {
+ return Err(TranslationError::InvalidType {
+ path: format!("$.input[{index}].type"),
+ expected: "a string",
+ });
+ }
+ None => None,
+ };
+ let is_message = item_type == Some("message")
+ || (item_type.is_none()
+ && item.contains_key("role")
+ && item.contains_key("content"));
+ match item_type {
+ _ if is_message => {
let role = request_role_from_responses(
item.get("role").and_then(Value::as_str),
&format!("$.input[{index}].role"),
@@ -423,6 +437,13 @@ fn decode_responses_input(
is_error: None,
});
}
+ None => {
+ return Err(TranslationError::InvalidValue {
+ path: format!("$.input[{index}].type"),
+ message: "missing type discriminator on a non-message input item"
+ .to_string(),
+ });
+ }
_ => {
let message = Message {
role: Role::User,
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index b076e3300..5c60f9b50 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -3,6 +3,7 @@
//! Streaming codec for OpenAI Responses API events.
+use serde::Serialize;
use serde_json::{Value, json};
use crate::LlmResponseChunk;
@@ -34,14 +35,65 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
state: &mut StreamTranslationState,
event: LlmResponseChunk,
) -> Vec<Value> {
- encode_responses_stream(state, event)
+ let events = encode_responses_stream(state, event);
+ add_sequence_numbers(state, events)
+ }
+
+ fn observe_replayed_event(
+ &self,
+ state: &mut StreamTranslationState,
+ raw: &Value,
+ normalized: Vec<LlmResponseChunk>,
+ ) {
+ let replayed_terminal = normalized
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
+ // Exact replay emits `raw` once. Normalized encodings only advance codec state;
+ // their generated events are discarded and must not create sequence-number gaps.
+ for chunk in normalized {
+ drop(encode_responses_stream(state, chunk));
+ }
+ state.response_sequence_number = raw
+ .get("sequence_number")
+ .and_then(Value::as_u64)
+ .map_or(state.response_sequence_number.saturating_add(1), |number| {
+ number.saturating_add(1)
+ });
+ if replayed_terminal {
+ state.finished = true;
+ }
}
fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {
- finish_responses_stream(state)
+ let events = finish_responses_stream(state);
+ add_sequence_numbers(state, events)
}
}
+/// Required fields shared by Responses stream snapshots.
+#[derive(Serialize)]
+struct ResponsesStreamResponse {
+ id: String,
+ object: &'static str,
+ created_at: u64,
+ completed_at: Option<u64>,
+ error: Option<Value>,
+ incomplete_details: Option<Value>,
+ instructions: Option<Value>,
+ metadata: Option<Value>,
+ model: String,
+ output: Vec<Value>,
+ parallel_tool_calls: bool,
+ frequency_penalty: Option<f64>,
+ presence_penalty: Option<f64>,
+ status: &'static str,
+ temperature: Option<f64>,
+ tool_choice: &'static str,
+ tools: Vec<Value>,
+ top_p: Option<f64>,
+ usage: Value,
+}
+
// Decodes one OpenAI Responses event into neutral streaming events.
fn decode_responses_stream(
state: &mut StreamTranslationState,
@@ -228,6 +280,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
"output_index": output_index,
"item": {
"type": "message",
+ "id": format!("msg_{output_index}"),
"role": "assistant",
"status": status,
"content": [{"type": "output_text", "text": state.response_text}],
@@ -269,6 +322,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
output_index,
json!({
"type": "message",
+ "id": format!("msg_{output_index}"),
"role": "assistant",
"status": status,
"content": [{"type": "output_text", "text": state.response_text}],
@@ -310,15 +364,7 @@ fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
out.push(json!({
"type": event_type,
- "response": {
- "id": responses_id(state),
- "object": "response",
- "status": status,
- "incomplete_details": incomplete_details,
- "model": target_model_or_source_model(state),
- "output": output,
- "usage": responses_usage_value(&state.usage),
- },
+ "response": responses_stream_response(state, status, incomplete_details, output),
}));
state.finished = true;
out
@@ -397,17 +443,54 @@ fn ensure_responses_created(state: &mut StreamTranslationState) -> Vec<Value> {
state.response_created = true;
vec![json!({
"type": "response.created",
- "response": {
- "id": responses_id(state),
- "object": "response",
- "status": "in_progress",
- "model": target_model_or_source_model(state),
- "output": [],
- "usage": responses_usage_value(&state.usage),
- },
+ "response": responses_stream_response(state, "in_progress", None, Vec::new()),
})]
}
+// Builds a schema-complete Responses snapshot for strict generated clients.
+fn responses_stream_response(
+ state: &StreamTranslationState,
+ status: &'static str,
+ incomplete_details: Option<Value>,
+ output: Vec<Value>,
+) -> ResponsesStreamResponse {
+ ResponsesStreamResponse {
+ id: responses_id(state),
+ object: "response",
+ created_at: 0,
+ completed_at: None,
+ error: None,
+ incomplete_details,
+ instructions: None,
+ metadata: None,
+ model: target_model_or_source_model(state),
+ output,
+ parallel_tool_calls: true,
+ frequency_penalty: None,
+ presence_penalty: None,
+ status,
+ temperature: None,
+ tool_choice: "auto",
+ tools: Vec::new(),
+ top_p: None,
+ usage: responses_usage_value(&state.usage),
+ }
+}
+
+// Assigns monotonically increasing sequence numbers to generated Responses events.
+fn add_sequence_numbers(state: &mut StreamTranslationState, mut events: Vec<Value>) -> Vec<Value> {
+ for event in &mut events {
+ if let Some(object) = event.as_object_mut() {
+ object.insert(
+ "sequence_number".to_string(),
+ Value::from(state.response_sequence_number),
+ );
+ state.response_sequence_number = state.response_sequence_number.saturating_add(1);
+ }
+ }
+ events
+}
+
// Accumulates assistant text and emits Responses text delta events.
fn encode_responses_text_delta(state: &mut StreamTranslationState, text: String) -> Vec<Value> {
let mut out = ensure_responses_created(state);
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index bcd586a27..991b45059 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -58,6 +58,7 @@ pub struct StreamTranslationState {
pub(crate) response_reasoning_output_index: Option<usize>,
pub(crate) response_reasoning_text: String,
pub(crate) next_response_output_index: usize,
+ pub(crate) response_sequence_number: u64,
pub(crate) reasoning_block_index: Option<usize>,
pub(crate) reasoning_block_started: bool,
diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs
index b7c4121a9..9abe14a89 100644
--- a/crates/switchyard-translation/tests/request_translation.rs
+++ b/crates/switchyard-translation/tests/request_translation.rs
@@ -732,6 +732,52 @@ fn responses_unknown_input_item_is_preserved_for_openai_chat() -> TestResult {
Ok(())
}
+// Responses accepts message-shaped input items without an explicit discriminator.
+#[test]
+fn responses_input_message_without_type_translates_normally() -> TestResult {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-4",
+ "input": [{"role": "user", "content": "hello"}]
+ });
+
+ let output = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )?
+ .body;
+
+ assert_eq!(
+ output["messages"],
+ json!([{"role": "user", "content": "hello"}])
+ );
+ Ok(())
+}
+
+// A discriminator-less object that is not message-shaped must not silently become prompt text.
+#[test]
+fn responses_input_without_type_or_message_shape_is_rejected() {
+ let engine = TranslationEngine::default();
+ let body = json!({
+ "model": "gpt-4",
+ "input": [{"payload": "ambiguous"}]
+ });
+
+ let error = engine
+ .translate_request(
+ WireFormat::OpenAiResponses,
+ WireFormat::OpenAiChat,
+ &body,
+ &TranslationPolicy::default(),
+ )
+ .expect_err("ambiguous input item should be rejected");
+
+ assert!(error.to_string().contains("$.input[0].type"));
+}
+
// Verifies orphan Responses tool outputs degrade to readable user text.
#[test]
fn responses_orphan_function_call_output_degrades_to_user_text_for_openai_chat() -> TestResult {
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 98cf5f1ce..31e2ffb58 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -7,7 +7,7 @@ pub mod common;
use pretty_assertions::assert_eq;
use serde_json::json;
-use switchyard_protocol::{ResponseAccumulator, StopReason};
+use switchyard_protocol::{LlmResponseStreamEvent, ResponseAccumulator, StopReason};
use switchyard_translation::{
LlmResponseChunk, StreamTranslationState, TranslationEngine, WireFormat, decode_stream_event,
};
@@ -186,6 +186,55 @@ fn replayed_nonterminal_event_advances_encoder_state_before_finish() -> TestResu
Ok(())
}
+// Exact replay advances sequencing by the one raw event actually emitted, not discarded
+// synthetic events produced while advancing encoder state.
+#[test]
+fn responses_replay_without_sequence_advances_by_one_emitted_event() -> TestResult {
+ let engine = TranslationEngine::default();
+ let format = WireFormat::OpenAiResponses;
+ let raw = json!({
+ "type": "response.output_item.added",
+ "output_index": 0,
+ "item": {
+ "type": "function_call",
+ "id": "fc_0",
+ "call_id": "call_0",
+ "name": "bash",
+ "arguments": ""
+ }
+ });
+ let replayed = LlmResponseStreamEvent::preserved(
+ format,
+ raw.clone(),
+ vec![LlmResponseChunk::ToolCallDelta {
+ index: 0,
+ id: Some("call_0".to_string()),
+ name: Some("bash".to_string()),
+ arguments_delta: None,
+ }],
+ );
+ let mut state = StreamTranslationState::new(format, format);
+
+ assert_eq!(
+ engine.encode_stream_event(&mut state, format, replayed)?,
+ vec![raw]
+ );
+ let generated = engine.encode_stream_event(
+ &mut state,
+ format,
+ LlmResponseStreamEvent::new(vec![LlmResponseChunk::ToolCallDelta {
+ index: 0,
+ id: None,
+ name: None,
+ arguments_delta: Some("{}".to_string()),
+ }]),
+ )?;
+
+ assert_eq!(generated.len(), 1);
+ assert_eq!(generated[0]["sequence_number"], 1);
+ Ok(())
+}
+
#[test]
fn replayed_anthropic_terminal_delta_finishes_with_message_stop_only() -> TestResult {
let engine = TranslationEngine::default();
@@ -1104,6 +1153,76 @@ fn openai_chat_stream_usage_without_breakdowns_still_emits_responses_usage_detai
Ok(())
}
+// Responses terminal snapshots include every field required by strict generated clients.
+#[test]
+fn responses_completed_event_is_schema_complete_and_retains_message_id() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut state =
+ StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses);
+ let chunk = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [{
+ "index": 0,
+ "delta": {"content": "hello"},
+ "finish_reason": "stop"
+ }]
+ });
+
+ let mut events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ &chunk,
+ )?;
+ events.extend(engine.finish_stream(&mut state, WireFormat::OpenAiResponses)?);
+
+ for (expected, event) in events.iter().enumerate() {
+ assert_eq!(event["sequence_number"], expected as u64);
+ }
+ let completed = events
+ .iter()
+ .find(|event| event["type"] == "response.completed")
+ .ok_or("expected response.completed")?;
+ let response = completed["response"]
+ .as_object()
+ .ok_or("completed response should be an object")?;
+ for field in [
+ "id",
+ "object",
+ "created_at",
+ "completed_at",
+ "error",
+ "incomplete_details",
+ "instructions",
+ "metadata",
+ "model",
+ "output",
+ "parallel_tool_calls",
+ "frequency_penalty",
+ "presence_penalty",
+ "status",
+ "temperature",
+ "tool_choice",
+ "tools",
+ "top_p",
+ "usage",
+ ] {
+ assert!(
+ response.contains_key(field),
+ "missing response field {field}"
+ );
+ }
+ assert_eq!(response["output"][0]["id"], "msg_0");
+ let done = events
+ .iter()
+ .find(|event| event["type"] == "response.output_item.done")
+ .ok_or("expected response.output_item.done")?;
+ assert_eq!(done["item"]["id"], "msg_0");
+ Ok(())
+}
+
// Verifies a streamed token-limit stop terminates with response.incomplete.
#[test]
fn openai_chat_length_finish_translates_to_responses_incomplete_event() -> TestResult {