Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): recover Responses terminal snapshot output (#777)

Graham King44bce73be696
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index 5ea41338c..b3a0b5f7e 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -239,6 +239,17 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
if state.finished {
return Vec::new();
}
+ if state.tool_states.values().any(|tool| {
+ tool.id.as_deref().is_none_or(str::is_empty)
+ || tool.name.as_deref().is_none_or(str::is_empty)
+ }) {
+ return encode_anthropic_stream(
+ state,
+ LlmResponseChunk::StreamError {
+ message: "Tool call ended without a non-empty ID and name".to_string(),
+ },
+ );
+ }
let mut out = Vec::new();
if !state.emitted_message_start {
out.extend(encode_anthropic_stream(
@@ -508,11 +519,11 @@ fn encode_anthropic_tool_delta(
.iter()
.any(|(&tool_index, tool)| tool_index != index && tool.started);
let tool = state.tool_states.entry(index).or_default();
- if id.is_some() {
- tool.id = id.map(|id| sanitize_anthropic_tool_use_id(&id));
+ if let Some(id) = id.filter(|id| !id.is_empty()) {
+ tool.id = Some(sanitize_anthropic_tool_use_id(&id));
}
- if name.is_some() {
- tool.name = name;
+ if let Some(name) = name.filter(|name| !name.is_empty()) {
+ tool.name = Some(name);
}
if let Some(delta) = arguments_delta {
tool.arguments.push_str(&delta);
@@ -528,7 +539,11 @@ fn encode_anthropic_tool_delta(
}
if !tool.started {
- let Some(name) = tool.name.clone() else {
+ // A published tool-use ID cannot be replaced when a later snapshot supplies it.
+ let (Some(id), Some(name)) = (
+ tool.id.as_deref().filter(|id| !id.is_empty()),
+ tool.name.as_deref().filter(|name| !name.is_empty()),
+ ) else {
return out;
};
let content_index = state.next_content_index;
@@ -541,7 +556,7 @@ fn encode_anthropic_tool_delta(
"index": content_index,
"content_block": {
"type": "tool_use",
- "id": tool.id.clone().unwrap_or_else(|| format!("toolu_{index}")),
+ "id": id,
"name": name,
"input": {},
},
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 6d7e0e581..a179d4ee7 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -134,11 +134,23 @@ fn decode_responses_stream(
.get("delta")
.and_then(Value::as_str)
.map(|text| {
+ let index = event
+ .get("output_index")
+ .and_then(Value::as_u64)
+ .unwrap_or(0) as usize;
+ let content_index = event
+ .get("content_index")
+ .and_then(Value::as_u64)
+ .unwrap_or(0) as usize;
+ state
+ .decoded_response_text
+ .entry(index)
+ .or_default()
+ .entry(content_index)
+ .or_default()
+ .push_str(text);
vec![LlmResponseChunk::TextDelta {
- index: event
- .get("output_index")
- .and_then(Value::as_u64)
- .unwrap_or(0) as usize,
+ index,
text: text.to_string(),
}]
})
@@ -247,18 +259,19 @@ fn decode_responses_stream(
}
Some("response.completed") => {
let mut out = Vec::new();
- // Some providers surface reasoning only in the final output array. Position is the
- // output index; anything already decoded is skipped by the helper.
+ // Some providers send output only in the final snapshot. Reconcile it with
+ // decoded deltas before emitting the stop, without repeating streamed content.
if let Some(items) = event
.get("response")
.and_then(|response| response.get("output"))
.and_then(Value::as_array)
{
for (position, item) in items.iter().enumerate() {
- if let Some(item) = item.as_object()
- && item.get("type").and_then(Value::as_str) == Some("reasoning")
- {
- out.extend(decode_responses_reasoning_item(item, position, state));
+ if let Some(item) = item.as_object() {
+ out.extend(decode_responses_completed_item(item, position, state));
+ if matches!(out.last(), Some(LlmResponseChunk::StreamError { .. })) {
+ return out;
+ }
}
}
}
@@ -278,12 +291,7 @@ fn decode_responses_stream(
// use for such output; the stream must too, or stop-reason-driven tool loops
// (Anthropic `tool_use`, Chat `tool_calls`) stop without running the tool.
// Carries the Anthropic spelling because every encoder already maps it.
- let has_tool_call = event
- .get("response")
- .and_then(|response| response.get("output"))
- .and_then(Value::as_array)
- .is_some_and(|items| items.iter().any(is_responses_tool_call_item));
- let reason = (has_tool_call || state.decoded_tool_call).then(|| "tool_use".to_string());
+ let reason = state.decoded_tool_call.then(|| "tool_use".to_string());
out.push(LlmResponseChunk::MessageStop { reason });
out
}
@@ -649,6 +657,17 @@ fn decode_responses_output_item_added(
return Vec::new();
}
state.decoded_tool_call = true;
+ let id = item
+ .get("call_id")
+ .or_else(|| item.get("id"))
+ .and_then(Value::as_str);
+ let name = item.get("name").and_then(Value::as_str);
+ state
+ .tool_states
+ .entry(index)
+ .or_default()
+ .has_decoded_identity |=
+ id.is_some_and(|id| !id.is_empty()) && name.is_some_and(|name| !name.is_empty());
// A freeform call's `input` becomes the single `input` argument; it is only complete on
// the done event, so nothing is emitted for it here beyond id and name.
let arguments_delta = if item_type == Some("custom_tool_call") {
@@ -669,28 +688,12 @@ fn decode_responses_output_item_added(
}
vec![LlmResponseChunk::ToolCallDelta {
index,
- id: item
- .get("call_id")
- .or_else(|| item.get("id"))
- .and_then(Value::as_str)
- .map(ToOwned::to_owned),
- name: item
- .get("name")
- .and_then(Value::as_str)
- .map(ToOwned::to_owned),
+ id: id.map(ToOwned::to_owned),
+ name: name.map(ToOwned::to_owned),
arguments_delta,
}]
}
-// Whether a Responses output item is a client tool call the consumer must run.
-fn is_responses_tool_call_item(item: &Value) -> bool {
- matches!(
- item.get("type").and_then(Value::as_str),
- Some("function_call" | "custom_tool_call")
- )
-}
-
-// Emits a final tool-call argument delta when Responses only supplies arguments at item end.
fn decode_responses_output_item_done(
event: &Value,
state: &mut StreamTranslationState,
@@ -702,10 +705,42 @@ fn decode_responses_output_item_done(
.get("output_index")
.and_then(Value::as_u64)
.unwrap_or(0) as usize;
+ decode_responses_completed_item(item, index, state)
+}
+
+fn decode_responses_completed_item(
+ item: &serde_json::Map<String, Value>,
+ index: usize,
+ state: &mut StreamTranslationState,
+) -> Vec<LlmResponseChunk> {
if item.get("type").and_then(Value::as_str) == Some("reasoning") {
return decode_responses_reasoning_item(item, index, state);
}
let item_type = item.get("type").and_then(Value::as_str);
+ if item_type == Some("message") {
+ let mut out = Vec::new();
+ if let Some(content) = item.get("content").and_then(Value::as_array) {
+ for (content_index, part) in content.iter().enumerate() {
+ if part.get("type").and_then(Value::as_str) != Some("output_text") {
+ continue;
+ }
+ if let Some(text) = part.get("text").and_then(Value::as_str) {
+ let decoded = state
+ .decoded_response_text
+ .entry(index)
+ .or_default()
+ .entry(content_index)
+ .or_default();
+ match snapshot_suffix(decoded, text) {
+ Ok(Some(text)) => out.push(LlmResponseChunk::TextDelta { index, text }),
+ Ok(None) => {}
+ Err(error) => return vec![error],
+ }
+ }
+ }
+ }
+ return out;
+ }
if item_type != Some("function_call") && item_type != Some("custom_tool_call") {
return Vec::new();
}
@@ -720,25 +755,49 @@ fn decode_responses_output_item_done(
let arguments = custom_arguments
.as_deref()
.or_else(|| item.get("arguments").and_then(Value::as_str));
- if let Some(arguments) = arguments {
- // Compared against what THIS decoder has seen. Reading the encoder's
- // `arguments` instead only deduplicates when a single state performs
- // both halves of the translation, and silently duplicates when a
- // caller buffers the stream with its own state.
- let tool = state.tool_states.entry(index).or_default();
- if !arguments.is_empty() && arguments != tool.decoded_arguments {
- tool.decoded_arguments.push_str(arguments);
- return vec![LlmResponseChunk::ToolCallDelta {
- index,
- id: None,
- name: None,
- arguments_delta: Some(arguments.to_string()),
- }];
- }
+ let tool = state.tool_states.entry(index).or_default();
+ let arguments_delta =
+ match arguments.map(|arguments| snapshot_suffix(&mut tool.decoded_arguments, arguments)) {
+ Some(Ok(delta)) => delta,
+ Some(Err(error)) => return vec![error],
+ None => None,
+ };
+ let id = item
+ .get("call_id")
+ .or_else(|| item.get("id"))
+ .and_then(Value::as_str);
+ let name = item.get("name").and_then(Value::as_str);
+ let needs_identity = !tool.has_decoded_identity;
+ tool.has_decoded_identity |=
+ id.is_some_and(|id| !id.is_empty()) && name.is_some_and(|name| !name.is_empty());
+ if needs_identity || arguments_delta.is_some() {
+ return vec![LlmResponseChunk::ToolCallDelta {
+ index,
+ id: id.filter(|_| needs_identity).map(ToOwned::to_owned),
+ name: name.filter(|_| needs_identity).map(ToOwned::to_owned),
+ arguments_delta,
+ }];
}
Vec::new()
}
+// A snapshot may extend streamed content, but cannot retract content already sent.
+fn snapshot_suffix(
+ decoded: &mut String,
+ snapshot: &str,
+) -> Result<Option<String>, LlmResponseChunk> {
+ let Some(suffix) = snapshot.strip_prefix(decoded.as_str()) else {
+ return Err(LlmResponseChunk::StreamError {
+ message: "Responses snapshot conflicts with streamed content".to_string(),
+ });
+ };
+ if suffix.is_empty() {
+ return Ok(None);
+ }
+ decoded.push_str(suffix);
+ Ok(Some(suffix.to_owned()))
+}
+
// Emits the initial Responses created event once per stream.
fn ensure_responses_created(state: &mut StreamTranslationState) -> Vec<Value> {
if state.response_created {
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index b53a0293e..3732ff2b6 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -62,6 +62,9 @@ pub struct StreamTranslationState {
/// Set once a tool call was observed while DECODING, so a terminal event that names no
/// stop reason can still report tool use.
pub(crate) decoded_tool_call: bool,
+ /// Text decoded from Responses, keyed by output index and then content index.
+ #[serde(default)]
+ pub(crate) decoded_response_text: BTreeMap<usize, BTreeMap<usize, String>>,
pub(crate) response_created: bool,
pub(crate) response_text_started: bool,
@@ -115,6 +118,8 @@ pub(crate) struct StreamToolState {
/// own state and encodes later, the field is empty and the duplicate is
/// emitted.
pub(crate) decoded_arguments: String,
+ #[serde(default)]
+ pub(crate) has_decoded_identity: bool,
pub(crate) pending_arguments: String,
pub(crate) started: bool,
pub(crate) content_index: Option<usize>,
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 474509c25..7893ff22b 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -2766,3 +2766,173 @@ fn responses_stream_empty_reasoning_delta_opens_no_summary_part() -> TestResult
assert!(types.contains(&"response.output_item.added"), "{types:?}");
Ok(())
}
+
+#[test]
+fn responses_terminal_snapshots_recover_missing_output_once() -> TestResult {
+ let engine = TranslationEngine::default();
+ let source = WireFormat::OpenAiResponses;
+ let output = json!([
+ {"type": "message", "role": "assistant", "content": [
+ {"type": "output_text", "text": "Visible answer"}
+ ]},
+ {"type": "function_call", "id": "fc_1", "call_id": "call_1",
+ "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"}
+ ]);
+ let response =
+ json!({"id": "resp_1", "model": "fixture", "status": "completed", "output": output});
+ let mut expected = engine
+ .decode_response(source, &response, &Default::default())?
+ .response;
+ if let switchyard_protocol::ContentBlock::ToolCall(call) = &mut expected.outputs[0].content[1] {
+ call.arguments = json!({"city": "Paris"});
+ }
+ for target in [WireFormat::OpenAiChat, WireFormat::AnthropicMessages] {
+ for mode in [
+ "terminal",
+ "done",
+ "partial",
+ "full",
+ "name_only",
+ "id_only",
+ ] {
+ let mut events = vec![
+ json!({"type": "response.created", "response": {"id": "resp_1", "model": "fixture"}}),
+ ];
+ if matches!(mode, "partial" | "full") {
+ events.push(
+ json!({"type": "response.output_text.delta", "output_index": 0,
+ "delta": if mode == "full" { "Visible answer" } else { "Visible " }}),
+ );
+ events.push(json!({"type": "response.output_item.added", "output_index": 1,
+ "item": {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": ""}}));
+ events.push(
+ json!({"type": "response.function_call_arguments.delta", "output_index": 1,
+ "delta": if mode == "full" { "{\"city\":\"Paris\"}" } else { "{\"city\":" }}),
+ );
+ }
+ if matches!(mode, "name_only" | "id_only") {
+ let mut item =
+ json!({"type": "function_call", "arguments": "{\"city\":\"Paris\"}"});
+ if mode == "name_only" {
+ item["name"] = json!("get_weather");
+ } else {
+ item["call_id"] = json!("call_1");
+ }
+ for event_type in ["response.output_item.added", "response.output_item.done"] {
+ events.push(json!({"type": event_type, "output_index": 1, "item": item}));
+ }
+ } else if mode != "terminal" {
+ for (index, item) in output.as_array().unwrap().iter().enumerate() {
+ events.push(json!({"type": "response.output_item.done", "output_index": index, "item": item}));
+ }
+ }
+ events.push(json!({"type": "response.completed", "response": response}));
+ let mut decoder = StreamTranslationState::new(source, target);
+ let mut encoder = StreamTranslationState::new(source, target);
+ let mut reader = StreamTranslationState::new(target, target);
+ let mut accumulator = ResponseAccumulator::new();
+ for event in events {
+ let is_incomplete_identity = matches!(mode, "name_only" | "id_only")
+ && event["type"] != "response.completed";
+ let decoded = engine.decode_stream_event(&mut decoder, source, event)?;
+ for encoded in engine.encode_stream_event(&mut encoder, target, decoded)? {
+ if target == WireFormat::AnthropicMessages && is_incomplete_identity {
+ assert_ne!(encoded["content_block"]["type"], "tool_use");
+ }
+ for chunk in decode_stream_event(&mut reader, target, &encoded) {
+ accumulator.push(chunk);
+ }
+ }
+ }
+ for encoded in engine.finish_stream(&mut encoder, target)? {
+ for chunk in decode_stream_event(&mut reader, target, &encoded) {
+ accumulator.push(chunk);
+ }
+ }
+ assert_eq!(
+ expected.outputs,
+ accumulator.finish().outputs,
+ "{target:?}: {mode}"
+ );
+ }
+ }
+ for event_type in ["response.output_item.added", "response.output_item.done"] {
+ for identity in [
+ json!({}),
+ json!({"call_id": "", "name": "get_weather"}),
+ json!({"id": "fc_1", "name": ""}),
+ ] {
+ let mut state = StreamTranslationState::new(source, WireFormat::OpenAiChat);
+ let mut item = identity;
+ item["type"] = json!("function_call");
+ let incomplete = engine.decode_stream_event(
+ &mut state,
+ source,
+ json!({"type": event_type, "output_index": 1, "item": item}),
+ )?;
+ let target = WireFormat::AnthropicMessages;
+ let mut encoder = StreamTranslationState::new(source, target);
+ let early = engine.encode_stream_event(&mut encoder, target, incomplete)?;
+ assert!(
+ early
+ .iter()
+ .all(|event| event["content_block"]["type"] != "tool_use")
+ );
+ let terminal = engine.finish_stream(&mut encoder, target)?;
+ assert_eq!(terminal.len(), 1);
+ assert_eq!(terminal[0]["type"], "error");
+ assert!(engine.finish_stream(&mut encoder, target)?.is_empty());
+ let decoded = engine.decode_stream_event(
+ &mut state,
+ source,
+ json!({"type": "response.completed", "response": response}),
+ )?;
+ assert!(
+ decoded.normalized().iter().any(|chunk| matches!(chunk,
+ LlmResponseChunk::ToolCallDelta { id: Some(id), name: Some(name), .. }
+ if id == "call_1" && name == "get_weather"
+ )),
+ "{event_type}: {item}"
+ );
+ engine.decode_stream_event(
+ &mut state,
+ source,
+ json!({"type": event_type, "output_index": 1, "item": item}),
+ )?;
+ let repeated = engine.decode_stream_event(
+ &mut state,
+ source,
+ json!({"type": "response.completed", "response": response}),
+ )?;
+ assert!(
+ !repeated
+ .normalized()
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::ToolCallDelta { .. }))
+ );
+ }
+ }
+ for delta in [
+ json!({"type": "response.output_text.delta", "output_index": 0, "delta": "Different"}),
+ json!({"type": "response.function_call_arguments.delta", "output_index": 1, "delta": "["}),
+ ] {
+ let mut state = StreamTranslationState::new(source, WireFormat::OpenAiChat);
+ engine.decode_stream_event(&mut state, source, delta)?;
+ let decoded = engine.decode_stream_event(
+ &mut state,
+ source,
+ json!({"type": "response.completed", "response": response}),
+ )?;
+ assert!(matches!(
+ decoded.normalized().last(),
+ Some(LlmResponseChunk::StreamError { .. })
+ ));
+ assert!(
+ !decoded
+ .normalized()
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }))
+ );
+ }
+ Ok(())
+}