Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): recover Responses terminal snapshot output (#777)
Graham King44bce73be696
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rsindex 5ea41338c..b3a0b5f7e 100644--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs@@ -239,6 +239,17 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {if state.finished {return Vec::new();}+ if state.tool_states.values().any(|tool| {+ tool.id.as_deref().is_none_or(str::is_empty)+ || tool.name.as_deref().is_none_or(str::is_empty)+ }) {+ return encode_anthropic_stream(+ state,+ LlmResponseChunk::StreamError {+ message: "Tool call ended without a non-empty ID and name".to_string(),+ },+ );+ }let mut out = Vec::new();if !state.emitted_message_start {out.extend(encode_anthropic_stream(@@ -508,11 +519,11 @@ fn encode_anthropic_tool_delta(.iter().any(|(&tool_index, tool)| tool_index != index && tool.started);let tool = state.tool_states.entry(index).or_default();- if id.is_some() {- tool.id = id.map(|id| sanitize_anthropic_tool_use_id(&id));+ if let Some(id) = id.filter(|id| !id.is_empty()) {+ tool.id = Some(sanitize_anthropic_tool_use_id(&id));}- if name.is_some() {- tool.name = name;+ if let Some(name) = name.filter(|name| !name.is_empty()) {+ tool.name = Some(name);}if let Some(delta) = arguments_delta {tool.arguments.push_str(&delta);@@ -528,7 +539,11 @@ fn encode_anthropic_tool_delta(}if !tool.started {- let Some(name) = tool.name.clone() else {+ // A published tool-use ID cannot be replaced when a later snapshot supplies it.+ let (Some(id), Some(name)) = (+ tool.id.as_deref().filter(|id| !id.is_empty()),+ tool.name.as_deref().filter(|name| !name.is_empty()),+ ) else {return out;};let content_index = state.next_content_index;@@ -541,7 +556,7 @@ fn encode_anthropic_tool_delta("index": content_index,"content_block": {"type": "tool_use",- "id": tool.id.clone().unwrap_or_else(|| format!("toolu_{index}")),+ "id": id,"name": name,"input": {},},diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 6d7e0e581..a179d4ee7 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -134,11 +134,23 @@ fn decode_responses_stream(.get("delta").and_then(Value::as_str).map(|text| {+ let index = event+ .get("output_index")+ .and_then(Value::as_u64)+ .unwrap_or(0) as usize;+ let content_index = event+ .get("content_index")+ .and_then(Value::as_u64)+ .unwrap_or(0) as usize;+ state+ .decoded_response_text+ .entry(index)+ .or_default()+ .entry(content_index)+ .or_default()+ .push_str(text);vec![LlmResponseChunk::TextDelta {- index: event- .get("output_index")- .and_then(Value::as_u64)- .unwrap_or(0) as usize,+ index,text: text.to_string(),}]})@@ -247,18 +259,19 @@ fn decode_responses_stream(}Some("response.completed") => {let mut out = Vec::new();- // Some providers surface reasoning only in the final output array. Position is the- // output index; anything already decoded is skipped by the helper.+ // Some providers send output only in the final snapshot. Reconcile it with+ // decoded deltas before emitting the stop, without repeating streamed content.if let Some(items) = event.get("response").and_then(|response| response.get("output")).and_then(Value::as_array){for (position, item) in items.iter().enumerate() {- if let Some(item) = item.as_object()- && item.get("type").and_then(Value::as_str) == Some("reasoning")- {- out.extend(decode_responses_reasoning_item(item, position, state));+ if let Some(item) = item.as_object() {+ out.extend(decode_responses_completed_item(item, position, state));+ if matches!(out.last(), Some(LlmResponseChunk::StreamError { .. })) {+ return out;+ }}}}@@ -278,12 +291,7 @@ fn decode_responses_stream(// use for such output; the stream must too, or stop-reason-driven tool loops// (Anthropic `tool_use`, Chat `tool_calls`) stop without running the tool.// Carries the Anthropic spelling because every encoder already maps it.- let has_tool_call = event- .get("response")- .and_then(|response| response.get("output"))- .and_then(Value::as_array)- .is_some_and(|items| items.iter().any(is_responses_tool_call_item));- let reason = (has_tool_call || state.decoded_tool_call).then(|| "tool_use".to_string());+ let reason = state.decoded_tool_call.then(|| "tool_use".to_string());out.push(LlmResponseChunk::MessageStop { reason });out}@@ -649,6 +657,17 @@ fn decode_responses_output_item_added(return Vec::new();}state.decoded_tool_call = true;+ let id = item+ .get("call_id")+ .or_else(|| item.get("id"))+ .and_then(Value::as_str);+ let name = item.get("name").and_then(Value::as_str);+ state+ .tool_states+ .entry(index)+ .or_default()+ .has_decoded_identity |=+ id.is_some_and(|id| !id.is_empty()) && name.is_some_and(|name| !name.is_empty());// A freeform call's `input` becomes the single `input` argument; it is only complete on// the done event, so nothing is emitted for it here beyond id and name.let arguments_delta = if item_type == Some("custom_tool_call") {@@ -669,28 +688,12 @@ fn decode_responses_output_item_added(}vec![LlmResponseChunk::ToolCallDelta {index,- id: item- .get("call_id")- .or_else(|| item.get("id"))- .and_then(Value::as_str)- .map(ToOwned::to_owned),- name: item- .get("name")- .and_then(Value::as_str)- .map(ToOwned::to_owned),+ id: id.map(ToOwned::to_owned),+ name: name.map(ToOwned::to_owned),arguments_delta,}]}-// Whether a Responses output item is a client tool call the consumer must run.-fn is_responses_tool_call_item(item: &Value) -> bool {- matches!(- item.get("type").and_then(Value::as_str),- Some("function_call" | "custom_tool_call")- )-}--// Emits a final tool-call argument delta when Responses only supplies arguments at item end.fn decode_responses_output_item_done(event: &Value,state: &mut StreamTranslationState,@@ -702,10 +705,42 @@ fn decode_responses_output_item_done(.get("output_index").and_then(Value::as_u64).unwrap_or(0) as usize;+ decode_responses_completed_item(item, index, state)+}++fn decode_responses_completed_item(+ item: &serde_json::Map<String, Value>,+ index: usize,+ state: &mut StreamTranslationState,+) -> Vec<LlmResponseChunk> {if item.get("type").and_then(Value::as_str) == Some("reasoning") {return decode_responses_reasoning_item(item, index, state);}let item_type = item.get("type").and_then(Value::as_str);+ if item_type == Some("message") {+ let mut out = Vec::new();+ if let Some(content) = item.get("content").and_then(Value::as_array) {+ for (content_index, part) in content.iter().enumerate() {+ if part.get("type").and_then(Value::as_str) != Some("output_text") {+ continue;+ }+ if let Some(text) = part.get("text").and_then(Value::as_str) {+ let decoded = state+ .decoded_response_text+ .entry(index)+ .or_default()+ .entry(content_index)+ .or_default();+ match snapshot_suffix(decoded, text) {+ Ok(Some(text)) => out.push(LlmResponseChunk::TextDelta { index, text }),+ Ok(None) => {}+ Err(error) => return vec![error],+ }+ }+ }+ }+ return out;+ }if item_type != Some("function_call") && item_type != Some("custom_tool_call") {return Vec::new();}@@ -720,25 +755,49 @@ fn decode_responses_output_item_done(let arguments = custom_arguments.as_deref().or_else(|| item.get("arguments").and_then(Value::as_str));- if let Some(arguments) = arguments {- // Compared against what THIS decoder has seen. Reading the encoder's- // `arguments` instead only deduplicates when a single state performs- // both halves of the translation, and silently duplicates when a- // caller buffers the stream with its own state.- let tool = state.tool_states.entry(index).or_default();- if !arguments.is_empty() && arguments != tool.decoded_arguments {- tool.decoded_arguments.push_str(arguments);- return vec![LlmResponseChunk::ToolCallDelta {- index,- id: None,- name: None,- arguments_delta: Some(arguments.to_string()),- }];- }+ let tool = state.tool_states.entry(index).or_default();+ let arguments_delta =+ match arguments.map(|arguments| snapshot_suffix(&mut tool.decoded_arguments, arguments)) {+ Some(Ok(delta)) => delta,+ Some(Err(error)) => return vec![error],+ None => None,+ };+ let id = item+ .get("call_id")+ .or_else(|| item.get("id"))+ .and_then(Value::as_str);+ let name = item.get("name").and_then(Value::as_str);+ let needs_identity = !tool.has_decoded_identity;+ tool.has_decoded_identity |=+ id.is_some_and(|id| !id.is_empty()) && name.is_some_and(|name| !name.is_empty());+ if needs_identity || arguments_delta.is_some() {+ return vec![LlmResponseChunk::ToolCallDelta {+ index,+ id: id.filter(|_| needs_identity).map(ToOwned::to_owned),+ name: name.filter(|_| needs_identity).map(ToOwned::to_owned),+ arguments_delta,+ }];}Vec::new()}+// A snapshot may extend streamed content, but cannot retract content already sent.+fn snapshot_suffix(+ decoded: &mut String,+ snapshot: &str,+) -> Result<Option<String>, LlmResponseChunk> {+ let Some(suffix) = snapshot.strip_prefix(decoded.as_str()) else {+ return Err(LlmResponseChunk::StreamError {+ message: "Responses snapshot conflicts with streamed content".to_string(),+ });+ };+ if suffix.is_empty() {+ return Ok(None);+ }+ decoded.push_str(suffix);+ Ok(Some(suffix.to_owned()))+}+// Emits the initial Responses created event once per stream.fn ensure_responses_created(state: &mut StreamTranslationState) -> Vec<Value> {if state.response_created {diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rsindex b53a0293e..3732ff2b6 100644--- a/crates/switchyard-translation/src/codecs/stream.rs+++ b/crates/switchyard-translation/src/codecs/stream.rs@@ -62,6 +62,9 @@ pub struct StreamTranslationState {/// Set once a tool call was observed while DECODING, so a terminal event that names no/// stop reason can still report tool use.pub(crate) decoded_tool_call: bool,+ /// Text decoded from Responses, keyed by output index and then content index.+ #[serde(default)]+ pub(crate) decoded_response_text: BTreeMap<usize, BTreeMap<usize, String>>,pub(crate) response_created: bool,pub(crate) response_text_started: bool,@@ -115,6 +118,8 @@ pub(crate) struct StreamToolState {/// own state and encodes later, the field is empty and the duplicate is/// emitted.pub(crate) decoded_arguments: String,+ #[serde(default)]+ pub(crate) has_decoded_identity: bool,pub(crate) pending_arguments: String,pub(crate) started: bool,pub(crate) content_index: Option<usize>,diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex 474509c25..7893ff22b 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -2766,3 +2766,173 @@ fn responses_stream_empty_reasoning_delta_opens_no_summary_part() -> TestResultassert!(types.contains(&"response.output_item.added"), "{types:?}");Ok(())}++#[test]+fn responses_terminal_snapshots_recover_missing_output_once() -> TestResult {+ let engine = TranslationEngine::default();+ let source = WireFormat::OpenAiResponses;+ let output = json!([+ {"type": "message", "role": "assistant", "content": [+ {"type": "output_text", "text": "Visible answer"}+ ]},+ {"type": "function_call", "id": "fc_1", "call_id": "call_1",+ "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"}+ ]);+ let response =+ json!({"id": "resp_1", "model": "fixture", "status": "completed", "output": output});+ let mut expected = engine+ .decode_response(source, &response, &Default::default())?+ .response;+ if let switchyard_protocol::ContentBlock::ToolCall(call) = &mut expected.outputs[0].content[1] {+ call.arguments = json!({"city": "Paris"});+ }+ for target in [WireFormat::OpenAiChat, WireFormat::AnthropicMessages] {+ for mode in [+ "terminal",+ "done",+ "partial",+ "full",+ "name_only",+ "id_only",+ ] {+ let mut events = vec![+ json!({"type": "response.created", "response": {"id": "resp_1", "model": "fixture"}}),+ ];+ if matches!(mode, "partial" | "full") {+ events.push(+ json!({"type": "response.output_text.delta", "output_index": 0,+ "delta": if mode == "full" { "Visible answer" } else { "Visible " }}),+ );+ events.push(json!({"type": "response.output_item.added", "output_index": 1,+ "item": {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": ""}}));+ events.push(+ json!({"type": "response.function_call_arguments.delta", "output_index": 1,+ "delta": if mode == "full" { "{\"city\":\"Paris\"}" } else { "{\"city\":" }}),+ );+ }+ if matches!(mode, "name_only" | "id_only") {+ let mut item =+ json!({"type": "function_call", "arguments": "{\"city\":\"Paris\"}"});+ if mode == "name_only" {+ item["name"] = json!("get_weather");+ } else {+ item["call_id"] = json!("call_1");+ }+ for event_type in ["response.output_item.added", "response.output_item.done"] {+ events.push(json!({"type": event_type, "output_index": 1, "item": item}));+ }+ } else if mode != "terminal" {+ for (index, item) in output.as_array().unwrap().iter().enumerate() {+ events.push(json!({"type": "response.output_item.done", "output_index": index, "item": item}));+ }+ }+ events.push(json!({"type": "response.completed", "response": response}));+ let mut decoder = StreamTranslationState::new(source, target);+ let mut encoder = StreamTranslationState::new(source, target);+ let mut reader = StreamTranslationState::new(target, target);+ let mut accumulator = ResponseAccumulator::new();+ for event in events {+ let is_incomplete_identity = matches!(mode, "name_only" | "id_only")+ && event["type"] != "response.completed";+ let decoded = engine.decode_stream_event(&mut decoder, source, event)?;+ for encoded in engine.encode_stream_event(&mut encoder, target, decoded)? {+ if target == WireFormat::AnthropicMessages && is_incomplete_identity {+ assert_ne!(encoded["content_block"]["type"], "tool_use");+ }+ for chunk in decode_stream_event(&mut reader, target, &encoded) {+ accumulator.push(chunk);+ }+ }+ }+ for encoded in engine.finish_stream(&mut encoder, target)? {+ for chunk in decode_stream_event(&mut reader, target, &encoded) {+ accumulator.push(chunk);+ }+ }+ assert_eq!(+ expected.outputs,+ accumulator.finish().outputs,+ "{target:?}: {mode}"+ );+ }+ }+ for event_type in ["response.output_item.added", "response.output_item.done"] {+ for identity in [+ json!({}),+ json!({"call_id": "", "name": "get_weather"}),+ json!({"id": "fc_1", "name": ""}),+ ] {+ let mut state = StreamTranslationState::new(source, WireFormat::OpenAiChat);+ let mut item = identity;+ item["type"] = json!("function_call");+ let incomplete = engine.decode_stream_event(+ &mut state,+ source,+ json!({"type": event_type, "output_index": 1, "item": item}),+ )?;+ let target = WireFormat::AnthropicMessages;+ let mut encoder = StreamTranslationState::new(source, target);+ let early = engine.encode_stream_event(&mut encoder, target, incomplete)?;+ assert!(+ early+ .iter()+ .all(|event| event["content_block"]["type"] != "tool_use")+ );+ let terminal = engine.finish_stream(&mut encoder, target)?;+ assert_eq!(terminal.len(), 1);+ assert_eq!(terminal[0]["type"], "error");+ assert!(engine.finish_stream(&mut encoder, target)?.is_empty());+ let decoded = engine.decode_stream_event(+ &mut state,+ source,+ json!({"type": "response.completed", "response": response}),+ )?;+ assert!(+ decoded.normalized().iter().any(|chunk| matches!(chunk,+ LlmResponseChunk::ToolCallDelta { id: Some(id), name: Some(name), .. }+ if id == "call_1" && name == "get_weather"+ )),+ "{event_type}: {item}"+ );+ engine.decode_stream_event(+ &mut state,+ source,+ json!({"type": event_type, "output_index": 1, "item": item}),+ )?;+ let repeated = engine.decode_stream_event(+ &mut state,+ source,+ json!({"type": "response.completed", "response": response}),+ )?;+ assert!(+ !repeated+ .normalized()+ .iter()+ .any(|chunk| matches!(chunk, LlmResponseChunk::ToolCallDelta { .. }))+ );+ }+ }+ for delta in [+ json!({"type": "response.output_text.delta", "output_index": 0, "delta": "Different"}),+ json!({"type": "response.function_call_arguments.delta", "output_index": 1, "delta": "["}),+ ] {+ let mut state = StreamTranslationState::new(source, WireFormat::OpenAiChat);+ engine.decode_stream_event(&mut state, source, delta)?;+ let decoded = engine.decode_stream_event(+ &mut state,+ source,+ json!({"type": "response.completed", "response": response}),+ )?;+ assert!(matches!(+ decoded.normalized().last(),+ Some(LlmResponseChunk::StreamError { .. })+ ));+ assert!(+ !decoded+ .normalized()+ .iter()+ .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }))+ );+ }+ Ok(())+}