Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): reject incomplete upstream streams (#425)
yoliverasPozod0f249ad41b9
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex a4457b76b..4731599bc 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -201,6 +201,15 @@ fn decode_responses_stream(Some("response.incomplete") => vec![LlmResponseChunk::MessageStop {reason: Some("max_tokens".to_string()),}],+ Some("response.failed") => vec![LlmResponseChunk::StreamError {+ message: event+ .get("response")+ .and_then(|response| response.get("error"))+ .and_then(|error| error.get("message"))+ .and_then(Value::as_str)+ .unwrap_or("unknown Responses stream error")+ .to_string(),+ }],Some("error") => vec![LlmResponseChunk::StreamError {message: event.get("message")diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex 106701b22..7306aa983 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -18,8 +18,9 @@ use switchyard_protocol::LlmClientError;use crate::codecs::stream::encode_response_stream_event;use crate::sse;use crate::{- AggLlmResponse, FormatId, LlmRequest, LlmResponseStream, LlmResponseStreamEvent, Result,- StreamCodecRegistry, StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat,+ AggLlmResponse, FormatId, LlmRequest, LlmResponseChunk, LlmResponseStream,+ LlmResponseStreamEvent, Result, StreamCodecRegistry, StreamTranslationState, TranslationEngine,+ TranslationPolicy, WireFormat,};static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =@@ -208,12 +209,16 @@ fn stamp_streamed_response_model(}}-/// Decodes a byte stream of `source`-format SSE frames into neutral IR chunks.+/// Decodes provider SSE bytes into normalized stream events.////// Operates on raw bytes, not any HTTP client type: the caller adapts its/// transport's body stream into `Stream<Item = Result<Vec<u8>, _>>`. Frames are/// buffered across chunks (a partial frame waits for its boundary); the source/// stream codec is resolved once and reused for every frame.+///+/// The decoder tracks the source format's protocol-specific terminal event. If+/// EOF arrives without that required event, the stream yields a deferred+/// [`LlmClientError::ResponseTranslation`] error after any valid decoded events.pub fn decode_stream<S>(bytes: S,source: WireFormat,@@ -243,6 +248,8 @@ where..StreamTranslationState::default()};let mut frame = String::new();+ let mut saw_terminal = false;+ let mut saw_error = false;let stream = Box::pin(try_stream! {futures::pin_mut!(lines);while let Some(line) = lines.next().await {@@ -254,9 +261,18 @@ whereframe.clear();match parsed {sse::SseFrame::Empty => {}- sse::SseFrame::Done => break,+ sse::SseFrame::Done => {+ saw_terminal |= source != WireFormat::AnthropicMessages;+ break;+ }sse::SseFrame::Data(value) => {+ saw_terminal |= sse::is_terminal_event(source, &value);let normalized = codec.decode_event(&mut state, &value);+ saw_error |= normalized.iter().any(|chunk| matches!(+ chunk,+ LlmResponseChunk::DecodeError { .. }+ | LlmResponseChunk::StreamError { .. }+ ));yield LlmResponseStreamEvent::preserved(source_format.clone(),value,@@ -276,11 +292,29 @@ whereif !frame.trim_end().is_empty() {let parsed = sse::parse_json_sse_frame(&frame, marker).map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;- if let sse::SseFrame::Data(value) = parsed {- let normalized = codec.decode_event(&mut state, &value);- yield LlmResponseStreamEvent::preserved(source_format, value, normalized);+ match parsed {+ sse::SseFrame::Done => {+ saw_terminal |= source != WireFormat::AnthropicMessages;+ }+ sse::SseFrame::Data(value) => {+ saw_terminal |= sse::is_terminal_event(source, &value);+ let normalized = codec.decode_event(&mut state, &value);+ saw_error |= normalized.iter().any(|chunk| matches!(+ chunk,+ LlmResponseChunk::DecodeError { .. }+ | LlmResponseChunk::StreamError { .. }+ ));+ yield LlmResponseStreamEvent::preserved(source_format, value, normalized);+ }+ sse::SseFrame::Empty => {}}}++ if !saw_terminal && !saw_error {+ Err(LlmClientError::ResponseTranslation(format!(+ "{source} stream ended before a terminal event"+ )))?;+ }});Ok(stream)}@@ -726,13 +760,66 @@ mod tests {fn decode_stream_decodes_trailing_frame_without_blank_line() -> Result<(), BoxError> {// A non-standard upstream omits the final blank line; the last frame// must still be decoded rather than dropped.- let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}".to_vec();+ let sse =+ b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}\n\ndata: [DONE]".to_vec();let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;assert_eq!(text_of(&chunks), "tail");Ok(())}+ #[test]+ fn decode_stream_rejects_eof_before_a_terminal_event() -> Result<(), BoxError> {+ // Preserve valid content before surfacing premature EOF as the terminal stream error.+ let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"partial\"},\"finish_reason\":null}]}\n\n".to_vec();+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });+ let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());++ let Some(Ok(first)) = results.first() else {+ return Err("expected the partial event".into());+ };+ assert_eq!(text_of(std::slice::from_ref(first)), "partial");+ let Some(Err(LlmClientError::ResponseTranslation(message))) = results.last() else {+ panic!("expected incomplete OpenAI stream to fail");+ };+ assert_eq!(message, "openai_chat stream ended before a terminal event");+ Ok(())+ }++ #[test]+ fn responses_failed_before_done_remains_a_stream_error() -> Result<(), BoxError> {+ let sse = b"data: {\"type\":\"response.failed\",\"response\":{\"error\":{\"message\":\"upstream failed\"}}}\n\ndata: [DONE]\n\n".to_vec();+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });+ let events = decode_all(bytes, WireFormat::OpenAiResponses)?;++ let Some(LlmResponseChunk::StreamError { message }) =+ events.first().and_then(|event| event.normalized().first())+ else {+ return Err("expected response.failed to decode as StreamError".into());+ };+ assert_eq!(message, "upstream failed");++ let chunks: LlmResponseStream = stream::iter(+ events+ .into_iter()+ .map(Ok::<LlmResponseStreamEvent, LlmClientError>),+ )+ .boxed();+ let replayed =+ block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>())+ .into_iter()+ .collect::<Result<Vec<Value>, BoxError>>()?;++ assert_eq!(+ replayed,+ vec![json!({+ "type": "response.failed",+ "response": {"error": {"message": "upstream failed"}}+ })]+ );+ Ok(())+ }+#[test]fn decode_stream_decodes_crlf_delimited_frames() -> Result<(), BoxError> {// CRLF framing: blank lines are `\r\n\r\n` and the bare `\r` must notdiff --git a/crates/switchyard-translation/src/sse.rs b/crates/switchyard-translation/src/sse.rsindex 439df5d45..bf24aabbb 100644--- a/crates/switchyard-translation/src/sse.rs+++ b/crates/switchyard-translation/src/sse.rs@@ -39,6 +39,33 @@ fn data_field_value(line: &str) -> Option<String> {Some(value.strip_prefix(' ').unwrap_or(value).to_string())}+/// Returns whether a provider event explicitly completes its wire-format stream.+pub(crate) fn is_terminal_event(format: WireFormat, event: &Value) -> bool {+ match format {+ WireFormat::OpenAiChat => event+ .get("choices")+ .and_then(Value::as_array)+ .into_iter()+ .flatten()+ .any(|choice| {+ choice+ .get("finish_reason")+ .and_then(Value::as_str)+ .is_some()+ }),+ WireFormat::AnthropicMessages => {+ event.get("type").and_then(Value::as_str) == Some("message_stop")+ }+ WireFormat::OpenAiResponses => matches!(+ event+ .get("type")+ .or_else(|| event.get("event"))+ .and_then(Value::as_str),+ Some("response.completed" | "response.incomplete" | "response.failed")+ ),+ }+}+pub(crate) fn parse_json_sse_frame(frame: &str,done_marker: Option<&str>,@@ -164,4 +191,33 @@ mod tests {fn anthropic_accepts_optional_done_marker() {assert_eq!(done_marker(WireFormat::AnthropicMessages), Some("[DONE]"));}++ #[test]+ fn recognizes_provider_terminal_events() {+ // Each source format requires its own protocol-specific terminal event.+ assert!(is_terminal_event(+ WireFormat::OpenAiChat,+ &json!({"choices": [{"finish_reason": "stop"}]})+ ));+ assert!(is_terminal_event(+ WireFormat::AnthropicMessages,+ &json!({"type": "message_stop"})+ ));+ assert!(is_terminal_event(+ WireFormat::OpenAiResponses,+ &json!({"type": "response.completed"})+ ));+ assert!(is_terminal_event(+ WireFormat::OpenAiResponses,+ &json!({"type": "response.incomplete"})+ ));+ assert!(is_terminal_event(+ WireFormat::OpenAiResponses,+ &json!({"type": "response.failed"})+ ));+ assert!(!is_terminal_event(+ WireFormat::OpenAiChat,+ &json!({"choices": [{"finish_reason": null}]})+ ));+ }}