Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): upstream sse error prop (#573)
Greg Clark4fb3251020a8
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rsindex 7a70ca299..6cf4eb6d9 100644--- a/crates/protocol/src/stream.rs+++ b/crates/protocol/src/stream.rs@@ -12,6 +12,7 @@ use futures::{Stream, StreamExt};use http::StatusCode;use serde::{Deserialize, Serialize};use serde_json::Value;+use thiserror::Error;use crate::{LlmClientError,@@ -24,6 +25,18 @@ use crate::{/// code to propagate; 502 matches how a failed upstream call surfaces elsewhere.const MID_STREAM_UPSTREAM_STATUS: StatusCode = StatusCode::BAD_GATEWAY;+/// Why a translated event stream stopped early.+#[derive(Debug, Error)]+pub enum LlmStreamError {+ /// An upstream sse error+ #[error("upstream stream error: {0}")]+ Upstream(Value),++ /// The stream itself failed+ #[error(transparent)]+ Client(#[from] LlmClientError),+}+/// A boxed, `Send` stream of response events. Each item may fail independently mid-stream.pub type LlmResponseStream =Pin<Box<dyn Stream<Item = Result<LlmResponseStreamEvent, LlmClientError>> + Send>>;diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rsindex 6354945ec..d94cb220e 100644--- a/crates/switchyard-server/src/sse.rs+++ b/crates/switchyard-server/src/sse.rs@@ -8,7 +8,7 @@ use std::convert::Infallible;use axum::response::sse::{Event, Sse};use futures_util::Stream;use serde_json::{Value, json};-use switchyard_translation::{RawEventStream, WireFormat};+use switchyard_translation::{LlmStreamError, RawEventStream, WireFormat};/// Boxed stream type accepted by Axum's SSE response wrapper.pub(crate) type SseFrameStream =@@ -31,7 +31,16 @@ pub(crate) fn frame_stream(error_event(target_format, error.to_string())}},- Err(error) => {+ // The upstream's own error event, already in the target format: forward it+ // verbatim so its code and type survive, rather than synthesizing one.+ Err(LlmStreamError::Upstream(value)) => {+ failed = true;+ frame_event(target_format, value.clone()).unwrap_or_else(|error| {+ tracing::warn!(error = %error, "in-band error event could not be framed");+ error_event(target_format, value.to_string())+ })+ }+ Err(LlmStreamError::Client(error)) => {tracing::warn!(error = %error, "stream iteration failed");failed = true;error_event(target_format, error.to_string())@@ -43,6 +52,8 @@ pub(crate) fn frame_stream(}}+ // `[DONE]` is the OpenAI Chat success sentinel: clients stop reading there and+ // keep what they have as a finished answer, so it must not follow a failed turn.if !failed && target_format == WireFormat::OpenAiChat {yield Ok(Event::default().data("[DONE]"));}@@ -93,26 +104,34 @@ fn error_event(target_format: WireFormat, message: String) -> Event {#[cfg(test)]mod tests {- use std::{error::Error, io};+ use std::error::Error;use axum::{body::to_bytes, response::IntoResponse};use futures_util::stream;+ use switchyard_protocol::LlmClientError;use super::*;- type TestResult = Result<(), Box<dyn Error + Send + Sync>>;+ type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;++ // Renders a framed body for one Chat stream.+ async fn chat_body(items: Vec<Result<Value, LlmStreamError>>) -> TestResult<String> {+ let stream: RawEventStream = Box::pin(stream::iter(items));+ let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();+ Ok(String::from_utf8(+ to_bytes(response.into_body(), usize::MAX).await?.to_vec(),+ )?)+ }#[tokio::test]async fn stream_error_terminates_without_done_marker() -> TestResult {- let failure: Box<dyn Error + Send + Sync> = Box::new(io::Error::other("boom"));- let stream: RawEventStream = Box::pin(stream::iter(vec![+ let failure = LlmClientError::General("boom".to_string());+ let body = chat_body(vec![Ok(json!({"id": "before"})),- Err(failure),+ Err(LlmStreamError::Client(failure)),Ok(json!({"id": "after"})),- ]));-- let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();- let body = String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?;+ ])+ .await?;// A stream error is terminal: later chunks and success markers must not be emitted.assert!(body.contains("before"));@@ -121,4 +140,27 @@ mod tests {assert!(!body.contains("[DONE]"));Ok(())}++ #[tokio::test]+ async fn in_band_error_is_forwarded_verbatim_without_done_marker() -> TestResult {+ let upstream_error = json!({+ "error": {"code": "stream_failed", "message": "boom", "type": "upstream_stream_error"}+ });+ let body = chat_body(vec![+ Ok(json!({"id": "before"})),+ Err(LlmStreamError::Upstream(upstream_error)),+ Ok(json!({"id": "after"})),+ ])+ .await?;++ // The upstream owns this error, so its code and type reach the client unchanged+ // rather than being flattened into a synthesized SwitchyardError frame.+ assert!(body.contains("before"));+ assert!(body.contains("stream_failed"));+ assert!(body.contains("upstream_stream_error"));+ assert!(!body.contains("SwitchyardError"));+ assert!(!body.contains("after"));+ assert!(!body.contains("[DONE]"));+ Ok(())+ }}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 32290ea36..704be1865 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -2766,6 +2766,12 @@ async fn chat_stream_error_does_not_emit_success_terminal_chunk() -> TestResult!after_error.contains(r#""finish_reason":"stop""#),"a finish_reason=stop chunk followed an upstream stream error:\n{body}");+ // `[DONE]` is the Chat success sentinel: an SDK client stops there and keeps the+ // truncated turn as a completed answer, so a failed stream must not emit it.+ assert!(+ !after_error.contains("[DONE]"),+ "a [DONE] success sentinel followed an upstream stream error:\n{body}"+ );Ok(())}diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex 7306aa983..a1cdd1d3c 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -19,8 +19,8 @@ use crate::codecs::stream::encode_response_stream_event;use crate::sse;use crate::{AggLlmResponse, FormatId, LlmRequest, LlmResponseChunk, LlmResponseStream,- LlmResponseStreamEvent, Result, StreamCodecRegistry, StreamTranslationState, TranslationEngine,- TranslationPolicy, WireFormat,+ LlmResponseStreamEvent, LlmStreamError, Result, StreamCodecRegistry, StreamTranslationState,+ TranslationEngine, TranslationPolicy, WireFormat,};static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =@@ -91,12 +91,8 @@ pub fn encode_aggregated_response_with_extensions(/// A stream of wire-format event objects in one format — the unframed body of an/// SSE response. The serving layer frames each `Value` (e.g. as an SSE/// `data:`/`event:` block).-pub type RawEventStream = Pin<- Box<- dyn Stream<Item = std::result::Result<Value, Box<dyn std::error::Error + Send + Sync>>>- + Send,- >,->;+pub type RawEventStream =+ Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;/// Encodes a stream of IR chunks into a stream of target-format wire events.///@@ -148,20 +144,24 @@ pub fn encode_stream_with_extensions(let events = try_stream! {while let Some(item) = chunks.next().await {- let event = item?;- for mut value in- encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event)- {- stamp_streamed_response_model(- &mut value,- target,- served_model_for_events.as_deref(),- );- crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);+ let event = item.map_err(LlmStreamError::Client)?;+ let mut encoded =+ encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);+ for value in &mut encoded {+ stamp_streamed_response_model(value, target, served_model_for_events.as_deref());+ crate::codex_namespaces::restore_qualified_tool_names(value, &origins);+ }+ let terminal = encoded.pop();+ for value in encoded {yield value;}- if state.errored {- return;+ match (state.errored, terminal) {+ (false, Some(value)) => yield value,+ (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,+ (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(+ "stream ended after an error without a terminal event".to_string(),+ )))?,+ _ => ()}}for mut value in codec.finish(&mut state) {@@ -348,7 +348,7 @@ mod tests {decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,encode_request, encode_stream, stamp_streamed_response_model,};- use crate::{LlmResponseStream, WireFormat};+ use crate::{LlmResponseStream, LlmStreamError, WireFormat};// A boxed stream item error, matching the streamed IR contract.type BoxError = Box<dyn std::error::Error + Send + Sync>;@@ -438,7 +438,7 @@ mod tests {.collect::<Vec<_>>(),).into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;let content: String = events.iter()@@ -481,7 +481,7 @@ mod tests {.collect::<Vec<_>>(),).into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;assert_eq!(events[0]["type"], "message_start");assert_eq!(events[0]["message"]["model"], "served/model");@@ -510,7 +510,7 @@ mod tests {encode_stream(chunks, WireFormat::AnthropicMessages, None)?.collect::<Vec<_>>(),).into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;assert_eq!(events[0]["message"]["model"], "upstream/model");Ok(())@@ -530,8 +530,9 @@ mod tests {}// An in-band error is terminal for every target format: the encoder emits the pre-error- // content and the error, then drops any later chunk. Production truncates the source before- // the encoder, so this contract is only observable by driving encode_stream directly.+ // content, surfaces the error event through the error channel, and drops any later chunk.+ // Production truncates the source before the encoder, so this contract is only observable+ // by driving encode_stream directly.#[test]fn encode_stream_stops_after_an_in_band_error() -> Result<(), BoxError> {for message in [@@ -561,27 +562,60 @@ mod tests {.into()),]).boxed();- let events = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>())- .into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;- let body = serde_json::to_string(&events)?;+ let results = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>());+ let (data, errors): (Vec<_>, Vec<_>) = results.into_iter().partition(Result::is_ok);+ let data = data.into_iter().flatten().collect::<Vec<Value>>();+ let body = serde_json::to_string(&data)?;+assert!(body.contains("before"),"{target:?}: pre-error content missing:\n{body}");- assert!(- body.contains("boom"),- "{target:?}: error event missing:\n{body}"- );assert!(!body.contains("after"),"{target:?}/{message:?}: content leaked after the error:\n{body}");+ // The error rides the error channel, carrying the codec's own encoded event+ // so the consumer can forward the upstream payload rather than rebuild it.+ let [Err(LlmStreamError::Upstream(event))] = errors.as_slice() else {+ panic!("{target:?}/{message:?}: expected one in-band error, got {errors:?}");+ };+ assert!(+ serde_json::to_string(event)?.contains("boom"),+ "{target:?}: error event lost its message: {event}"+ );}}Ok(())}+ // A clean stop stays entirely on the data channel, so a consumer that appends a+ // format-level success sentinel only does so for a turn that actually completed.+ #[test]+ fn encode_stream_keeps_a_normal_stop_off_the_error_channel() -> Result<(), BoxError> {+ let chunks: LlmResponseStream = stream::iter(vec![+ Ok(LlmResponseChunk::TextDelta {+ index: 0,+ text: "hi".to_string(),+ }+ .into()),+ Ok(LlmResponseChunk::MessageStop {+ reason: Some("stop".to_string()),+ }+ .into()),+ ])+ .boxed();++ let results =+ block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());++ assert!(+ results.iter().all(Result::is_ok),+ "a completed turn reported an error: {results:?}"+ );+ Ok(())+ }+// A replayed provider error ends the stream before the encoder polls the source again.#[test]fn encode_stream_stops_polling_after_a_replayed_error() -> Result<(), BoxError> {@@ -599,11 +633,14 @@ mod tests {.boxed();let events =- block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>())- .into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());- assert_eq!(events, vec![json!({"type": "error", "message": "boom"})]);+ // The replayed provider event is the terminal frame, so it arrives verbatim on the+ // error channel and nothing follows it.+ let [Err(LlmStreamError::Upstream(event))] = events.as_slice() else {+ return Err(format!("expected one in-band error, got {events:?}").into());+ };+ assert_eq!(*event, json!({"type": "error", "message": "boom"}));Ok(())}@@ -631,7 +668,7 @@ mod tests {let events =block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>()).into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;let body = serde_json::to_string(&events)?;assert!(events@@ -679,7 +716,7 @@ mod tests {let replayed =block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>()).into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;assert_eq!(replayed, vec![provider_event]);Ok(())@@ -806,16 +843,18 @@ mod tests {).boxed();let replayed =- block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>())- .into_iter()- .collect::<Result<Vec<Value>, BoxError>>()?;+ block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());+ // `response.failed` stays a terminal error, replayed with its payload intact.+ let [Err(LlmStreamError::Upstream(event))] = replayed.as_slice() else {+ return Err(format!("expected one in-band error, got {replayed:?}").into());+ };assert_eq!(- replayed,- vec![json!({+ *event,+ json!({"type": "response.failed","response": {"error": {"message": "upstream failed"}}- })]+ }));Ok(())}diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rsindex 27cdc1578..8f9bbe976 100644--- a/crates/switchyard-translation/src/lib.rs+++ b/crates/switchyard-translation/src/lib.rs@@ -19,7 +19,8 @@ pub mod stream;pub mod util;pub use switchyard_protocol::stream::{- LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ProviderStreamEvent,+ LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError,+ ProviderStreamEvent,};pub use switchyard_protocol::{format, llm};