Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): upstream sse error prop (#573)

Greg Clark4fb3251020a8
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs
index 7a70ca299..6cf4eb6d9 100644
--- a/crates/protocol/src/stream.rs
+++ b/crates/protocol/src/stream.rs
@@ -12,6 +12,7 @@ use futures::{Stream, StreamExt};
use http::StatusCode;
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use thiserror::Error;
use crate::{
LlmClientError,
@@ -24,6 +25,18 @@ use crate::{
/// code to propagate; 502 matches how a failed upstream call surfaces elsewhere.
const MID_STREAM_UPSTREAM_STATUS: StatusCode = StatusCode::BAD_GATEWAY;
+/// Why a translated event stream stopped early.
+#[derive(Debug, Error)]
+pub enum LlmStreamError {
+ /// An upstream sse error
+ #[error("upstream stream error: {0}")]
+ Upstream(Value),
+
+ /// The stream itself failed
+ #[error(transparent)]
+ Client(#[from] LlmClientError),
+}
+
/// A boxed, `Send` stream of response events. Each item may fail independently mid-stream.
pub type LlmResponseStream =
Pin<Box<dyn Stream<Item = Result<LlmResponseStreamEvent, LlmClientError>> + Send>>;
diff --git a/crates/switchyard-server/src/sse.rs b/crates/switchyard-server/src/sse.rs
index 6354945ec..d94cb220e 100644
--- a/crates/switchyard-server/src/sse.rs
+++ b/crates/switchyard-server/src/sse.rs
@@ -8,7 +8,7 @@ use std::convert::Infallible;
use axum::response::sse::{Event, Sse};
use futures_util::Stream;
use serde_json::{Value, json};
-use switchyard_translation::{RawEventStream, WireFormat};
+use switchyard_translation::{LlmStreamError, RawEventStream, WireFormat};
/// Boxed stream type accepted by Axum's SSE response wrapper.
pub(crate) type SseFrameStream =
@@ -31,7 +31,16 @@ pub(crate) fn frame_stream(
error_event(target_format, error.to_string())
}
},
- Err(error) => {
+ // The upstream's own error event, already in the target format: forward it
+ // verbatim so its code and type survive, rather than synthesizing one.
+ Err(LlmStreamError::Upstream(value)) => {
+ failed = true;
+ frame_event(target_format, value.clone()).unwrap_or_else(|error| {
+ tracing::warn!(error = %error, "in-band error event could not be framed");
+ error_event(target_format, value.to_string())
+ })
+ }
+ Err(LlmStreamError::Client(error)) => {
tracing::warn!(error = %error, "stream iteration failed");
failed = true;
error_event(target_format, error.to_string())
@@ -43,6 +52,8 @@ pub(crate) fn frame_stream(
}
}
+ // `[DONE]` is the OpenAI Chat success sentinel: clients stop reading there and
+ // keep what they have as a finished answer, so it must not follow a failed turn.
if !failed && target_format == WireFormat::OpenAiChat {
yield Ok(Event::default().data("[DONE]"));
}
@@ -93,26 +104,34 @@ fn error_event(target_format: WireFormat, message: String) -> Event {
#[cfg(test)]
mod tests {
- use std::{error::Error, io};
+ use std::error::Error;
use axum::{body::to_bytes, response::IntoResponse};
use futures_util::stream;
+ use switchyard_protocol::LlmClientError;
use super::*;
- type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
+ type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
+
+ // Renders a framed body for one Chat stream.
+ async fn chat_body(items: Vec<Result<Value, LlmStreamError>>) -> TestResult<String> {
+ let stream: RawEventStream = Box::pin(stream::iter(items));
+ let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();
+ Ok(String::from_utf8(
+ to_bytes(response.into_body(), usize::MAX).await?.to_vec(),
+ )?)
+ }
#[tokio::test]
async fn stream_error_terminates_without_done_marker() -> TestResult {
- let failure: Box<dyn Error + Send + Sync> = Box::new(io::Error::other("boom"));
- let stream: RawEventStream = Box::pin(stream::iter(vec![
+ let failure = LlmClientError::General("boom".to_string());
+ let body = chat_body(vec![
Ok(json!({"id": "before"})),
- Err(failure),
+ Err(LlmStreamError::Client(failure)),
Ok(json!({"id": "after"})),
- ]));
-
- let response = frame_stream(stream, WireFormat::OpenAiChat).into_response();
- let body = String::from_utf8(to_bytes(response.into_body(), usize::MAX).await?.to_vec())?;
+ ])
+ .await?;
// A stream error is terminal: later chunks and success markers must not be emitted.
assert!(body.contains("before"));
@@ -121,4 +140,27 @@ mod tests {
assert!(!body.contains("[DONE]"));
Ok(())
}
+
+ #[tokio::test]
+ async fn in_band_error_is_forwarded_verbatim_without_done_marker() -> TestResult {
+ let upstream_error = json!({
+ "error": {"code": "stream_failed", "message": "boom", "type": "upstream_stream_error"}
+ });
+ let body = chat_body(vec![
+ Ok(json!({"id": "before"})),
+ Err(LlmStreamError::Upstream(upstream_error)),
+ Ok(json!({"id": "after"})),
+ ])
+ .await?;
+
+ // The upstream owns this error, so its code and type reach the client unchanged
+ // rather than being flattened into a synthesized SwitchyardError frame.
+ assert!(body.contains("before"));
+ assert!(body.contains("stream_failed"));
+ assert!(body.contains("upstream_stream_error"));
+ assert!(!body.contains("SwitchyardError"));
+ assert!(!body.contains("after"));
+ assert!(!body.contains("[DONE]"));
+ Ok(())
+ }
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 32290ea36..704be1865 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -2766,6 +2766,12 @@ async fn chat_stream_error_does_not_emit_success_terminal_chunk() -> TestResult
!after_error.contains(r#""finish_reason":"stop""#),
"a finish_reason=stop chunk followed an upstream stream error:\n{body}"
);
+ // `[DONE]` is the Chat success sentinel: an SDK client stops there and keeps the
+ // truncated turn as a completed answer, so a failed stream must not emit it.
+ assert!(
+ !after_error.contains("[DONE]"),
+ "a [DONE] success sentinel followed an upstream stream error:\n{body}"
+ );
Ok(())
}
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index 7306aa983..a1cdd1d3c 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -19,8 +19,8 @@ use crate::codecs::stream::encode_response_stream_event;
use crate::sse;
use crate::{
AggLlmResponse, FormatId, LlmRequest, LlmResponseChunk, LlmResponseStream,
- LlmResponseStreamEvent, Result, StreamCodecRegistry, StreamTranslationState, TranslationEngine,
- TranslationPolicy, WireFormat,
+ LlmResponseStreamEvent, LlmStreamError, Result, StreamCodecRegistry, StreamTranslationState,
+ TranslationEngine, TranslationPolicy, WireFormat,
};
static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =
@@ -91,12 +91,8 @@ pub fn encode_aggregated_response_with_extensions(
/// A stream of wire-format event objects in one format — the unframed body of an
/// SSE response. The serving layer frames each `Value` (e.g. as an SSE
/// `data:`/`event:` block).
-pub type RawEventStream = Pin<
- Box<
- dyn Stream<Item = std::result::Result<Value, Box<dyn std::error::Error + Send + Sync>>>
- + Send,
- >,
->;
+pub type RawEventStream =
+ Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;
/// Encodes a stream of IR chunks into a stream of target-format wire events.
///
@@ -148,20 +144,24 @@ pub fn encode_stream_with_extensions(
let events = try_stream! {
while let Some(item) = chunks.next().await {
- let event = item?;
- for mut value in
- encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event)
- {
- stamp_streamed_response_model(
- &mut value,
- target,
- served_model_for_events.as_deref(),
- );
- crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);
+ let event = item.map_err(LlmStreamError::Client)?;
+ let mut encoded =
+ encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);
+ for value in &mut encoded {
+ stamp_streamed_response_model(value, target, served_model_for_events.as_deref());
+ crate::codex_namespaces::restore_qualified_tool_names(value, &origins);
+ }
+ let terminal = encoded.pop();
+ for value in encoded {
yield value;
}
- if state.errored {
- return;
+ match (state.errored, terminal) {
+ (false, Some(value)) => yield value,
+ (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,
+ (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(
+ "stream ended after an error without a terminal event".to_string(),
+ )))?,
+ _ => ()
}
}
for mut value in codec.finish(&mut state) {
@@ -348,7 +348,7 @@ mod tests {
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
encode_request, encode_stream, stamp_streamed_response_model,
};
- use crate::{LlmResponseStream, WireFormat};
+ use crate::{LlmResponseStream, LlmStreamError, WireFormat};
// A boxed stream item error, matching the streamed IR contract.
type BoxError = Box<dyn std::error::Error + Send + Sync>;
@@ -438,7 +438,7 @@ mod tests {
.collect::<Vec<_>>(),
)
.into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;
let content: String = events
.iter()
@@ -481,7 +481,7 @@ mod tests {
.collect::<Vec<_>>(),
)
.into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;
assert_eq!(events[0]["type"], "message_start");
assert_eq!(events[0]["message"]["model"], "served/model");
@@ -510,7 +510,7 @@ mod tests {
encode_stream(chunks, WireFormat::AnthropicMessages, None)?.collect::<Vec<_>>(),
)
.into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;
assert_eq!(events[0]["message"]["model"], "upstream/model");
Ok(())
@@ -530,8 +530,9 @@ mod tests {
}
// An in-band error is terminal for every target format: the encoder emits the pre-error
- // content and the error, then drops any later chunk. Production truncates the source before
- // the encoder, so this contract is only observable by driving encode_stream directly.
+ // content, surfaces the error event through the error channel, and drops any later chunk.
+ // Production truncates the source before the encoder, so this contract is only observable
+ // by driving encode_stream directly.
#[test]
fn encode_stream_stops_after_an_in_band_error() -> Result<(), BoxError> {
for message in [
@@ -561,27 +562,60 @@ mod tests {
.into()),
])
.boxed();
- let events = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>())
- .into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
- let body = serde_json::to_string(&events)?;
+ let results = block_on(encode_stream(chunks, target, None)?.collect::<Vec<_>>());
+ let (data, errors): (Vec<_>, Vec<_>) = results.into_iter().partition(Result::is_ok);
+ let data = data.into_iter().flatten().collect::<Vec<Value>>();
+ let body = serde_json::to_string(&data)?;
+
assert!(
body.contains("before"),
"{target:?}: pre-error content missing:\n{body}"
);
- assert!(
- body.contains("boom"),
- "{target:?}: error event missing:\n{body}"
- );
assert!(
!body.contains("after"),
"{target:?}/{message:?}: content leaked after the error:\n{body}"
);
+ // The error rides the error channel, carrying the codec's own encoded event
+ // so the consumer can forward the upstream payload rather than rebuild it.
+ let [Err(LlmStreamError::Upstream(event))] = errors.as_slice() else {
+ panic!("{target:?}/{message:?}: expected one in-band error, got {errors:?}");
+ };
+ assert!(
+ serde_json::to_string(event)?.contains("boom"),
+ "{target:?}: error event lost its message: {event}"
+ );
}
}
Ok(())
}
+ // A clean stop stays entirely on the data channel, so a consumer that appends a
+ // format-level success sentinel only does so for a turn that actually completed.
+ #[test]
+ fn encode_stream_keeps_a_normal_stop_off_the_error_channel() -> Result<(), BoxError> {
+ let chunks: LlmResponseStream = stream::iter(vec![
+ Ok(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hi".to_string(),
+ }
+ .into()),
+ Ok(LlmResponseChunk::MessageStop {
+ reason: Some("stop".to_string()),
+ }
+ .into()),
+ ])
+ .boxed();
+
+ let results =
+ block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
+
+ assert!(
+ results.iter().all(Result::is_ok),
+ "a completed turn reported an error: {results:?}"
+ );
+ Ok(())
+ }
+
// A replayed provider error ends the stream before the encoder polls the source again.
#[test]
fn encode_stream_stops_polling_after_a_replayed_error() -> Result<(), BoxError> {
@@ -599,11 +633,14 @@ mod tests {
.boxed();
let events =
- block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>())
- .into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
- assert_eq!(events, vec![json!({"type": "error", "message": "boom"})]);
+ // The replayed provider event is the terminal frame, so it arrives verbatim on the
+ // error channel and nothing follows it.
+ let [Err(LlmStreamError::Upstream(event))] = events.as_slice() else {
+ return Err(format!("expected one in-band error, got {events:?}").into());
+ };
+ assert_eq!(*event, json!({"type": "error", "message": "boom"}));
Ok(())
}
@@ -631,7 +668,7 @@ mod tests {
let events =
block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
.into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;
let body = serde_json::to_string(&events)?;
assert!(
events
@@ -679,7 +716,7 @@ mod tests {
let replayed =
block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
.into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ .collect::<Result<Vec<Value>, LlmStreamError>>()?;
assert_eq!(replayed, vec![provider_event]);
Ok(())
@@ -806,16 +843,18 @@ mod tests {
)
.boxed();
let replayed =
- block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>())
- .into_iter()
- .collect::<Result<Vec<Value>, BoxError>>()?;
+ block_on(encode_stream(chunks, WireFormat::OpenAiResponses, None)?.collect::<Vec<_>>());
+ // `response.failed` stays a terminal error, replayed with its payload intact.
+ let [Err(LlmStreamError::Upstream(event))] = replayed.as_slice() else {
+ return Err(format!("expected one in-band error, got {replayed:?}").into());
+ };
assert_eq!(
- replayed,
- vec![json!({
+ *event,
+ json!({
"type": "response.failed",
"response": {"error": {"message": "upstream failed"}}
- })]
+ })
);
Ok(())
}
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 27cdc1578..8f9bbe976 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -19,7 +19,8 @@ pub mod stream;
pub mod util;
pub use switchyard_protocol::stream::{
- LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ProviderStreamEvent,
+ LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError,
+ ProviderStreamEvent,
};
pub use switchyard_protocol::{format, llm};