Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore(protocol): Use LlmClientError for stream too (#137)

Graham King5349735c8ec9
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index bf73cb7d4..cc1c2b88a 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -206,7 +206,7 @@ impl TranslatingLlmClient {
// transport-agnostic and lives in `switchyard-translation`.
let bytes = http_response.bytes_stream().map(|chunk| {
chunk.map(|bytes| bytes.to_vec()).map_err(|error| {
- let error = if error.is_timeout() {
+ if error.is_timeout() {
LlmClientError::Timeout {
source: Box::new(error),
}
@@ -214,12 +214,10 @@ impl TranslatingLlmClient {
LlmClientError::Transport {
source: Box::new(error),
}
- };
- Box::new(error) as Box<dyn std::error::Error + Send + Sync>
+ }
})
});
- let chunks = decode_stream(bytes, wire_format)
- .map_err(|source| LlmClientError::InvalidResponse { source })?;
+ let chunks = decode_stream(bytes, wire_format)?;
LlmResponse::Stream(chunks)
} else {
let body = http_response
@@ -288,8 +286,7 @@ impl TranslatingLlmClient {
Ok(RawResponse::Buffered(body))
}
LlmResponse::Stream(chunks) => {
- let events = encode_stream(chunks, wire_format, requested_model)
- .map_err(|source| LlmClientError::InvalidResponse { source })?;
+ let events = encode_stream(chunks, wire_format, requested_model)?;
Ok(RawResponse::Stream(events))
}
}
@@ -402,9 +399,17 @@ mod tests {
)]
}
- fn truncated_response_server() -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
+ fn truncated_response_server(
+ content_type: &str,
+ body: &str,
+ ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let address = listener.local_addr()?;
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\
+ Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len() + 100
+ );
let handle = std::thread::spawn(move || {
let (mut stream, _) = listener.accept()?;
let mut request = [0_u8; 1024];
@@ -414,10 +419,7 @@ mod tests {
"client closed before sending a request",
));
}
- stream.write_all(
- b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
- Content-Length: 100\r\nConnection: close\r\n\r\n{}",
- )
+ stream.write_all(response.as_bytes())
});
Ok((format!("http://{address}/v1"), handle))
}
@@ -600,7 +602,7 @@ mod tests {
#[tokio::test]
async fn response_body_io_failure_is_a_transport_error(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
- let (base_url, server) = truncated_response_server()?;
+ let (base_url, server) = truncated_response_server("application/json", "{}")?;
let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
let result = client
.call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
@@ -624,6 +626,28 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn streaming_body_io_failure_preserves_transport_error(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";
+ let (base_url, server) = truncated_response_server("text/event-stream", body)?;
+ let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)
+ .await?;
+ let result = response.llm_response.into_agg().await;
+ server
+ .join()
+ .map_err(|_| std::io::Error::other("response server thread panicked"))??;
+
+ let Err(error) = result else {
+ panic!("expected the truncated stream body to fail");
+ };
+
+ assert!(matches!(error, LlmClientError::Transport { .. }));
+ Ok(())
+ }
+
#[tokio::test]
async fn rewrites_model_to_resolved_upstream_id(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rs
index 5705865fd..b0c66a204 100644
--- a/crates/libsy/examples/ensemble.rs
+++ b/crates/libsy/examples/ensemble.rs
@@ -461,7 +461,7 @@ async fn main() -> Result<()> {
.llm_response
.into_agg()
.await
- .map_err(|error| LibsyError::external_boxed("aggregating response", error))?,
+ .map_err(|error| LibsyError::external("aggregating response", error))?,
)
);
Ok(())
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index 7cf53ea9c..892e162e3 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -85,7 +85,7 @@ impl ResearchAgent {
.llm_response
.into_agg()
.await
- .map_err(|error| LibsyError::external_boxed("aggregating response", error))?;
+ .map_err(|error| LibsyError::external("aggregating response", error))?;
notes.push(completion_text(&aggregate));
}
Ok(notes.join("\n"))
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index ad8833e49..6567f5206 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -80,7 +80,7 @@ impl ResearchAgent {
Step::ReturnToAgent(response) => {
let aggregate =
response.llm_response.into_agg().await.map_err(|error| {
- LibsyError::external_boxed("aggregating response", error)
+ LibsyError::external("aggregating response", error)
})?;
notes.push(completion_text(&aggregate));
}
diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs
index 08884e72a..a2b4f594c 100644
--- a/crates/libsy/examples/streaming_agent.rs
+++ b/crates/libsy/examples/streaming_agent.rs
@@ -86,7 +86,7 @@ async fn main() -> Result<()> {
print!("agent sees: ");
while let Some(chunk) = chunks.next().await {
let chunk = chunk.map_err(|error| {
- LibsyError::external_boxed("reading response stream", error)
+ LibsyError::external("reading response stream", error)
})?;
if let LlmResponseChunk::TextDelta { text, .. } = chunk {
print!("{text}");
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 780d23117..b4e5f0b97 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -259,9 +259,7 @@ mod tests {
.into_agg()
.await
.map(|agg| completion_text(&agg))
- .map_err(|error| {
- LibsyError::external_boxed("aggregating fall-through response", error)
- })?;
+ .map_err(|error| LibsyError::external("aggregating fall-through response", error))?;
Ok((text, trace))
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 6f668311b..31ac50383 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -247,7 +247,9 @@ mod tests {
};
self.seen
.lock()
- .map_err(|_| switchyard_protocol::LlmClientError::Other("lock poisoned".into()))?
+ .map_err(|_| {
+ switchyard_protocol::LlmClientError::General("lock poisoned".to_string())
+ })?
.push(request);
Ok(Response {
llm_response: LlmResponse::Agg(text_response(None, completion)),
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index fdfb153cc..041c78f64 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -651,10 +651,11 @@ mod tests {
]);
let (trace, response) = orch.run(Context::default(), request()).await?;
// `run` handed back the live stream; the caller folds it to a buffered aggregate.
- let agg =
- response.llm_response.into_agg().await.map_err(|error| {
- LibsyError::external_boxed("aggregating response stream", error)
- })?;
+ let agg = response
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|error| LibsyError::external("aggregating response stream", error))?;
assert_eq!(completion_text(&agg), "hello");
assert_eq!(agg.model.as_deref(), Some("stream/model"));
assert_eq!(trace.len(), 1);
@@ -670,13 +671,13 @@ mod tests {
index: 0,
text: "partial".to_string(),
},
- LlmResponseChunk::Error {
+ LlmResponseChunk::StreamError {
message: "upstream exploded".to_string(),
},
]);
let (_, response) = orch.run(Context::default(), request()).await?;
match response.llm_response.into_agg().await {
- Ok(_) => Err(test_error("expected a mid-stream error, got an aggregate")),
+ Ok(_) => panic!("expected a mid-stream error, got an aggregate"),
Err(err) => {
assert!(err.to_string().contains("upstream exploded"));
Ok(())
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index 86b8f8877..503d916da 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -95,14 +95,6 @@ impl LibsyError {
source: Box::new(source),
}
}
-
- /// Preserve an already boxed foreign error.
- pub fn external_boxed(
- operation: &'static str,
- source: Box<dyn StdError + Send + Sync>,
- ) -> Self {
- Self::External { operation, source }
- }
}
/// Failures in the type-erased promise-over-stream driver.
@@ -151,7 +143,7 @@ mod tests {
fn client_call_preserves_target_and_source() {
let error = LibsyError::client_call(
"strong",
- LlmClientError::Other(Box::new(std::io::Error::other("upstream down"))),
+ LlmClientError::General("upstream down".to_string()),
);
match &error {
LibsyError::ClientCall { target, source } => {
diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs
index 47753c979..c04cb03e2 100644
--- a/crates/protocol/src/client.rs
+++ b/crates/protocol/src/client.rs
@@ -23,7 +23,7 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
/// Failures a routed LLM client can surface to its caller.
///
/// The variants classify failures that routing hosts commonly need to handle,
-/// while boxed sources preserve implementation-specific detail. `Other` is the
+/// while boxed sources preserve implementation-specific detail. `General` is the
/// escape hatch for failures that do not fit a shared category.
#[non_exhaustive]
#[derive(Debug, Error)]
@@ -90,9 +90,18 @@ pub enum LlmClientError {
source: BoxError,
},
- /// A client-specific failure. Prefer adding variants than using this.
- #[error(transparent)]
- Other(#[from] BoxError),
+ /// A call across a foreign-function boundary (e.g. a Python-implemented client)
+ /// failed. The boxed source is the foreign error itself.
+ #[error("foreign function interface error: {source}")]
+ Ffi {
+ /// Foreign-language failure, preserved verbatim.
+ #[source]
+ source: BoxError,
+ },
+
+ /// A string message. Useful in testing, but prefer adding variants over using this.
+ #[error("{0}")]
+ General(String),
}
/// A decision/trace object produced by an algorithm.
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs
index 102f9b7d9..0db72066a 100644
--- a/crates/protocol/src/stream.rs
+++ b/crates/protocol/src/stream.rs
@@ -6,21 +6,26 @@
//! or the terminal [`AggLlmResponse`].
use std::collections::BTreeMap;
-use std::error::Error;
use std::pin::Pin;
use futures::{Stream, StreamExt};
use serde::{Deserialize, Serialize};
use serde_json::Value;
-use crate::llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage};
+use crate::{
+ llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage},
+ LlmClientError,
+};
-/// Boxed, thread-safe error carried by a stream item.
-type BoxError = Box<dyn Error + Send + Sync>;
+/// Status reported for an upstream error delivered inside a streaming body. The
+/// upstream already sent a success status line before failing, so there is no real
+/// code to propagate; 502 matches how a failed upstream call surfaces elsewhere.
+const MID_STREAM_UPSTREAM_STATUS: u16 = 502;
/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a
/// streaming backend. Each item may fail independently mid-stream.
-pub type LlmResponseStream = Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, BoxError>> + Send>>;
+pub type LlmResponseStream =
+ Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, LlmClientError>> + Send>>;
/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or the
/// terminal buffered [`Agg`](LlmResponse::Agg)regate.
@@ -44,16 +49,28 @@ impl LlmResponse {
/// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream`
/// to completion, folding its chunks into an [`AggLlmResponse`] via
- /// [`ResponseAccumulator`]. A stream item error, or an in-band
- /// [`LlmResponseChunk::Error`], aborts with `Err`.
- pub async fn into_agg(self) -> Result<AggLlmResponse, BoxError> {
+ /// [`ResponseAccumulator`]. A stream item error aborts with `Err`, as does an
+ /// in-band [`LlmResponseChunk::DecodeError`] (as `ResponseTranslation`) or
+ /// [`LlmResponseChunk::StreamError`] (as `UpstreamHttp`).
+ pub async fn into_agg(self) -> Result<AggLlmResponse, LlmClientError> {
match self {
LlmResponse::Agg(agg) => Ok(agg),
LlmResponse::Stream(mut stream) => {
let mut accumulator = ResponseAccumulator::new();
while let Some(item) = stream.next().await {
match item? {
- LlmResponseChunk::Error { message } => return Err(message.into()),
+ LlmResponseChunk::DecodeError { message } => {
+ return Err(LlmClientError::ResponseTranslation(message));
+ }
+ LlmResponseChunk::StreamError { message } => {
+ // The upstream reported the failure inside the response body, so
+ // there is no real status line to carry; 502 stands in for "the
+ // upstream failed" the same way a failed non-streaming call would.
+ return Err(LlmClientError::UpstreamHttp {
+ status: MID_STREAM_UPSTREAM_STATUS,
+ body: message,
+ });
+ }
chunk => accumulator.push(chunk),
}
}
@@ -98,7 +115,10 @@ pub enum LlmResponseChunk {
MessageStop {
reason: Option<String>,
},
- Error {
+ DecodeError {
+ message: String,
+ },
+ StreamError {
message: String,
},
}
@@ -175,7 +195,7 @@ impl ResponseAccumulator {
LlmResponseChunk::MessageStop { reason } => {
self.stop_reason = Some(stop_reason_from_str(reason.as_deref()));
}
- LlmResponseChunk::Error { .. } => {}
+ LlmResponseChunk::DecodeError { .. } | LlmResponseChunk::StreamError { .. } => {}
}
}
@@ -236,6 +256,9 @@ fn stop_reason_from_str(reason: Option<&str>) -> StopReason {
#[cfg(test)]
mod tests {
+ use futures::executor::block_on;
+ use futures::stream;
+
use super::*;
use serde_json::json;
@@ -338,4 +361,18 @@ mod tests {
]
);
}
+
+ #[test]
+ fn into_agg_preserves_stream_item_error() {
+ let response = LlmResponse::Stream(Box::pin(stream::once(async {
+ Err(LlmClientError::Timeout {
+ source: Box::new(std::io::Error::other("timed out")),
+ })
+ })));
+
+ let Err(error) = block_on(response.into_agg()) else {
+ panic!("expected stream aggregation to fail");
+ };
+ assert!(matches!(error, LlmClientError::Timeout { .. }));
+ }
}
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 6a9918c2d..a0e21f139 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -194,12 +194,14 @@ fn subagent_override_algorithm(
}
fn other_python_error(error: PyErr) -> LlmClientError {
- LlmClientError::Other(Box::new(std::io::Error::other(error.to_string())))
+ LlmClientError::Ffi {
+ source: Box::new(error),
+ }
}
fn invalid_python_response(error: PyErr) -> LlmClientError {
LlmClientError::InvalidResponse {
- source: Box::new(std::io::Error::other(error.to_string())),
+ source: Box::new(error),
}
}
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index 8a6a6ee72..1a9bfdec3 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -48,7 +48,7 @@ fn decode_anthropic_stream(
event: &Value,
) -> Vec<LlmResponseChunk> {
let Some(object) = event.as_object() else {
- return vec![LlmResponseChunk::Error {
+ return vec![LlmResponseChunk::DecodeError {
message: "Anthropic stream event is not an object".to_string(),
}];
};
@@ -107,7 +107,7 @@ fn decode_anthropic_stream(
Some("message_stop") => vec![LlmResponseChunk::MessageStop {
reason: state.stop_reason.clone(),
}],
- Some("error") => vec![LlmResponseChunk::Error {
+ Some("error") => vec![LlmResponseChunk::StreamError {
message: object
.get("error")
.and_then(Value::as_object)
@@ -181,7 +181,7 @@ fn encode_anthropic_stream(
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- LlmResponseChunk::Error { message } => {
+ LlmResponseChunk::StreamError { message } | LlmResponseChunk::DecodeError { message } => {
vec![json!({"type": "error", "error": {"message": message}})]
}
}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index d14bdc321..c74774046 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -48,11 +48,23 @@ fn decode_openai_chat_stream(
event: &Value,
) -> Vec<LlmResponseChunk> {
let Some(object) = event.as_object() else {
- return vec![LlmResponseChunk::Error {
+ return vec![LlmResponseChunk::DecodeError {
message: "OpenAI stream event is not an object".to_string(),
}];
};
+ // Upstream error frames carry no choices, so without this they would decode to a bare
+ // `MessageStart` and the error text would be dropped.
+ if let Some(error) = object.get("error") {
+ return vec![LlmResponseChunk::StreamError {
+ message: error
+ .get("message")
+ .and_then(Value::as_str)
+ .unwrap_or("unknown OpenAI stream error")
+ .to_string(),
+ }];
+ }
+
let mut out = Vec::new();
if !state.saw_message_start {
state.saw_message_start = true;
@@ -204,7 +216,9 @@ fn encode_openai_chat_stream(
Some(openai_usage_value(state)),
)]
}
- LlmResponseChunk::Error { message } => vec![json!({"error": {"message": message}})],
+ LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {
+ vec![json!({"error": {"message": message}})]
+ }
}
}
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index e30190874..bf4e884bb 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -137,7 +137,7 @@ fn decode_responses_stream(
out.push(LlmResponseChunk::MessageStop { reason: None });
out
}
- Some("error") => vec![LlmResponseChunk::Error {
+ Some("error") => vec![LlmResponseChunk::StreamError {
message: event
.get("message")
.and_then(Value::as_str)
@@ -177,7 +177,7 @@ fn encode_responses_stream(
state.stop_reason = reason.or_else(|| state.stop_reason.clone());
Vec::new()
}
- LlmResponseChunk::Error { message } => {
+ LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {
vec![json!({"type": "error", "message": message})]
}
}
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index c2ddb7194..38cd73b91 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -206,7 +206,9 @@ pub fn decode_stream_event(
.codec(source)
.map(|codec| codec.decode_event(state, event))
.unwrap_or_else(|error| {
- vec![LlmResponseChunk::Error {
+ // A missing codec is a translation-side failure, not something the
+ // upstream sent, so it decodes to `DecodeError` rather than `StreamError`.
+ vec![LlmResponseChunk::DecodeError {
message: error.to_string(),
}]
})
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index 15f9c185c..d7486cf4a 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -13,6 +13,7 @@ use async_stream::try_stream;
use futures::io::AsyncBufReadExt;
use futures::{Stream, StreamExt, TryStreamExt};
use serde_json::Value;
+use switchyard_protocol::LlmClientError;
use crate::sse;
use crate::{
@@ -81,10 +82,15 @@ pub fn encode_stream(
chunks: LlmResponseStream,
target: WireFormat,
requested_model: Option<String>,
-) -> std::result::Result<RawEventStream, Box<dyn std::error::Error + Send + Sync>> {
+) -> std::result::Result<RawEventStream, LlmClientError> {
// The target is always a built-in wire format, so this lookup cannot fail; a
// failure returns as an `Err` rather than a panic.
- let codec = StreamCodecRegistry::with_builtins().codec(target)?;
+ let codec = StreamCodecRegistry::with_builtins()
+ .codec(target)
+ // Currently the only error is that the codec is missing, which is Configuration
+ .map_err(|err| LlmClientError::Configuration {
+ message: err.to_string(),
+ })?;
let mut state = StreamTranslationState {
target: Some(target.into()),
@@ -117,20 +123,22 @@ pub fn encode_stream(
pub fn decode_stream<S>(
bytes: S,
source: WireFormat,
-) -> std::result::Result<LlmResponseStream, Box<dyn std::error::Error + Send + Sync>>
+) -> std::result::Result<LlmResponseStream, LlmClientError>
where
- S: Stream<Item = std::result::Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>>>
- + Send
- + 'static,
+ S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
{
let marker = sse::done_marker(source);
// The source is always a built-in wire format, so this lookup cannot fail; a
// failure returns as an `Err` rather than a panic.
- let codec = StreamCodecRegistry::with_builtins().codec(source)?;
+ let codec = StreamCodecRegistry::with_builtins()
+ .codec(source)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
// Adapt the byte-chunk stream into an async line reader. The BufReader
// reassembles data split across network chunks (including multi-byte UTF-8),
// and `lines()` yields one SSE field line at a time. The stream is boxed to
- // an `io::Error` item so `into_async_read`'s error bound resolves cleanly.
+ // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The
+ // source error is boxed intact rather than stringified, so
+ // `llm_client_error_from_io` can recover its original variant on the way out.
let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =
Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));
let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();
@@ -140,10 +148,12 @@ where
let stream = Box::pin(try_stream! {
futures::pin_mut!(lines);
while let Some(line) = lines.next().await {
- let line = line?;
+ let line = line.map_err(llm_client_error_from_io)?;
// A blank line (allowing a bare CR for CRLF streams) ends the frame.
if line.trim_end().is_empty() {
- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)? {
+ if let Some(value) = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?
+ {
for event in codec.decode_event(&mut state, &value) {
yield event;
}
@@ -158,7 +168,9 @@ where
// A non-standard upstream might omit the final blank line; parse a trailing
// complete frame instead of losing its last chunk.
if !frame.trim_end().is_empty() {
- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)? {
+ if let Some(value) = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?
+ {
for event in codec.decode_event(&mut state, &value) {
yield event;
}
@@ -168,12 +180,28 @@ where
Ok(stream)
}
+// Recover transport errors wrapped for `AsyncRead`; other reader failures are
+// invalid upstream responses.
+fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {
+ let kind = error.kind();
+ let message = error.to_string();
+ match error.into_inner() {
+ Some(source) => match source.downcast::<LlmClientError>() {
+ Ok(error) => *error,
+ Err(source) => LlmClientError::InvalidResponse { source },
+ },
+ None => LlmClientError::InvalidResponse {
+ source: Box::new(std::io::Error::new(kind, message)),
+ },
+ }
+}
+
#[cfg(test)]
mod tests {
use futures::executor::block_on;
use futures::{stream, Stream, StreamExt};
use serde_json::{json, Value};
- use switchyard_protocol::{completion_text, LlmResponseChunk};
+ use switchyard_protocol::{completion_text, LlmClientError, LlmResponseChunk};
use super::{
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
@@ -186,9 +214,9 @@ mod tests {
// Collects a decoded IR stream, surfacing the first error instead of panicking.
fn decode_all(
- bytes: impl Stream<Item = Result<Vec<u8>, BoxError>> + Send + 'static,
+ bytes: impl Stream<Item = Result<Vec<u8>, LlmClientError>> + Send + 'static,
source: WireFormat,
- ) -> Result<Vec<LlmResponseChunk>, BoxError> {
+ ) -> Result<Vec<LlmResponseChunk>, LlmClientError> {
block_on(decode_stream(bytes, source)?.collect::<Vec<_>>())
.into_iter()
.collect()
@@ -278,10 +306,11 @@ mod tests {
#[test]
fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {
- let chunks: LlmResponseStream = stream::iter(vec![Err::<LlmResponseChunk, BoxError>(
- "chunk exploded".into(),
- )])
- .boxed();
+ let chunks: LlmResponseStream =
+ stream::iter(vec![Err::<LlmResponseChunk, LlmClientError>(
+ LlmClientError::General("chunk exploded".to_string()),
+ )])
+ .boxed();
let results =
block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());
assert!(results.iter().any(Result::is_err));
@@ -289,12 +318,12 @@ mod tests {
}
#[test]
- fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), BoxError> {
+ fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {
let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
data: [DONE]\n\n"
.to_vec();
- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });
+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
assert_eq!(text_of(&chunks), "Hello world");
Ok(())
@@ -309,7 +338,7 @@ mod tests {
let bytes = stream::iter(
sse.into_bytes()
.into_iter()
- .map(|byte| Ok::<Vec<u8>, BoxError>(vec![byte])),
+ .map(|byte| Ok::<Vec<u8>, LlmClientError>(vec![byte])),
);
let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
assert_eq!(text_of(&chunks), "café");
@@ -321,7 +350,7 @@ mod tests {
// A non-standard upstream omits the final blank line; the last frame
// must still be decoded rather than dropped.
let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}".to_vec();
- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });
+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
assert_eq!(text_of(&chunks), "tail");
Ok(())
@@ -334,7 +363,7 @@ mod tests {
let sse =
b"data: {\"choices\":[{\"delta\":{\"content\":\"crlf\"}}]}\r\n\r\ndata: [DONE]\r\n\r\n"
.to_vec();
- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });
+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
assert_eq!(text_of(&chunks), "crlf");
Ok(())
@@ -344,13 +373,30 @@ mod tests {
fn decode_stream_propagates_source_errors() -> Result<(), BoxError> {
// A transport error mid-stream surfaces as an error item, not a panic.
let bytes = stream::iter(vec![
- Ok::<Vec<u8>, BoxError>(
+ Ok::<Vec<u8>, LlmClientError>(
b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n\n".to_vec(),
),
- Err::<Vec<u8>, BoxError>("upstream exploded".into()),
+ Err::<Vec<u8>, LlmClientError>(LlmClientError::Transport {
+ source: Box::new(std::io::Error::other("upstream exploded")),
+ }),
]);
let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
- assert!(results.iter().any(Result::is_err));
+ let Some(Err(error)) = results.last() else {
+ panic!("expected the source error");
+ };
+ assert!(matches!(error, LlmClientError::Transport { .. }));
+ Ok(())
+ }
+
+ #[test]
+ fn decode_stream_classifies_invalid_sse_json() -> Result<(), BoxError> {
+ let bytes =
+ stream::once(async { Ok::<Vec<u8>, LlmClientError>(b"data: {invalid}\n\n".to_vec()) });
+ let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());
+ let Some(Err(error)) = results.last() else {
+ panic!("expected invalid SSE JSON to fail");
+ };
+ assert!(matches!(error, LlmClientError::ResponseTranslation(_)));
Ok(())
}
}
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 1c1685ea3..b0cbbc538 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -7,7 +7,7 @@ use pretty_assertions::assert_eq;
use serde_json::json;
use switchyard_protocol::{ResponseAccumulator, StopReason};
use switchyard_translation::{
- decode_stream_event, StreamTranslationState, TranslationEngine, WireFormat,
+ decode_stream_event, LlmResponseChunk, StreamTranslationState, TranslationEngine, WireFormat,
};
type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
@@ -545,3 +545,20 @@ fn anthropic_message_stop_does_not_overwrite_max_tokens_stop_reason() -> TestRes
);
Ok(())
}
+
+// An OpenAI-shaped error frame carries no `choices`, so it must decode to a stream error
+// instead of a bare message start that silently drops the upstream message.
+#[test]
+fn openai_chat_error_frame_decodes_to_stream_error() -> TestResult {
+ let mut state = StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiChat);
+ let event = json!({"error": {"message": "upstream exploded", "type": "server_error"}});
+
+ let chunks = decode_stream_event(&mut state, WireFormat::OpenAiChat, &event);
+
+ assert_eq!(chunks.len(), 1);
+ match &chunks[0] {
+ LlmResponseChunk::StreamError { message } => assert_eq!(message, "upstream exploded"),
+ other => return Err(format!("expected StreamError, got {other:?}").into()),
+ }
+ Ok(())
+}