Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore(protocol): Use LlmClientError for stream too (#137)
Graham King5349735c8ec9
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex bf73cb7d4..cc1c2b88a 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -206,7 +206,7 @@ impl TranslatingLlmClient {// transport-agnostic and lives in `switchyard-translation`.let bytes = http_response.bytes_stream().map(|chunk| {chunk.map(|bytes| bytes.to_vec()).map_err(|error| {- let error = if error.is_timeout() {+ if error.is_timeout() {LlmClientError::Timeout {source: Box::new(error),}@@ -214,12 +214,10 @@ impl TranslatingLlmClient {LlmClientError::Transport {source: Box::new(error),}- };- Box::new(error) as Box<dyn std::error::Error + Send + Sync>+ }})});- let chunks = decode_stream(bytes, wire_format)- .map_err(|source| LlmClientError::InvalidResponse { source })?;+ let chunks = decode_stream(bytes, wire_format)?;LlmResponse::Stream(chunks)} else {let body = http_response@@ -288,8 +286,7 @@ impl TranslatingLlmClient {Ok(RawResponse::Buffered(body))}LlmResponse::Stream(chunks) => {- let events = encode_stream(chunks, wire_format, requested_model)- .map_err(|source| LlmClientError::InvalidResponse { source })?;+ let events = encode_stream(chunks, wire_format, requested_model)?;Ok(RawResponse::Stream(events))}}@@ -402,9 +399,17 @@ mod tests {)]}- fn truncated_response_server() -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {+ fn truncated_response_server(+ content_type: &str,+ body: &str,+ ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {let listener = std::net::TcpListener::bind("127.0.0.1:0")?;let address = listener.local_addr()?;+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\+ Content-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len() + 100+ );let handle = std::thread::spawn(move || {let (mut stream, _) = listener.accept()?;let mut request = [0_u8; 1024];@@ -414,10 +419,7 @@ mod tests {"client closed before sending a request",));}- stream.write_all(- b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\- Content-Length: 100\r\nConnection: close\r\n\r\n{}",- )+ stream.write_all(response.as_bytes())});Ok((format!("http://{address}/v1"), handle))}@@ -600,7 +602,7 @@ mod tests {#[tokio::test]async fn response_body_io_failure_is_a_transport_error() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {- let (base_url, server) = truncated_response_server()?;+ let (base_url, server) = truncated_response_server("application/json", "{}")?;let client = TranslatingLlmClient::new(&chat_map(&base_url))?;let result = client.call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)@@ -624,6 +626,28 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn streaming_body_io_failure_preserves_transport_error(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";+ let (base_url, server) = truncated_response_server("text/event-stream", body)?;+ let client = TranslatingLlmClient::new(&chat_map(&base_url))?;+ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)+ .await?;+ let result = response.llm_response.into_agg().await;+ server+ .join()+ .map_err(|_| std::io::Error::other("response server thread panicked"))??;++ let Err(error) = result else {+ panic!("expected the truncated stream body to fail");+ };++ assert!(matches!(error, LlmClientError::Transport { .. }));+ Ok(())+ }+#[tokio::test]async fn rewrites_model_to_resolved_upstream_id() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rsindex 5705865fd..b0c66a204 100644--- a/crates/libsy/examples/ensemble.rs+++ b/crates/libsy/examples/ensemble.rs@@ -461,7 +461,7 @@ async fn main() -> Result<()> {.llm_response.into_agg().await- .map_err(|error| LibsyError::external_boxed("aggregating response", error))?,+ .map_err(|error| LibsyError::external("aggregating response", error))?,));Ok(())diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rsindex 7cf53ea9c..892e162e3 100644--- a/crates/libsy/examples/research_agent.rs+++ b/crates/libsy/examples/research_agent.rs@@ -85,7 +85,7 @@ impl ResearchAgent {.llm_response.into_agg().await- .map_err(|error| LibsyError::external_boxed("aggregating response", error))?;+ .map_err(|error| LibsyError::external("aggregating response", error))?;notes.push(completion_text(&aggregate));}Ok(notes.join("\n"))diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rsindex ad8833e49..6567f5206 100644--- a/crates/libsy/examples/research_agent_core.rs+++ b/crates/libsy/examples/research_agent_core.rs@@ -80,7 +80,7 @@ impl ResearchAgent {Step::ReturnToAgent(response) => {let aggregate =response.llm_response.into_agg().await.map_err(|error| {- LibsyError::external_boxed("aggregating response", error)+ LibsyError::external("aggregating response", error)})?;notes.push(completion_text(&aggregate));}diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rsindex 08884e72a..a2b4f594c 100644--- a/crates/libsy/examples/streaming_agent.rs+++ b/crates/libsy/examples/streaming_agent.rs@@ -86,7 +86,7 @@ async fn main() -> Result<()> {print!("agent sees: ");while let Some(chunk) = chunks.next().await {let chunk = chunk.map_err(|error| {- LibsyError::external_boxed("reading response stream", error)+ LibsyError::external("reading response stream", error)})?;if let LlmResponseChunk::TextDelta { text, .. } = chunk {print!("{text}");diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 780d23117..b4e5f0b97 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -259,9 +259,7 @@ mod tests {.into_agg().await.map(|agg| completion_text(&agg))- .map_err(|error| {- LibsyError::external_boxed("aggregating fall-through response", error)- })?;+ .map_err(|error| LibsyError::external("aggregating fall-through response", error))?;Ok((text, trace))}diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 6f668311b..31ac50383 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -247,7 +247,9 @@ mod tests {};self.seen.lock()- .map_err(|_| switchyard_protocol::LlmClientError::Other("lock poisoned".into()))?+ .map_err(|_| {+ switchyard_protocol::LlmClientError::General("lock poisoned".to_string())+ })?.push(request);Ok(Response {llm_response: LlmResponse::Agg(text_response(None, completion)),diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex fdfb153cc..041c78f64 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -651,10 +651,11 @@ mod tests {]);let (trace, response) = orch.run(Context::default(), request()).await?;// `run` handed back the live stream; the caller folds it to a buffered aggregate.- let agg =- response.llm_response.into_agg().await.map_err(|error| {- LibsyError::external_boxed("aggregating response stream", error)- })?;+ let agg = response+ .llm_response+ .into_agg()+ .await+ .map_err(|error| LibsyError::external("aggregating response stream", error))?;assert_eq!(completion_text(&agg), "hello");assert_eq!(agg.model.as_deref(), Some("stream/model"));assert_eq!(trace.len(), 1);@@ -670,13 +671,13 @@ mod tests {index: 0,text: "partial".to_string(),},- LlmResponseChunk::Error {+ LlmResponseChunk::StreamError {message: "upstream exploded".to_string(),},]);let (_, response) = orch.run(Context::default(), request()).await?;match response.llm_response.into_agg().await {- Ok(_) => Err(test_error("expected a mid-stream error, got an aggregate")),+ Ok(_) => panic!("expected a mid-stream error, got an aggregate"),Err(err) => {assert!(err.to_string().contains("upstream exploded"));Ok(())diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rsindex 86b8f8877..503d916da 100644--- a/crates/libsy/src/error.rs+++ b/crates/libsy/src/error.rs@@ -95,14 +95,6 @@ impl LibsyError {source: Box::new(source),}}-- /// Preserve an already boxed foreign error.- pub fn external_boxed(- operation: &'static str,- source: Box<dyn StdError + Send + Sync>,- ) -> Self {- Self::External { operation, source }- }}/// Failures in the type-erased promise-over-stream driver.@@ -151,7 +143,7 @@ mod tests {fn client_call_preserves_target_and_source() {let error = LibsyError::client_call("strong",- LlmClientError::Other(Box::new(std::io::Error::other("upstream down"))),+ LlmClientError::General("upstream down".to_string()),);match &error {LibsyError::ClientCall { target, source } => {diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rsindex 47753c979..c04cb03e2 100644--- a/crates/protocol/src/client.rs+++ b/crates/protocol/src/client.rs@@ -23,7 +23,7 @@ pub type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;/// Failures a routed LLM client can surface to its caller.////// The variants classify failures that routing hosts commonly need to handle,-/// while boxed sources preserve implementation-specific detail. `Other` is the+/// while boxed sources preserve implementation-specific detail. `General` is the/// escape hatch for failures that do not fit a shared category.#[non_exhaustive]#[derive(Debug, Error)]@@ -90,9 +90,18 @@ pub enum LlmClientError {source: BoxError,},- /// A client-specific failure. Prefer adding variants than using this.- #[error(transparent)]- Other(#[from] BoxError),+ /// A call across a foreign-function boundary (e.g. a Python-implemented client)+ /// failed. The boxed source is the foreign error itself.+ #[error("foreign function interface error: {source}")]+ Ffi {+ /// Foreign-language failure, preserved verbatim.+ #[source]+ source: BoxError,+ },++ /// A string message. Useful in testing, but prefer adding variants over using this.+ #[error("{0}")]+ General(String),}/// A decision/trace object produced by an algorithm.diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rsindex 102f9b7d9..0db72066a 100644--- a/crates/protocol/src/stream.rs+++ b/crates/protocol/src/stream.rs@@ -6,21 +6,26 @@//! or the terminal [`AggLlmResponse`].use std::collections::BTreeMap;-use std::error::Error;use std::pin::Pin;use futures::{Stream, StreamExt};use serde::{Deserialize, Serialize};use serde_json::Value;-use crate::llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage};+use crate::{+ llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage},+ LlmClientError,+};-/// Boxed, thread-safe error carried by a stream item.-type BoxError = Box<dyn Error + Send + Sync>;+/// Status reported for an upstream error delivered inside a streaming body. The+/// upstream already sent a success status line before failing, so there is no real+/// code to propagate; 502 matches how a failed upstream call surfaces elsewhere.+const MID_STREAM_UPSTREAM_STATUS: u16 = 502;/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a/// streaming backend. Each item may fail independently mid-stream.-pub type LlmResponseStream = Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, BoxError>> + Send>>;+pub type LlmResponseStream =+ Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, LlmClientError>> + Send>>;/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or the/// terminal buffered [`Agg`](LlmResponse::Agg)regate.@@ -44,16 +49,28 @@ impl LlmResponse {/// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream`/// to completion, folding its chunks into an [`AggLlmResponse`] via- /// [`ResponseAccumulator`]. A stream item error, or an in-band- /// [`LlmResponseChunk::Error`], aborts with `Err`.- pub async fn into_agg(self) -> Result<AggLlmResponse, BoxError> {+ /// [`ResponseAccumulator`]. A stream item error aborts with `Err`, as does an+ /// in-band [`LlmResponseChunk::DecodeError`] (as `ResponseTranslation`) or+ /// [`LlmResponseChunk::StreamError`] (as `UpstreamHttp`).+ pub async fn into_agg(self) -> Result<AggLlmResponse, LlmClientError> {match self {LlmResponse::Agg(agg) => Ok(agg),LlmResponse::Stream(mut stream) => {let mut accumulator = ResponseAccumulator::new();while let Some(item) = stream.next().await {match item? {- LlmResponseChunk::Error { message } => return Err(message.into()),+ LlmResponseChunk::DecodeError { message } => {+ return Err(LlmClientError::ResponseTranslation(message));+ }+ LlmResponseChunk::StreamError { message } => {+ // The upstream reported the failure inside the response body, so+ // there is no real status line to carry; 502 stands in for "the+ // upstream failed" the same way a failed non-streaming call would.+ return Err(LlmClientError::UpstreamHttp {+ status: MID_STREAM_UPSTREAM_STATUS,+ body: message,+ });+ }chunk => accumulator.push(chunk),}}@@ -98,7 +115,10 @@ pub enum LlmResponseChunk {MessageStop {reason: Option<String>,},- Error {+ DecodeError {+ message: String,+ },+ StreamError {message: String,},}@@ -175,7 +195,7 @@ impl ResponseAccumulator {LlmResponseChunk::MessageStop { reason } => {self.stop_reason = Some(stop_reason_from_str(reason.as_deref()));}- LlmResponseChunk::Error { .. } => {}+ LlmResponseChunk::DecodeError { .. } | LlmResponseChunk::StreamError { .. } => {}}}@@ -236,6 +256,9 @@ fn stop_reason_from_str(reason: Option<&str>) -> StopReason {#[cfg(test)]mod tests {+ use futures::executor::block_on;+ use futures::stream;+use super::*;use serde_json::json;@@ -338,4 +361,18 @@ mod tests {]);}++ #[test]+ fn into_agg_preserves_stream_item_error() {+ let response = LlmResponse::Stream(Box::pin(stream::once(async {+ Err(LlmClientError::Timeout {+ source: Box::new(std::io::Error::other("timed out")),+ })+ })));++ let Err(error) = block_on(response.into_agg()) else {+ panic!("expected stream aggregation to fail");+ };+ assert!(matches!(error, LlmClientError::Timeout { .. }));+ }}diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 6a9918c2d..a0e21f139 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -194,12 +194,14 @@ fn subagent_override_algorithm(}fn other_python_error(error: PyErr) -> LlmClientError {- LlmClientError::Other(Box::new(std::io::Error::other(error.to_string())))+ LlmClientError::Ffi {+ source: Box::new(error),+ }}fn invalid_python_response(error: PyErr) -> LlmClientError {LlmClientError::InvalidResponse {- source: Box::new(std::io::Error::other(error.to_string())),+ source: Box::new(error),}}diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rsindex 8a6a6ee72..1a9bfdec3 100644--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs@@ -48,7 +48,7 @@ fn decode_anthropic_stream(event: &Value,) -> Vec<LlmResponseChunk> {let Some(object) = event.as_object() else {- return vec![LlmResponseChunk::Error {+ return vec![LlmResponseChunk::DecodeError {message: "Anthropic stream event is not an object".to_string(),}];};@@ -107,7 +107,7 @@ fn decode_anthropic_stream(Some("message_stop") => vec![LlmResponseChunk::MessageStop {reason: state.stop_reason.clone(),}],- Some("error") => vec![LlmResponseChunk::Error {+ Some("error") => vec![LlmResponseChunk::StreamError {message: object.get("error").and_then(Value::as_object)@@ -181,7 +181,7 @@ fn encode_anthropic_stream(state.stop_reason = reason.or_else(|| state.stop_reason.clone());Vec::new()}- LlmResponseChunk::Error { message } => {+ LlmResponseChunk::StreamError { message } | LlmResponseChunk::DecodeError { message } => {vec![json!({"type": "error", "error": {"message": message}})]}}diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex d14bdc321..c74774046 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -48,11 +48,23 @@ fn decode_openai_chat_stream(event: &Value,) -> Vec<LlmResponseChunk> {let Some(object) = event.as_object() else {- return vec![LlmResponseChunk::Error {+ return vec![LlmResponseChunk::DecodeError {message: "OpenAI stream event is not an object".to_string(),}];};+ // Upstream error frames carry no choices, so without this they would decode to a bare+ // `MessageStart` and the error text would be dropped.+ if let Some(error) = object.get("error") {+ return vec![LlmResponseChunk::StreamError {+ message: error+ .get("message")+ .and_then(Value::as_str)+ .unwrap_or("unknown OpenAI stream error")+ .to_string(),+ }];+ }+let mut out = Vec::new();if !state.saw_message_start {state.saw_message_start = true;@@ -204,7 +216,9 @@ fn encode_openai_chat_stream(Some(openai_usage_value(state)),)]}- LlmResponseChunk::Error { message } => vec![json!({"error": {"message": message}})],+ LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {+ vec![json!({"error": {"message": message}})]+ }}}diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex e30190874..bf4e884bb 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -137,7 +137,7 @@ fn decode_responses_stream(out.push(LlmResponseChunk::MessageStop { reason: None });out}- Some("error") => vec![LlmResponseChunk::Error {+ Some("error") => vec![LlmResponseChunk::StreamError {message: event.get("message").and_then(Value::as_str)@@ -177,7 +177,7 @@ fn encode_responses_stream(state.stop_reason = reason.or_else(|| state.stop_reason.clone());Vec::new()}- LlmResponseChunk::Error { message } => {+ LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {vec![json!({"type": "error", "message": message})]}}diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rsindex c2ddb7194..38cd73b91 100644--- a/crates/switchyard-translation/src/codecs/stream.rs+++ b/crates/switchyard-translation/src/codecs/stream.rs@@ -206,7 +206,9 @@ pub fn decode_stream_event(.codec(source).map(|codec| codec.decode_event(state, event)).unwrap_or_else(|error| {- vec![LlmResponseChunk::Error {+ // A missing codec is a translation-side failure, not something the+ // upstream sent, so it decodes to `DecodeError` rather than `StreamError`.+ vec![LlmResponseChunk::DecodeError {message: error.to_string(),}]})diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex 15f9c185c..d7486cf4a 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -13,6 +13,7 @@ use async_stream::try_stream;use futures::io::AsyncBufReadExt;use futures::{Stream, StreamExt, TryStreamExt};use serde_json::Value;+use switchyard_protocol::LlmClientError;use crate::sse;use crate::{@@ -81,10 +82,15 @@ pub fn encode_stream(chunks: LlmResponseStream,target: WireFormat,requested_model: Option<String>,-) -> std::result::Result<RawEventStream, Box<dyn std::error::Error + Send + Sync>> {+) -> std::result::Result<RawEventStream, LlmClientError> {// The target is always a built-in wire format, so this lookup cannot fail; a// failure returns as an `Err` rather than a panic.- let codec = StreamCodecRegistry::with_builtins().codec(target)?;+ let codec = StreamCodecRegistry::with_builtins()+ .codec(target)+ // Currently the only error is that the codec is missing, which is Configuration+ .map_err(|err| LlmClientError::Configuration {+ message: err.to_string(),+ })?;let mut state = StreamTranslationState {target: Some(target.into()),@@ -117,20 +123,22 @@ pub fn encode_stream(pub fn decode_stream<S>(bytes: S,source: WireFormat,-) -> std::result::Result<LlmResponseStream, Box<dyn std::error::Error + Send + Sync>>+) -> std::result::Result<LlmResponseStream, LlmClientError>where- S: Stream<Item = std::result::Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>>>- + Send- + 'static,+ S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,{let marker = sse::done_marker(source);// The source is always a built-in wire format, so this lookup cannot fail; a// failure returns as an `Err` rather than a panic.- let codec = StreamCodecRegistry::with_builtins().codec(source)?;+ let codec = StreamCodecRegistry::with_builtins()+ .codec(source)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;// Adapt the byte-chunk stream into an async line reader. The BufReader// reassembles data split across network chunks (including multi-byte UTF-8),// and `lines()` yields one SSE field line at a time. The stream is boxed to- // an `io::Error` item so `into_async_read`'s error bound resolves cleanly.+ // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The+ // source error is boxed intact rather than stringified, so+ // `llm_client_error_from_io` can recover its original variant on the way out.let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();@@ -140,10 +148,12 @@ wherelet stream = Box::pin(try_stream! {futures::pin_mut!(lines);while let Some(line) = lines.next().await {- let line = line?;+ let line = line.map_err(llm_client_error_from_io)?;// A blank line (allowing a bare CR for CRLF streams) ends the frame.if line.trim_end().is_empty() {- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)? {+ if let Some(value) = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?+ {for event in codec.decode_event(&mut state, &value) {yield event;}@@ -158,7 +168,9 @@ where// A non-standard upstream might omit the final blank line; parse a trailing// complete frame instead of losing its last chunk.if !frame.trim_end().is_empty() {- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)? {+ if let Some(value) = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?+ {for event in codec.decode_event(&mut state, &value) {yield event;}@@ -168,12 +180,28 @@ whereOk(stream)}+// Recover transport errors wrapped for `AsyncRead`; other reader failures are+// invalid upstream responses.+fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {+ let kind = error.kind();+ let message = error.to_string();+ match error.into_inner() {+ Some(source) => match source.downcast::<LlmClientError>() {+ Ok(error) => *error,+ Err(source) => LlmClientError::InvalidResponse { source },+ },+ None => LlmClientError::InvalidResponse {+ source: Box::new(std::io::Error::new(kind, message)),+ },+ }+}+#[cfg(test)]mod tests {use futures::executor::block_on;use futures::{stream, Stream, StreamExt};use serde_json::{json, Value};- use switchyard_protocol::{completion_text, LlmResponseChunk};+ use switchyard_protocol::{completion_text, LlmClientError, LlmResponseChunk};use super::{decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,@@ -186,9 +214,9 @@ mod tests {// Collects a decoded IR stream, surfacing the first error instead of panicking.fn decode_all(- bytes: impl Stream<Item = Result<Vec<u8>, BoxError>> + Send + 'static,+ bytes: impl Stream<Item = Result<Vec<u8>, LlmClientError>> + Send + 'static,source: WireFormat,- ) -> Result<Vec<LlmResponseChunk>, BoxError> {+ ) -> Result<Vec<LlmResponseChunk>, LlmClientError> {block_on(decode_stream(bytes, source)?.collect::<Vec<_>>()).into_iter().collect()@@ -278,10 +306,11 @@ mod tests {#[test]fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {- let chunks: LlmResponseStream = stream::iter(vec![Err::<LlmResponseChunk, BoxError>(- "chunk exploded".into(),- )])- .boxed();+ let chunks: LlmResponseStream =+ stream::iter(vec![Err::<LlmResponseChunk, LlmClientError>(+ LlmClientError::General("chunk exploded".to_string()),+ )])+ .boxed();let results =block_on(encode_stream(chunks, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>());assert!(results.iter().any(Result::is_err));@@ -289,12 +318,12 @@ mod tests {}#[test]- fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), BoxError> {+ fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\data: [DONE]\n\n".to_vec();- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;assert_eq!(text_of(&chunks), "Hello world");Ok(())@@ -309,7 +338,7 @@ mod tests {let bytes = stream::iter(sse.into_bytes().into_iter()- .map(|byte| Ok::<Vec<u8>, BoxError>(vec![byte])),+ .map(|byte| Ok::<Vec<u8>, LlmClientError>(vec![byte])),);let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;assert_eq!(text_of(&chunks), "café");@@ -321,7 +350,7 @@ mod tests {// A non-standard upstream omits the final blank line; the last frame// must still be decoded rather than dropped.let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"tail\"}}]}".to_vec();- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;assert_eq!(text_of(&chunks), "tail");Ok(())@@ -334,7 +363,7 @@ mod tests {let sse =b"data: {\"choices\":[{\"delta\":{\"content\":\"crlf\"}}]}\r\n\r\ndata: [DONE]\r\n\r\n".to_vec();- let bytes = stream::once(async move { Ok::<Vec<u8>, BoxError>(sse) });+ let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;assert_eq!(text_of(&chunks), "crlf");Ok(())@@ -344,13 +373,30 @@ mod tests {fn decode_stream_propagates_source_errors() -> Result<(), BoxError> {// A transport error mid-stream surfaces as an error item, not a panic.let bytes = stream::iter(vec![- Ok::<Vec<u8>, BoxError>(+ Ok::<Vec<u8>, LlmClientError>(b"data: {\"choices\":[{\"delta\":{\"content\":\"x\"}}]}\n\n".to_vec(),),- Err::<Vec<u8>, BoxError>("upstream exploded".into()),+ Err::<Vec<u8>, LlmClientError>(LlmClientError::Transport {+ source: Box::new(std::io::Error::other("upstream exploded")),+ }),]);let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());- assert!(results.iter().any(Result::is_err));+ let Some(Err(error)) = results.last() else {+ panic!("expected the source error");+ };+ assert!(matches!(error, LlmClientError::Transport { .. }));+ Ok(())+ }++ #[test]+ fn decode_stream_classifies_invalid_sse_json() -> Result<(), BoxError> {+ let bytes =+ stream::once(async { Ok::<Vec<u8>, LlmClientError>(b"data: {invalid}\n\n".to_vec()) });+ let results = block_on(decode_stream(bytes, WireFormat::OpenAiChat)?.collect::<Vec<_>>());+ let Some(Err(error)) = results.last() else {+ panic!("expected invalid SSE JSON to fail");+ };+ assert!(matches!(error, LlmClientError::ResponseTranslation(_)));Ok(())}}diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex 1c1685ea3..b0cbbc538 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -7,7 +7,7 @@ use pretty_assertions::assert_eq;use serde_json::json;use switchyard_protocol::{ResponseAccumulator, StopReason};use switchyard_translation::{- decode_stream_event, StreamTranslationState, TranslationEngine, WireFormat,+ decode_stream_event, LlmResponseChunk, StreamTranslationState, TranslationEngine, WireFormat,};type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;@@ -545,3 +545,20 @@ fn anthropic_message_stop_does_not_overwrite_max_tokens_stop_reason() -> TestRes);Ok(())}++// An OpenAI-shaped error frame carries no `choices`, so it must decode to a stream error+// instead of a bare message start that silently drops the upstream message.+#[test]+fn openai_chat_error_frame_decodes_to_stream_error() -> TestResult {+ let mut state = StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiChat);+ let event = json!({"error": {"message": "upstream exploded", "type": "server_error"}});++ let chunks = decode_stream_event(&mut state, WireFormat::OpenAiChat, &event);++ assert_eq!(chunks.len(), 1);+ match &chunks[0] {+ LlmResponseChunk::StreamError { message } => assert_eq!(message, "upstream exploded"),+ other => return Err(format!("expected StreamError, got {other:?}").into()),+ }+ Ok(())+}