Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(llm-client): Retry stream disconnects before doing fallback (#714)

Graham King9f038384f8c6
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 9d2acd16d..d36f732ee 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -15,8 +15,8 @@ use reqwest::RequestBuilder;
use reqwest::header::{HeaderMap, RETRY_AFTER};
use serde_json::{Map, Value, json};
use switchyard_protocol::{
- LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,
- Response, RoutedLlmClient,
+ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, Metadata,
+ ModelId, Request, Response, RoutedLlmClient,
};
use switchyard_translation::{
TranslationError, WireFormat, decode_aggregated_response, decode_request, decode_stream,
@@ -224,7 +224,7 @@ impl TranslatingLlmClient {
/// forwarded headers plus the backend's static headers and auth, and return the
/// successful upstream response. A
/// buffered response is fully collected within the retry boundary; a streamed
- /// response is returned as soon as its successful headers arrive. A non-success
+ /// response has its first decoded event checked within that boundary. A non-success
/// status maps to a typed error — a 400 is classified as a context-window
/// overflow via the backend's provider rules. Shared by
/// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the
@@ -359,9 +359,24 @@ impl TranslatingLlmClient {
let status = response.status();
if status.is_success() {
if streaming {
- // Streaming body failures happen after the retry boundary.
+ let upstream_headers = response.headers().clone();
+ let chunks = match prepare_response_stream(response, backend, model).await {
+ Ok(chunks) => chunks,
+ Err(error) => {
+ metrics::record_upstream_attempt(None);
+ return Err(AttemptFailure {
+ error,
+ status: Some(status),
+ retry_after: None,
+ });
+ }
+ };
metrics::record_upstream_attempt(Some(status.as_u16()));
- return Ok(EncodedResponse::Streaming(response));
+ return Ok(EncodedResponse::Streaming {
+ status: status.as_u16(),
+ chunks,
+ upstream_headers,
+ });
}
let upstream_headers = response.headers().clone();
let body = match response.bytes().await {
@@ -467,34 +482,11 @@ impl TranslatingLlmClient {
.await?;
let (llm_response, upstream_headers) = match http_response {
- EncodedResponse::Streaming(http_response) => {
- // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
- // transport-agnostic and lives in `switchyard-translation`.
- let upstream_headers = http_response.headers().clone();
- let bytes = http_response.bytes_stream().map(|chunk| {
- chunk
- .map(|bytes| bytes.to_vec())
- .map_err(convert_reqwest_error)
- });
- let mut chunks = decode_stream(bytes, wire_format)?;
- // Providers reject an over-ceiling streaming request with an in-band
- // error event on an HTTP 200. Classify the first event before returning
- // the stream: nothing has reached the caller yet, so an overflow can
- // still fail the call and let routing try the next candidate.
- let llm_response = match chunks.next().await {
- None => LlmResponse::Stream(stream::empty().boxed()),
- Some(first) => {
- if let Some(message) = first_event_overflow(&first, backend) {
- return Err(LlmClientError::ContextWindowExceeded {
- model: model_id.clone(),
- message,
- });
- }
- LlmResponse::Stream(stream::once(ready(first)).chain(chunks).boxed())
- }
- };
- (llm_response, upstream_headers)
- }
+ EncodedResponse::Streaming {
+ chunks,
+ upstream_headers,
+ ..
+ } => (LlmResponse::Stream(chunks), upstream_headers),
EncodedResponse::Buffered {
body,
upstream_headers,
@@ -637,14 +629,18 @@ enum EncodedResponse {
body: Vec<u8>,
upstream_headers: HeaderMap,
},
- Streaming(reqwest::Response),
+ Streaming {
+ status: u16,
+ chunks: LlmResponseStream,
+ upstream_headers: HeaderMap,
+ },
}
impl EncodedResponse {
fn status(&self) -> u16 {
match self {
- EncodedResponse::Buffered { status, .. } => *status,
- EncodedResponse::Streaming(response) => response.status().as_u16(),
+ EncodedResponse::Buffered { status, .. }
+ | EncodedResponse::Streaming { status, .. } => *status,
}
}
}
@@ -669,6 +665,36 @@ impl AttemptFailure {
}
}
+async fn prepare_response_stream(
+ response: reqwest::Response,
+ backend: &Backend,
+ model: &ModelId,
+) -> Result<LlmResponseStream> {
+ let bytes = response.bytes_stream().map(|chunk| {
+ chunk
+ .map(|bytes| bytes.to_vec())
+ .map_err(convert_reqwest_error)
+ });
+ let mut chunks = decode_stream(bytes, backend.wire_format())?;
+ match chunks.next().await {
+ None => Ok(stream::empty().boxed()),
+ // Nothing has reached the caller, so transport failures can still be retried.
+ Some(Err(error @ (LlmClientError::Transport { .. } | LlmClientError::Timeout { .. }))) => {
+ Err(error)
+ }
+ Some(first) => {
+ // An in-band context overflow skips retries and advances to another candidate.
+ if let Some(message) = first_event_overflow(&first, backend) {
+ return Err(LlmClientError::ContextWindowExceeded {
+ model: model.clone(),
+ message,
+ });
+ }
+ Ok(stream::once(ready(first)).chain(chunks).boxed())
+ }
+ }
+}
+
// The overflow message when a stream's first event is an in-band provider rejection
// of the whole request, rather than the start of a response.
fn first_event_overflow(
@@ -1641,6 +1667,40 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn pre_first_event_transport_failure_uses_retry_budget()
+ -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let truncated = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
+ Content-Length: 100\r\nConnection: close\r\n\r\n";
+ let body = "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"recovered\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
+ let success = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\
+ Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+
+ for second in [success, truncated.to_string()] {
+ let is_exhausted = second == truncated;
+ let (base_url, server) = response_sequence_server(vec![truncated.to_string(), second])?;
+ let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
+ let result = client
+ .call_rewrite_model(request_for(Some("gpt"), true), None)
+ .await;
+ if is_exhausted {
+ assert!(matches!(result, Err(LlmClientError::Transport { .. })));
+ } else {
+ assert_eq!(
+ completion_text(&result?.llm_response.into_agg().await?),
+ "recovered"
+ );
+ }
+ server
+ .join()
+ .map_err(|_| std::io::Error::other("response server thread panicked"))??;
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn streaming_body_io_failure_preserves_transport_error()
-> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {