Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(llm-client): fall back when the first stream event is a context overflow (#542)

Francesco Parisio21f6951bb993
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 96d0ab20d..4a517936f 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -5,16 +5,18 @@
//! request, call the configured backend over HTTP, decode the neutral response.
use std::collections::{BTreeMap, HashMap};
+use std::future::ready;
use std::time::{Duration, SystemTime};
use async_trait::async_trait;
-use futures_util::StreamExt;
+use futures_util::{StreamExt, stream};
use http::StatusCode;
use reqwest::RequestBuilder;
use reqwest::header::{HeaderMap, RETRY_AFTER};
use serde_json::{Map, Value};
use switchyard_protocol::{
- LlmRequest, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient,
+ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,
+ Response, RoutedLlmClient,
};
use switchyard_translation::{
WireFormat, decode_aggregated_response, decode_request, decode_stream,
@@ -445,8 +447,23 @@ impl TranslatingLlmClient {
}
})
});
- let chunks = decode_stream(bytes, wire_format)?;
- LlmResponse::Stream(chunks)
+ let mut chunks = decode_stream(bytes, wire_format)?;
+ // Providers reject an over-ceiling streaming request with an in-band
+ // error event on an HTTP 200. Classify the first event before returning
+ // the stream: nothing has reached the caller yet, so an overflow can
+ // still fail the call and let routing try the next candidate.
+ match chunks.next().await {
+ None => LlmResponse::Stream(stream::empty().boxed()),
+ Some(first) => {
+ if let Some(message) = first_event_overflow(&first, backend) {
+ return Err(LlmClientError::ContextWindowExceeded {
+ model: model_id.clone(),
+ message,
+ });
+ }
+ LlmResponse::Stream(stream::once(ready(first)).chain(chunks).boxed())
+ }
+ }
}
EncodedResponse::Buffered { body, .. } => {
let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
@@ -595,6 +612,25 @@ impl AttemptFailure {
}
}
+// The overflow message when a stream's first event is an in-band provider rejection
+// of the whole request, rather than the start of a response.
+fn first_event_overflow(
+ first: &Result<LlmResponseStreamEvent>,
+ backend: &Backend,
+) -> Option<String> {
+ first
+ .as_ref()
+ .ok()?
+ .normalized()
+ .iter()
+ .find_map(|chunk| match chunk {
+ LlmResponseChunk::StreamError { message } if backend.is_context_overflow(message) => {
+ Some(message.clone())
+ }
+ _ => None,
+ })
+}
+
// Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely.
fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {
let value = headers.get(RETRY_AFTER)?.to_str().ok()?;
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index 73d897c39..df14f22f8 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -729,4 +729,136 @@ mod tests {
assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);
Ok(())
}
+
+ // Verbatim shape of an SGLang admission rejection on a streaming session:
+ // HTTP 200, and the overflow arrives as the first SSE event.
+ const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";
+
+ // A normal one-turn streamed answer, as the second candidate serves it.
+ const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";
+
+ /// Runs a streaming request through the weak→strong candidate loop, serving
+ /// `weak_sse` from the first candidate and [`STRONG_SSE`] from the second.
+ /// The server is returned so response streams stay readable after the call.
+ async fn run_streaming_candidates(
+ weak_sse: &'static str,
+ ) -> (
+ MockServer,
+ Arc<Mutex<Vec<String>>>,
+ Result<(ModelId, Response)>,
+ ) {
+ let server = MockServer::start().await;
+ let calls = Arc::new(Mutex::new(Vec::new()));
+ let observed_calls = Arc::clone(&calls);
+ Mock::given(method("POST"))
+ .respond_with(move |request: &wiremock::Request| {
+ let body: serde_json::Value =
+ serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);
+ let model = body["model"].as_str().unwrap_or_default().to_string();
+ observed_calls.lock().push(model.clone());
+ let sse = if model == "weak" {
+ weak_sse
+ } else {
+ STRONG_SSE
+ };
+ ResponseTemplate::new(200)
+ .insert_header("content-type", "text/event-stream")
+ .set_body_string(sse)
+ })
+ .mount(&server)
+ .await;
+
+ let backend = || {
+ Backend::OpenAiChat(HttpBackendConfig {
+ base_url: format!("{}/v1", server.uri()),
+ api_key: None,
+ forward_auth: false,
+ extra_headers: BTreeMap::new(),
+ extra_body: BTreeMap::new(),
+ max_retries: 0,
+ })
+ };
+ let client = Arc::new(
+ TranslatingLlmClient::new(&[
+ ModelConfig::new("weak", backend(), None),
+ ModelConfig::new("strong", backend(), None),
+ ])
+ .expect("building test client"),
+ );
+ let algorithm = Arc::new(CandidateAlgorithm {
+ models: vec!["weak".into(), "strong".into()],
+ });
+ let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());
+ llm_request.stream = true;
+ let request = Request {
+ llm_request,
+ raw_request: None,
+ metadata: None,
+ };
+ let result = run(algorithm, ClientRouter::single(client), request, None).await;
+ (server, calls, result)
+ }
+
+ #[tokio::test]
+ async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {
+ // Nothing has reached the caller when the first event arrives, so an overflow
+ // there falls through to the next candidate exactly as its buffered twin does.
+ let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;
+ let (_, response) = result?;
+ assert_eq!(&*calls.lock(), &["weak", "strong"]);
+ assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));
+ let aggregate = response
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|source| LibsyError::client_call("strong", source))?;
+ assert_eq!(completion_text(&aggregate), "ok");
+
+ // Only a classified overflow re-routes; any other in-band error event still
+ // streams through to the consumer without trying another candidate.
+ let (_server, calls, result) = run_streaming_candidates(
+ "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
+ )
+ .await;
+ let (_, response) = result?;
+ assert_eq!(&*calls.lock(), &["weak"]);
+ let error = response
+ .llm_response
+ .into_agg()
+ .await
+ .expect_err("the in-band error should surface to the consumer");
+ assert!(error.to_string().contains("engine exploded"));
+
+ // Once content has streamed, a retry would repeat delivered output, so a later
+ // overflow event keeps today's semantics and surfaces during aggregation.
+ let (_server, calls, result) = run_streaming_candidates(
+ "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",
+ )
+ .await;
+ let (_, response) = result?;
+ assert_eq!(&*calls.lock(), &["weak"]);
+ let error = response
+ .llm_response
+ .into_agg()
+ .await
+ .expect_err("the mid-stream overflow should surface to the consumer");
+ assert!(
+ error
+ .to_string()
+ .contains("exceeds the maximum allowed length")
+ );
+
+ // A stream that ends before any event is not a classified failure; it still
+ // aggregates to an empty response.
+ let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;
+ let (_, response) = result?;
+ assert_eq!(&*calls.lock(), &["weak"]);
+ let aggregate = response
+ .llm_response
+ .into_agg()
+ .await
+ .map_err(|source| LibsyError::client_call("weak", source))?;
+ assert_eq!(completion_text(&aggregate), "");
+ Ok(())
+ }
}
diff --git a/docs/operations/context_window.md b/docs/operations/context_window.md
index bd0e6b0b8..6d9f5a24e 100644
--- a/docs/operations/context_window.md
+++ b/docs/operations/context_window.md
@@ -17,6 +17,13 @@ way — HTTP 413 or 422, for example — is not recognized as one. The request
fails on the spot with the upstream's status code, and the upstream's body is
returned inside Switchyard's error message.
+Streaming requests are covered as well. Some engines reject an over-limit
+streaming request with HTTP 200 and deliver the error as the first stream
+event; when that first event carries a message matching the phrases above, the
+call fails as an overflow and falls through before anything reaches the client.
+An error event arriving after content has streamed is not retried — delivered
+output cannot be taken back — and surfaces to the client as before.
+
## Configuration
There is no eviction key. A route falls through whenever it has more than one