Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(llm-client): fall back when the first stream event is a context overflow (#542)
Francesco Parisio21f6951bb993
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 96d0ab20d..4a517936f 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -5,16 +5,18 @@//! request, call the configured backend over HTTP, decode the neutral response.use std::collections::{BTreeMap, HashMap};+use std::future::ready;use std::time::{Duration, SystemTime};use async_trait::async_trait;-use futures_util::StreamExt;+use futures_util::{StreamExt, stream};use http::StatusCode;use reqwest::RequestBuilder;use reqwest::header::{HeaderMap, RETRY_AFTER};use serde_json::{Map, Value};use switchyard_protocol::{- LlmRequest, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient,+ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Metadata, ModelId, Request,+ Response, RoutedLlmClient,};use switchyard_translation::{WireFormat, decode_aggregated_response, decode_request, decode_stream,@@ -445,8 +447,23 @@ impl TranslatingLlmClient {}})});- let chunks = decode_stream(bytes, wire_format)?;- LlmResponse::Stream(chunks)+ let mut chunks = decode_stream(bytes, wire_format)?;+ // Providers reject an over-ceiling streaming request with an in-band+ // error event on an HTTP 200. Classify the first event before returning+ // the stream: nothing has reached the caller yet, so an overflow can+ // still fail the call and let routing try the next candidate.+ match chunks.next().await {+ None => LlmResponse::Stream(stream::empty().boxed()),+ Some(first) => {+ if let Some(message) = first_event_overflow(&first, backend) {+ return Err(LlmClientError::ContextWindowExceeded {+ model: model_id.clone(),+ message,+ });+ }+ LlmResponse::Stream(stream::once(ready(first)).chain(chunks).boxed())+ }+ }}EncodedResponse::Buffered { body, .. } => {let body = serde_json::from_slice::<Value>(&body).map_err(|error| {@@ -595,6 +612,25 @@ impl AttemptFailure {}}+// The overflow message when a stream's first event is an in-band provider rejection+// of the whole request, rather than the start of a response.+fn first_event_overflow(+ first: &Result<LlmResponseStreamEvent>,+ backend: &Backend,+) -> Option<String> {+ first+ .as_ref()+ .ok()?+ .normalized()+ .iter()+ .find_map(|chunk| match chunk {+ LlmResponseChunk::StreamError { message } if backend.is_context_overflow(message) => {+ Some(message.clone())+ }+ _ => None,+ })+}+// Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely.fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {let value = headers.get(RETRY_AFTER)?.to_str().ok()?;diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 73d897c39..df14f22f8 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -729,4 +729,136 @@ mod tests {assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);Ok(())}++ // Verbatim shape of an SGLang admission rejection on a streaming session:+ // HTTP 200, and the overflow arrives as the first SSE event.+ const OVERFLOW_SSE: &str = "data: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens). Use a shorter input or enable --allow-auto-truncate.\",\"object\":\"error\",\"param\":null,\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n";++ // A normal one-turn streamed answer, as the second candidate serves it.+ const STRONG_SSE: &str = "data: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":null}]}\n\ndata: {\"id\":\"answer\",\"model\":\"strong\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n";++ /// Runs a streaming request through the weak→strong candidate loop, serving+ /// `weak_sse` from the first candidate and [`STRONG_SSE`] from the second.+ /// The server is returned so response streams stay readable after the call.+ async fn run_streaming_candidates(+ weak_sse: &'static str,+ ) -> (+ MockServer,+ Arc<Mutex<Vec<String>>>,+ Result<(ModelId, Response)>,+ ) {+ let server = MockServer::start().await;+ let calls = Arc::new(Mutex::new(Vec::new()));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |request: &wiremock::Request| {+ let body: serde_json::Value =+ serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);+ let model = body["model"].as_str().unwrap_or_default().to_string();+ observed_calls.lock().push(model.clone());+ let sse = if model == "weak" {+ weak_sse+ } else {+ STRONG_SSE+ };+ ResponseTemplate::new(200)+ .insert_header("content-type", "text/event-stream")+ .set_body_string(sse)+ })+ .mount(&server)+ .await;++ let backend = || {+ Backend::OpenAiChat(HttpBackendConfig {+ base_url: format!("{}/v1", server.uri()),+ api_key: None,+ forward_auth: false,+ extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),+ max_retries: 0,+ })+ };+ let client = Arc::new(+ TranslatingLlmClient::new(&[+ ModelConfig::new("weak", backend(), None),+ ModelConfig::new("strong", backend(), None),+ ])+ .expect("building test client"),+ );+ let algorithm = Arc::new(CandidateAlgorithm {+ models: vec!["weak".into(), "strong".into()],+ });+ let mut llm_request = text_request(Some("auto".to_string()), "hello".to_string());+ llm_request.stream = true;+ let request = Request {+ llm_request,+ raw_request: None,+ metadata: None,+ };+ let result = run(algorithm, ClientRouter::single(client), request, None).await;+ (server, calls, result)+ }++ #[tokio::test]+ async fn first_stream_event_overflow_falls_back_to_the_next_candidate() -> Result<()> {+ // Nothing has reached the caller when the first event arrives, so an overflow+ // there falls through to the next candidate exactly as its buffered twin does.+ let (_server, calls, result) = run_streaming_candidates(OVERFLOW_SSE).await;+ let (_, response) = result?;+ assert_eq!(&*calls.lock(), &["weak", "strong"]);+ assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));+ let aggregate = response+ .llm_response+ .into_agg()+ .await+ .map_err(|source| LibsyError::client_call("strong", source))?;+ assert_eq!(completion_text(&aggregate), "ok");++ // Only a classified overflow re-routes; any other in-band error event still+ // streams through to the consumer without trying another candidate.+ let (_server, calls, result) = run_streaming_candidates(+ "data: {\"error\":{\"code\":500,\"message\":\"engine exploded\",\"object\":\"error\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",+ )+ .await;+ let (_, response) = result?;+ assert_eq!(&*calls.lock(), &["weak"]);+ let error = response+ .llm_response+ .into_agg()+ .await+ .expect_err("the in-band error should surface to the consumer");+ assert!(error.to_string().contains("engine exploded"));++ // Once content has streamed, a retry would repeat delivered output, so a later+ // overflow event keeps today's semantics and surfaces during aggregation.+ let (_server, calls, result) = run_streaming_candidates(+ "data: {\"id\":\"turn\",\"model\":\"weak\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"partial\"},\"finish_reason\":null}]}\n\ndata: {\"error\":{\"code\":400,\"message\":\"Input length (600013 tokens) exceeds the maximum allowed length (536826 tokens).\",\"object\":\"error\",\"type\":\"BAD_REQUEST\"},\"model\":\"weak\"}\n\ndata: [DONE]\n\n",+ )+ .await;+ let (_, response) = result?;+ assert_eq!(&*calls.lock(), &["weak"]);+ let error = response+ .llm_response+ .into_agg()+ .await+ .expect_err("the mid-stream overflow should surface to the consumer");+ assert!(+ error+ .to_string()+ .contains("exceeds the maximum allowed length")+ );++ // A stream that ends before any event is not a classified failure; it still+ // aggregates to an empty response.+ let (_server, calls, result) = run_streaming_candidates("data: [DONE]\n\n").await;+ let (_, response) = result?;+ assert_eq!(&*calls.lock(), &["weak"]);+ let aggregate = response+ .llm_response+ .into_agg()+ .await+ .map_err(|source| LibsyError::client_call("weak", source))?;+ assert_eq!(completion_text(&aggregate), "");+ Ok(())+ }}diff --git a/docs/operations/context_window.md b/docs/operations/context_window.mdindex bd0e6b0b8..6d9f5a24e 100644--- a/docs/operations/context_window.md+++ b/docs/operations/context_window.md@@ -17,6 +17,13 @@ way — HTTP 413 or 422, for example — is not recognized as one. The requestfails on the spot with the upstream's status code, and the upstream's body isreturned inside Switchyard's error message.+Streaming requests are covered as well. Some engines reject an over-limit+streaming request with HTTP 200 and deliver the error as the first stream+event; when that first event carries a message matching the phrases above, the+call fails as an overflow and falls through before anything reaches the client.+An error event arriving after content has streamed is not retried — delivered+output cannot be taken back — and surfaces to the client as before.+## ConfigurationThere is no eviction key. A route falls through whenever it has more than one