Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(llm-client): add bounded upstream retries (#173)
nachiketb-nvidiaef646cd7632a
diff --git a/Cargo.lock b/Cargo.lockindex 74b718f5b..c15c0af6b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1750,12 +1750,14 @@ dependencies = ["async-trait","futures","futures-util",+ "httpdate","opentelemetry","reqwest","serde_json","switchyard-protocol","switchyard-translation","tokio",+ "tracing","wiremock",]diff --git a/Cargo.toml b/Cargo.tomlindex 0bca0c95b..7fd0c69cf 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -26,6 +26,7 @@ async-stream = "0.3"async-trait = "0.1"futures = "0.3"futures-util = "0.3"+httpdate = "1"parking_lot = "0.12"rand = "0.8"reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots", "stream"] }diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlindex 19b2470fe..54c6be9da 100644--- a/crates/libsy-llm-client/Cargo.toml+++ b/crates/libsy-llm-client/Cargo.toml@@ -18,9 +18,11 @@ reqwest.workspace = trueasync-trait.workspace = truefutures-util.workspace = trueopentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }+httpdate.workspace = trueserde_json.workspace = true+tokio.workspace = true+tracing.workspace = true[dev-dependencies]futures.workspace = true-tokio.workspace = truewiremock = "0.6"diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdindex 9e1a4ef81..a1d5bbbc4 100644--- a/crates/libsy-llm-client/README.md+++ b/crates/libsy-llm-client/README.md@@ -22,8 +22,9 @@ provider SDK.when set, otherwise the model's default backend.- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or`Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,- static `extra_headers`, default `extra_body` fields). The variant fixes the URL- path and auth scheme (Bearer vs `x-api-key` + `anthropic-version`).+ static `extra_headers`, default `extra_body` fields, and `max_retries`). The+ variant fixes the URL path and auth scheme (Bearer vs `x-api-key` ++ `anthropic-version`).- **Model rewrite.** The resolved model name is both the map key and the model idsent upstream — it overwrites whatever `model` the request arrived with.- **Streaming is chosen by the request.** If the encoded body has `stream: true`@@ -58,6 +59,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {api_key: std::env::var("OPENAI_API_KEY").ok(),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 2,};let models = [ModelConfig::new(@@ -173,6 +175,14 @@ fn build_multi_format_client(- Per-backend static headers go in `HttpBackendConfig::extra_headers`.- Per-target top-level request defaults go in `HttpBackendConfig::extra_body`.The merge is shallow and fields already present in the request take precedence.+- `HttpBackendConfig::max_retries` controls additional attempts after retryable+ transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body+ transport failures are retried; streaming body failures are not replayed after+ the response has been returned.++Retries replay the same upstream request. A transport failure can therefore+duplicate a request that the provider processed but did not finish returning,+and the retry budget plus capped `Retry-After` delays determines total latency.## Errorsdiff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rsindex 2c0555aac..f4c4f4fe0 100644--- a/crates/libsy-llm-client/src/backend.rs+++ b/crates/libsy-llm-client/src/backend.rs@@ -13,6 +13,9 @@ use crate::error::is_overflow_body;const ANTHROPIC_VERSION: &str = "2023-06-01";+/// Default number of retries for server-configured upstream calls.+pub const DEFAULT_MAX_RETRIES: u32 = 2;+// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new// provider-wrap is a one-line entry here, not a fork of the parsing logic.const OPENAI_OVERFLOW_PHRASES: &[&str] = &[@@ -42,6 +45,8 @@ pub struct HttpBackendConfig {pub extra_headers: BTreeMap<String, String>,/// Default top-level request fields, applied only when the request omits the key.pub extra_body: BTreeMap<String, Value>,+ /// Additional attempts after the initial upstream request.+ pub max_retries: u32,}impl fmt::Debug for HttpBackendConfig {@@ -51,6 +56,7 @@ impl fmt::Debug for HttpBackendConfig {.field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]")).field("extra_headers", &self.extra_headers).field("extra_body_keys", &self.extra_body.keys())+ .field("max_retries", &self.max_retries).finish()}}@@ -133,6 +139,11 @@ impl Backend {&self.config().extra_body}+ /// Additional attempts allowed after the initial request.+ pub fn max_retries(&self) -> u32 {+ self.config().max_retries+ }+/// Whether this backend speaks the Anthropic Messages wire format — the only/// one with a `count_tokens` endpoint.pub fn is_anthropic(&self) -> bool {@@ -196,6 +207,7 @@ mod tests {api_key: Some("secret".to_string()),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 0,}}diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex eeba631fb..565d3464a 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -5,9 +5,11 @@//! request, call the configured backend over HTTP, decode the neutral response.use std::collections::{BTreeMap, HashMap};+use std::time::{Duration, SystemTime};use async_trait::async_trait;use futures_util::StreamExt;+use reqwest::header::{HeaderMap, RETRY_AFTER};use reqwest::RequestBuilder;use serde_json::{Map, Value};use switchyard_protocol::{@@ -17,10 +19,11 @@ use switchyard_translation::{decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,encode_request, encode_stream, WireFormat,};+use tracing::Instrument;use crate::backend::Backend;use crate::error::{LlmClientError, Result};-use crate::metrics::record_upstream_attempt;+use crate::metrics::{is_retryable_http_status, record_upstream_attempt};use crate::raw::RawResponse;// TODO: Why is this here? What does it do?@@ -43,6 +46,10 @@ const RESERVED_HEADERS: &[&str] = &["content-type",];+const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);+const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2);+const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);+/// How one model is served: the `default_backend` used when the request does not/// pin a wire format, plus any `other_backends` reachable over additional formats.#[derive(Clone, Debug)]@@ -137,23 +144,24 @@ impl TranslatingLlmClient {/// Encode `llm_request` (its model restamped to `model`) for `wire_format`,/// POST it to `url` with the request's forwarded headers plus the backend's- /// static headers and auth, and return the successful upstream response and- /// whether the request asked for a streamed body. A non-success status maps- /// to a typed error — a 400 is classified as a context-window overflow via- /// the backend's provider rules. Shared by+ /// static headers and auth, and return the successful upstream response. A+ /// buffered response is fully collected within the retry boundary; a streamed+ /// response is returned as soon as its successful headers arrive. A non-success+ /// status maps to a typed error — a 400 is classified as a context-window+ /// overflow via the backend's provider rules. Shared by/// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the/// backend's completion URL and decodes a response) and/// [`count_tokens`](RoutedLlmClient::count_tokens) (which POSTs to the/// `count_tokens` URL and returns the raw JSON).async fn send_encoded(&self,- url: String,backend: &Backend,wire_format: WireFormat,mut llm_request: LlmRequest,metadata: Option<&Metadata>,model: &str,- ) -> Result<(reqwest::Response, bool)> {+ endpoint: UpstreamEndpoint,+ ) -> Result<EncodedResponse> {// The resolved name is the upstream model id (per the crate contract).llm_request.model = Some(model.to_string());let mut body = encode_request(&llm_request, wire_format)@@ -166,46 +174,142 @@ impl TranslatingLlmClient {if matches!(backend, Backend::OpenAiChat(_)) {ensure_openai_stream_usage(&mut body);}- let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);+ let streaming = endpoint.allows_streaming()+ && body.get("stream").and_then(Value::as_bool).unwrap_or(false);+ let url = endpoint.url(backend);++ let max_retries = u64::from(backend.max_retries());+ let max_attempts = max_retries + 1;+ let mut attempt = 0_u64;+ loop {+ let span = tracing::debug_span!(+ target: "libsy",+ "libsy.upstream_attempt",+ model,+ wire_format = %wire_format,+ attempt = attempt + 1,+ max_attempts,+ retry = attempt > 0,+ outcome = tracing::field::Empty,+ status_code = tracing::field::Empty,+ will_retry = tracing::field::Empty,+ retry_delay_ms = tracing::field::Empty,+ );+ let result = self+ .send_once(&url, backend, &body, metadata, model, streaming)+ .instrument(span.clone())+ .await;+ // The retained handle updates this same attempt span with its outcome.+ match result {+ Ok(response) => {+ span.record("outcome", "success");+ span.record("status_code", response.status());+ span.record("will_retry", false);+ return Ok(response);+ }+ Err(failure) => {+ let will_retry = attempt < max_retries && failure.is_retryable();+ span.record("outcome", "error");+ if let Some(status) = failure.status {+ span.record("status_code", status);+ }+ span.record("will_retry", will_retry);+ if !will_retry {+ return Err(failure.error);+ }- let builder = self.client.post(url).json(&body);+ let delay = retry_delay(attempt, failure.retry_after);+ span.record("retry_delay_ms", duration_millis(delay));+ // Close the attempt span before sleeping so backoff is not attempt latency.+ drop(span);+ tokio::time::sleep(delay).await;+ attempt += 1;+ }+ }+ }+ }++ // Performs one HTTP attempt and retains the retry metadata alongside any error.+ async fn send_once(+ &self,+ url: &str,+ backend: &Backend,+ body: &Value,+ metadata: Option<&Metadata>,+ model: &str,+ streaming: bool,+ ) -> std::result::Result<EncodedResponse, AttemptFailure> {+ let builder = self.client.post(url).json(body);let builder = forward_metadata_headers(builder, metadata);let builder = apply_extra_headers(builder, backend);let builder = backend.apply_auth(builder);- let http_response = match builder.send().await {- Ok(response) => {- record_upstream_attempt(Some(response.status().as_u16()));- response- }+ let response = match builder.send().await {+ Ok(response) => response,Err(error) => {record_upstream_attempt(None);- return Err(convert_reqwest_error(error));+ return Err(AttemptFailure {+ error: convert_reqwest_error(error),+ status: None,+ retry_after: None,+ });}};- let status = http_response.status();- if !status.is_success() {- let body = match http_response.text().await {+ let status = response.status();+ if status.is_success() {+ if streaming {+ // Streaming body failures happen after the retry boundary.+ record_upstream_attempt(Some(status.as_u16()));+ return Ok(EncodedResponse::Streaming(response));+ }+ let body = match response.bytes().await {Ok(body) => body,- Err(error) if error.is_timeout() => {- return Err(LlmClientError::Timeout {- source: Box::new(error),- })+ Err(error) => {+ record_upstream_attempt(None);+ return Err(AttemptFailure {+ error: convert_reqwest_error(error),+ status: Some(status.as_u16()),+ retry_after: None,+ });}- Err(error) => format!("<failed to read error body: {error}>"),};- if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {- return Err(LlmClientError::ContextWindowExceeded {- model: model.to_string(),- message: body,- });- }- return Err(LlmClientError::UpstreamHttp {+ record_upstream_attempt(Some(status.as_u16()));+ return Ok(EncodedResponse::Buffered {status: status.as_u16(),- body,+ body: body.to_vec(),});}- Ok((http_response, streaming))++ let retry_after = retry_after_delay(response.headers());+ let body = match response.text().await {+ Ok(body) => body,+ Err(error) => {+ record_upstream_attempt(None);+ return Err(AttemptFailure {+ error: convert_reqwest_error(error),+ status: Some(status.as_u16()),+ retry_after,+ });+ }+ };+ record_upstream_attempt(Some(status.as_u16()));+ let error =+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {+ LlmClientError::ContextWindowExceeded {+ model: model.to_string(),+ message: body,+ }+ } else {+ LlmClientError::UpstreamHttp {+ status: status.as_u16(),+ body,+ }+ };+ Err(AttemptFailure {+ error,+ status: Some(status.as_u16()),+ retry_after,+ })}/// Calls the backend for `model_name` (or the request's own model), over the@@ -252,43 +356,45 @@ impl TranslatingLlmClient {message: format!("model {model:?} has no backend for format {wire_format}"),})?;- let (http_response, streaming) = self+ let http_response = self.send_encoded(- backend.url(),backend,wire_format,llm_request,metadata.as_ref(),&model,+ UpstreamEndpoint::Completion,).await?;- let llm_response = if streaming {- // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is- // transport-agnostic and lives in `switchyard-translation`.- let bytes = http_response.bytes_stream().map(|chunk| {- chunk.map(|bytes| bytes.to_vec()).map_err(|error| {- if error.is_timeout() {- LlmClientError::Timeout {- source: Box::new(error),- }- } else {- LlmClientError::Transport {- source: Box::new(error),+ let llm_response = match http_response {+ EncodedResponse::Streaming(http_response) => {+ // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is+ // transport-agnostic and lives in `switchyard-translation`.+ let bytes = http_response.bytes_stream().map(|chunk| {+ chunk.map(|bytes| bytes.to_vec()).map_err(|error| {+ if error.is_timeout() {+ LlmClientError::Timeout {+ source: Box::new(error),+ }+ } else {+ LlmClientError::Transport {+ source: Box::new(error),+ }}- }- })- });- let chunks = decode_stream(bytes, wire_format)?;- LlmResponse::Stream(chunks)- } else {- let body = http_response- .json::<Value>()- .await- .map_err(convert_reqwest_error)?;- let agg = decode_aggregated_response(&body, wire_format)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;- LlmResponse::Agg(agg)+ })+ });+ let chunks = decode_stream(bytes, wire_format)?;+ LlmResponse::Stream(chunks)+ }+ EncodedResponse::Buffered { body, .. } => {+ let body = serde_json::from_slice::<Value>(&body).map_err(|error| {+ LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))+ })?;+ let agg = decode_aggregated_response(&body, wire_format)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ LlmResponse::Agg(agg)+ }};Ok(Response {@@ -387,38 +493,120 @@ impl RoutedLlmClient for TranslatingLlmClient {metadata,..} = request;- let (http_response, _) = self+ let http_response = self.send_encoded(- backend.count_tokens_url(),backend,WireFormat::AnthropicMessages,llm_request,metadata.as_ref(),model,+ UpstreamEndpoint::CountTokens,).await?;- let text = http_response.text().await.map_err(convert_reqwest_error)?;- serde_json::from_str(&text).map_err(|error| LlmClientError::InvalidResponse {+ let body = match http_response {+ EncodedResponse::Buffered { body, .. } => body,+ EncodedResponse::Streaming(_) => {+ return Err(LlmClientError::InvalidRequest {+ message: "count_tokens does not support streaming requests".to_string(),+ })+ }+ };+ serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {source: Box::new(error),})}}+#[derive(Clone, Copy)]+enum UpstreamEndpoint {+ Completion,+ CountTokens,+}++impl UpstreamEndpoint {+ fn url(self, backend: &Backend) -> String {+ match self {+ UpstreamEndpoint::Completion => backend.url(),+ UpstreamEndpoint::CountTokens => backend.count_tokens_url(),+ }+ }++ fn allows_streaming(self) -> bool {+ matches!(self, UpstreamEndpoint::Completion)+ }+}++enum EncodedResponse {+ Buffered { status: u16, body: Vec<u8> },+ Streaming(reqwest::Response),+}++impl EncodedResponse {+ fn status(&self) -> u16 {+ match self {+ EncodedResponse::Buffered { status, .. } => *status,+ EncodedResponse::Streaming(response) => response.status().as_u16(),+ }+ }+}++// The typed error decides retry eligibility; status and Retry-After feed+// attempt telemetry and delay selection.+struct AttemptFailure {+ error: LlmClientError,+ status: Option<u16>,+ retry_after: Option<Duration>,+}++impl AttemptFailure {+ fn is_retryable(&self) -> bool {+ match &self.error {+ LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,+ LlmClientError::UpstreamHttp { status, .. } => is_retryable_http_status(*status),+ _ => false,+ }+ }+}++// Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely.+fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {+ let value = headers.get(RETRY_AFTER)?.to_str().ok()?;+ let delay = if let Ok(seconds) = value.parse::<u64>() {+ Duration::from_secs(seconds)+ } else {+ let retry_at = httpdate::parse_http_date(value).ok()?;+ retry_at+ .duration_since(SystemTime::now())+ .unwrap_or(Duration::ZERO)+ };+ Some(delay.min(MAX_RETRY_AFTER))+}++fn retry_delay(retry_number: u64, retry_after: Option<Duration>) -> Duration {+ // Retry-After wins; otherwise double 250 ms up to the two-second cap.+ retry_after.unwrap_or_else(|| {+ let multiplier = 1_u32 << retry_number.min(3);+ INITIAL_RETRY_DELAY+ .saturating_mul(multiplier)+ .min(MAX_RETRY_BACKOFF)+ })+}++fn duration_millis(duration: Duration) -> u64 {+ u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)+}+fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {- // `Response::json` labels body collection and JSON parsing failures as- // decode errors; only the latter is a translation failure.+ // Reqwest labels truncated or otherwise unreadable response bodies as decode+ // errors, so distinguish them from serde JSON failures at the call site.if error.is_timeout() {LlmClientError::Timeout {source: Box::new(error),}- } else if error.is_body() {- LlmClientError::Transport {- source: Box::new(error),+ } else if error.is_builder() {+ LlmClientError::Configuration {+ message: format!("failed to build upstream request: {error}"),}- } else if error.is_decode()- && std::error::Error::source(&error).is_some_and(|source| source.is::<serde_json::Error>())- {- LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))} else {LlmClientError::Transport {source: Box::new(error),@@ -503,6 +691,8 @@ mod tests {use std::collections::BTreeMap;use std::error::Error;use std::io::{Read, Write};+ use std::sync::atomic::{AtomicUsize, Ordering};+ use std::sync::Arc;use std::thread::JoinHandle;use serde_json::json;@@ -519,6 +709,14 @@ mod tests {api_key: Some("secret".to_string()),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 0,+ }+ }++ fn config_with_retries(base_url: &str, max_retries: u32) -> HttpBackendConfig {+ HttpBackendConfig {+ max_retries,+ ..config(base_url)}}@@ -540,31 +738,69 @@ mod tests {vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]}+ fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec<ModelConfig> {+ vec![ModelConfig::new(+ "gpt",+ Backend::OpenAiChat(config_with_retries(base_url, max_retries)),+ None,+ )]+ }++ fn chat_success_response() -> ResponseTemplate {+ ResponseTemplate::new(200).set_body_json(json!({+ "id": "chatcmpl-1",+ "model": "gpt",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "recovered"},+ "finish_reason": "stop"+ }],+ "usage": {}+ }))+ }+fn truncated_response_server(content_type: &str,body: &str,) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {- let listener = std::net::TcpListener::bind("127.0.0.1:0")?;- let address = listener.local_addr()?;- let response = format!(+ response_sequence_server(vec![format!("HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\Content-Length: {}\r\nConnection: close\r\n\r\n{body}",body.len() + 100- );+ )])+ }++ fn response_sequence_server(+ responses: Vec<String>,+ ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let address = listener.local_addr()?;let handle = std::thread::spawn(move || {- let (mut stream, _) = listener.accept()?;- let mut request = [0_u8; 1024];- if stream.read(&mut request)? == 0 {- return Err(std::io::Error::new(- std::io::ErrorKind::UnexpectedEof,- "client closed before sending a request",- ));+ for response in responses {+ let (mut stream, _) = listener.accept()?;+ let mut request = [0_u8; 1024];+ if stream.read(&mut request)? == 0 {+ return Err(std::io::Error::new(+ std::io::ErrorKind::UnexpectedEof,+ "client closed before sending a request",+ ));+ }+ stream.write_all(response.as_bytes())?;}- stream.write_all(response.as_bytes())+ Ok(())});Ok((format!("http://{address}/v1"), handle))}+ fn raw_chat_success_response() -> String {+ let body = r#"{"id":"chatcmpl-1","model":"gpt","choices":[{"index":0,"message":{"role":"assistant","content":"recovered"},"finish_reason":"stop"}],"usage":{}}"#;+ format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\+ Content-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ )+ }+fn request_for(model: Option<&str>, stream: bool) -> Request {let mut llm_request = text_request(model.map(str::to_string), "hi");llm_request.stream = stream;@@ -719,12 +955,18 @@ mod tests {async fn invalid_json_is_a_response_translation_error() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {let server = MockServer::start().await;+ let calls = Arc::new(AtomicUsize::new(0));+ let observed_calls = Arc::clone(&calls);Mock::given(method("POST"))- .respond_with(ResponseTemplate::new(200).set_body_raw("not json", "application/json"))+ .respond_with(move |_: &wiremock::Request| {+ observed_calls.fetch_add(1, Ordering::SeqCst);+ ResponseTemplate::new(200).set_body_raw("not json", "application/json")+ }).mount(&server).await;- let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;+ let client =+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;let Err(error) = client.call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None).await@@ -737,6 +979,7 @@ mod tests {LlmClientError::ResponseTranslation(message)if message.contains("invalid upstream JSON")));+ assert_eq!(calls.load(Ordering::SeqCst), 1);Ok(())}@@ -767,12 +1010,41 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn response_body_transport_failures_are_retried(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let truncated_responses = [+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\+ Content-Length: 102\r\nConnection: close\r\n\r\n{}",+ "HTTP/1.1 401 Unauthorized\r\nContent-Type: text/plain\r\n\+ Content-Length: 100\r\nConnection: close\r\n\r\nbad",+ ];++ for truncated in truncated_responses {+ let (base_url, server) =+ response_sequence_server(vec![truncated.to_string(), raw_chat_success_response()])?;+ let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;+ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await?;+ server+ .join()+ .map_err(|_| std::io::Error::other("response server thread panicked"))??;++ assert_eq!(+ completion_text(&response.llm_response.into_agg().await?),+ "recovered"+ );+ }+ Ok(())+ }+#[tokio::test]async fn streaming_body_io_failure_preserves_transport_error() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";let (base_url, server) = truncated_response_server("text/event-stream", body)?;- let client = TranslatingLlmClient::new(&chat_map(&base_url))?;+ let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 2))?;let response = client.call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None).await?;@@ -961,6 +1233,213 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn retryable_http_failure_recovers_within_budget(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let calls = Arc::new(AtomicUsize::new(0));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |_: &wiremock::Request| {+ if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {+ ResponseTemplate::new(503)+ .insert_header("retry-after", "0")+ .set_body_string("temporarily unavailable")+ } else {+ chat_success_response()+ }+ })+ .mount(&server)+ .await;++ let client =+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;+ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await?;+ let agg = response.llm_response.into_agg().await?;++ assert_eq!(completion_text(&agg), "recovered");+ assert_eq!(calls.load(Ordering::SeqCst), 2);+ Ok(())+ }++ #[tokio::test]+ async fn deterministic_http_failure_is_not_retried(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let calls = Arc::new(AtomicUsize::new(0));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |_: &wiremock::Request| {+ observed_calls.fetch_add(1, Ordering::SeqCst);+ ResponseTemplate::new(401).set_body_string("invalid key")+ })+ .mount(&server)+ .await;++ let client =+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;+ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected an upstream error");+ };++ assert!(matches!(+ error,+ LlmClientError::UpstreamHttp {+ status: 401,+ body+ } if body == "invalid key"+ ));+ assert_eq!(calls.load(Ordering::SeqCst), 1);+ Ok(())+ }++ #[tokio::test]+ async fn retry_exhaustion_returns_the_final_upstream_error(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let calls = Arc::new(AtomicUsize::new(0));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |_: &wiremock::Request| {+ let attempt = observed_calls.fetch_add(1, Ordering::SeqCst) + 1;+ ResponseTemplate::new(500)+ .insert_header("retry-after", "0")+ .set_body_string(format!("attempt {attempt}"))+ })+ .mount(&server)+ .await;++ let client =+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;+ let Err(error) = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await+ else {+ panic!("expected retry exhaustion");+ };++ assert!(matches!(+ error,+ LlmClientError::UpstreamHttp {+ status: 500,+ body+ } if body == "attempt 3"+ ));+ assert_eq!(calls.load(Ordering::SeqCst), 3);+ Ok(())+ }++ #[tokio::test]+ async fn timeout_is_retried_before_a_response_is_returned(+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {+ let server = MockServer::start().await;+ let calls = Arc::new(AtomicUsize::new(0));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |_: &wiremock::Request| {+ if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {+ ResponseTemplate::new(200).set_delay(Duration::from_millis(500))+ } else {+ chat_success_response()+ }+ })+ .mount(&server)+ .await;++ let mut client =+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;+ client.client = reqwest::Client::builder()+ .timeout(Duration::from_millis(100))+ .build()?;+ let response = client+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)+ .await?;++ assert_eq!(+ completion_text(&response.llm_response.into_agg().await?),+ "recovered"+ );+ assert_eq!(calls.load(Ordering::SeqCst), 2);+ Ok(())+ }++ #[test]+ fn retryable_error_classes_are_explicit() {+ let transport = AttemptFailure {+ error: LlmClientError::Transport {+ source: std::io::Error::other("disconnected").into(),+ },+ status: None,+ retry_after: None,+ };+ assert!(transport.is_retryable());++ for status in [408, 429, 500, 503, 599] {+ let failure = AttemptFailure {+ error: LlmClientError::UpstreamHttp {+ status,+ body: String::new(),+ },+ status: Some(status),+ retry_after: None,+ };+ assert!(failure.is_retryable(), "HTTP {status} should retry");+ }+ for status in [400, 401, 409, 600] {+ let failure = AttemptFailure {+ error: LlmClientError::UpstreamHttp {+ status,+ body: String::new(),+ },+ status: Some(status),+ retry_after: None,+ };+ assert!(!failure.is_retryable(), "HTTP {status} should fail fast");+ }++ let configuration = AttemptFailure {+ error: LlmClientError::Configuration {+ message: "invalid header".to_string(),+ },+ status: None,+ retry_after: None,+ };+ assert!(!configuration.is_retryable());++ let context_window = AttemptFailure {+ error: LlmClientError::ContextWindowExceeded {+ model: "gpt".to_string(),+ message: "too long".to_string(),+ },+ status: Some(400),+ retry_after: None,+ };+ assert!(!context_window.is_retryable());+ }++ #[test]+ fn retry_after_supports_seconds_and_http_dates() {+ let mut headers = HeaderMap::new();+ headers.insert(RETRY_AFTER, reqwest::header::HeaderValue::from_static("3"));+ assert_eq!(retry_after_delay(&headers), Some(Duration::from_secs(3)));++ let retry_at = SystemTime::now() + Duration::from_secs(2);+ let value = httpdate::fmt_http_date(retry_at);+ let Ok(value) = reqwest::header::HeaderValue::from_str(&value) else {+ panic!("formatted HTTP date should be a valid header");+ };+ headers.insert(RETRY_AFTER, value);+ let Some(delay) = retry_after_delay(&headers) else {+ panic!("HTTP date should produce a retry delay");+ };+ assert!(delay <= Duration::from_secs(2));+ }+#[tokio::test]async fn routed_llm_client_exposes_timeout_variant() -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex 79fb84e3a..8fa213828 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -17,7 +17,7 @@ pub mod error;pub mod metrics;pub mod raw;-pub use backend::{Backend, HttpBackendConfig};+pub use backend::{Backend, HttpBackendConfig, DEFAULT_MAX_RETRIES};pub use client::{ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};pub use raw::RawResponse;diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rsindex 67202463d..50985e75a 100644--- a/crates/libsy-llm-client/src/metrics.rs+++ b/crates/libsy-llm-client/src/metrics.rs@@ -5,10 +5,15 @@use opentelemetry::{global, KeyValue};+pub(crate) const fn is_retryable_http_status(status: u16) -> bool {+ status == 408 || status == 429 || (status >= 500 && status <= 599)+}+pub const fn http_outcome_label(status: Option<u16>) -> &'static str {match status {Some(200..=299) => "success",- Some(429 | 500 | 504) | None => "retryable_error",+ Some(status) if is_retryable_http_status(status) => "retryable_error",+ None => "retryable_error",Some(_) => "other_error",}}@@ -53,3 +58,21 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {],);}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn outcome_labels_match_the_retry_policy() {+ for status in [408, 429, 500, 502, 599] {+ assert!(is_retryable_http_status(status));+ assert_eq!(http_outcome_label(Some(status)), "retryable_error");+ }+ for status in [400, 409, 499, 600] {+ assert!(!is_retryable_http_status(status));+ assert_eq!(http_outcome_label(Some(status)), "other_error");+ }+ assert_eq!(http_outcome_label(None), "retryable_error");+ }+}diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rsindex 14376fde4..d069e188b 100644--- a/crates/libsy/examples/ensemble.rs+++ b/crates/libsy/examples/ensemble.rs@@ -412,6 +412,7 @@ async fn main() -> Result<()> {),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 2,};let models: Vec<_> = CANDIDATE_MODELS.iter()diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.mdindex ff0ecbba8..44ed47d95 100644--- a/crates/switchyard-server/CONFIGURATION.md+++ b/crates/switchyard-server/CONFIGURATION.md@@ -9,6 +9,7 @@ Define the upstream once under `llm_clients`, then reference it from targets.format = "openai_chat"base_url = "https://example.com/v1"api_key_env = "PROVIDER_API_KEY"+max_retries = 2[targets.model]id = "provider/model"diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.mdindex a7060d9cd..34b93fac9 100644--- a/crates/switchyard-server/README.md+++ b/crates/switchyard-server/README.md@@ -12,6 +12,7 @@ schema_version = 1format = "openai_chat"base_url = "https://example.com/v1"api_key_env = "API_KEY"+max_retries = 2[targets.model_a]id = "model/a"@@ -63,6 +64,8 @@ Each target references an entry under `llm_clients`. All configured clients usenever contains the secret itself. If omitted, the client sends no authentication.Target-level `extra_body` values are shallow-merged into the upstream request whenthe request does not already contain that key.+`max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx+responses.Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit themfor equal weighting. The optional `seed` reproduces the selection sequence for the same call order.diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex b1524582d..0f4557ea4 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -12,11 +12,14 @@ use libsy::algorithms::{LlmTaskClassifier, Noop, Passthrough, Random, TaskClassiuse libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};use serde::Deserialize;use serde_json::Value;-use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};+use switchyard_llm_client::{+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, DEFAULT_MAX_RETRIES,+};use crate::{ServerError, ServerResult, ServerState};const SUPPORTED_SCHEMA_VERSION: u32 = 1;+const MAX_CONFIGURED_RETRIES: u32 = 10;/// Loads a TOML deployment file and constructs the complete server state.pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {@@ -141,6 +144,8 @@ struct LlmClientConfig {api_key_env: Option<String>,#[serde(default)]extra_headers: BTreeMap<String, String>,+ #[serde(default = "default_max_retries")]+ max_retries: u32,}#[derive(Debug, Deserialize)]@@ -210,6 +215,11 @@ fn build_backend("llm client {client_name} base_url must not be empty")));}+ if config.max_retries > MAX_CONFIGURED_RETRIES {+ return Err(ServerError::new(format!(+ "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"+ )));+ }let api_key = config.api_key_env.as_deref()@@ -237,6 +247,7 @@ fn build_backend(api_key,extra_headers: config.extra_headers.clone(),extra_body: extra_body.clone(),+ max_retries: config.max_retries,};Ok(match config.format {ClientFormat::OpenAiChat => Backend::OpenAiChat(http),@@ -245,6 +256,10 @@ fn build_backend(})}+const fn default_max_retries() -> u32 {+ DEFAULT_MAX_RETRIES+}+fn build_algorithm(route_name: &str,config: &RouteConfig,@@ -527,6 +542,53 @@ target = "weak"Ok(())}+ #[test]+ fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> {+ let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| {+ ServerError::new(format!("failed to parse default config: {error}"))+ })?;+ let Some(primary) = default.llm_clients.get("primary") else {+ return Err(ServerError::new("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);++ let explicit = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = 0",+ 1,+ );+ let config: ServerConfig = toml::from_str(&explicit).map_err(|error| {+ ServerError::new(format!("failed to parse explicit retry config: {error}"))+ })?;+ let Some(primary) = config.llm_clients.get("primary") else {+ return Err(ServerError::new("primary llm client is missing"));+ };+ assert_eq!(primary.max_retries, 0);+ Ok(())+ }++ #[test]+ fn retry_budget_rejects_negative_values() {+ let invalid = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = -1",+ 1,+ );+ assert!(error_message(&invalid).contains("max_retries"));+ }++ #[test]+ fn retry_budget_rejects_excessive_values() {+ let invalid = VALID_CONFIG.replacen(+ "base_url = \"https://example.test/v1\"",+ "base_url = \"https://example.test/v1\"\nmax_retries = 11",+ 1,+ );+ assert!(+ error_message(&invalid).contains("llm client primary max_retries must be at most 10")+ );+ }+#[test]fn api_key_environment_reference_is_validated() {let missing = VALID_CONFIG.replacen(diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 72fc36088..b173c46db 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -153,6 +153,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Serverapi_key: Some("test-key".to_string()),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 0,});let target_models = routes.iter()@@ -579,6 +580,7 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<Sapi_key: Some("k".to_string()),extra_headers: BTreeMap::new(),extra_body: BTreeMap::new(),+ max_retries: 0,};let strong: Arc<dyn RoutedLlmClient> =Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(