Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(llm-client): add bounded upstream retries (#173)

nachiketb-nvidiaef646cd7632a
diff --git a/Cargo.lock b/Cargo.lock
index 74b718f5b..c15c0af6b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1750,12 +1750,14 @@ dependencies = [
"async-trait",
"futures",
"futures-util",
+ "httpdate",
"opentelemetry",
"reqwest",
"serde_json",
"switchyard-protocol",
"switchyard-translation",
"tokio",
+ "tracing",
"wiremock",
]
diff --git a/Cargo.toml b/Cargo.toml
index 0bca0c95b..7fd0c69cf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,6 +26,7 @@ async-stream = "0.3"
async-trait = "0.1"
futures = "0.3"
futures-util = "0.3"
+httpdate = "1"
parking_lot = "0.12"
rand = "0.8"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots", "stream"] }
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml
index 19b2470fe..54c6be9da 100644
--- a/crates/libsy-llm-client/Cargo.toml
+++ b/crates/libsy-llm-client/Cargo.toml
@@ -18,9 +18,11 @@ reqwest.workspace = true
async-trait.workspace = true
futures-util.workspace = true
opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
+httpdate.workspace = true
serde_json.workspace = true
+tokio.workspace = true
+tracing.workspace = true
[dev-dependencies]
futures.workspace = true
-tokio.workspace = true
wiremock = "0.6"
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index 9e1a4ef81..a1d5bbbc4 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -22,8 +22,9 @@ provider SDK.
when set, otherwise the model's default backend.
- **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or
`Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`,
- static `extra_headers`, default `extra_body` fields). The variant fixes the URL
- path and auth scheme (Bearer vs `x-api-key` + `anthropic-version`).
+ static `extra_headers`, default `extra_body` fields, and `max_retries`). The
+ variant fixes the URL path and auth scheme (Bearer vs `x-api-key` +
+ `anthropic-version`).
- **Model rewrite.** The resolved model name is both the map key and the model id
sent upstream — it overwrites whatever `model` the request arrived with.
- **Streaming is chosen by the request.** If the encoded body has `stream: true`
@@ -58,6 +59,7 @@ fn build_client() -> switchyard_llm_client::Result<TranslatingLlmClient> {
api_key: std::env::var("OPENAI_API_KEY").ok(),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 2,
};
let models = [ModelConfig::new(
@@ -173,6 +175,14 @@ fn build_multi_format_client(
- Per-backend static headers go in `HttpBackendConfig::extra_headers`.
- Per-target top-level request defaults go in `HttpBackendConfig::extra_body`.
The merge is shallow and fields already present in the request take precedence.
+- `HttpBackendConfig::max_retries` controls additional attempts after retryable
+ transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body
+ transport failures are retried; streaming body failures are not replayed after
+ the response has been returned.
+
+Retries replay the same upstream request. A transport failure can therefore
+duplicate a request that the provider processed but did not finish returning,
+and the retry budget plus capped `Retry-After` delays determines total latency.
## Errors
diff --git a/crates/libsy-llm-client/src/backend.rs b/crates/libsy-llm-client/src/backend.rs
index 2c0555aac..f4c4f4fe0 100644
--- a/crates/libsy-llm-client/src/backend.rs
+++ b/crates/libsy-llm-client/src/backend.rs
@@ -13,6 +13,9 @@ use crate::error::is_overflow_body;
const ANTHROPIC_VERSION: &str = "2023-06-01";
+/// Default number of retries for server-configured upstream calls.
+pub const DEFAULT_MAX_RETRIES: u32 = 2;
+
// Canonical OpenAI phrase plus NVIDIA/LiteLLM wrap variants. Adding a new
// provider-wrap is a one-line entry here, not a fork of the parsing logic.
const OPENAI_OVERFLOW_PHRASES: &[&str] = &[
@@ -42,6 +45,8 @@ pub struct HttpBackendConfig {
pub extra_headers: BTreeMap<String, String>,
/// Default top-level request fields, applied only when the request omits the key.
pub extra_body: BTreeMap<String, Value>,
+ /// Additional attempts after the initial upstream request.
+ pub max_retries: u32,
}
impl fmt::Debug for HttpBackendConfig {
@@ -51,6 +56,7 @@ impl fmt::Debug for HttpBackendConfig {
.field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
.field("extra_headers", &self.extra_headers)
.field("extra_body_keys", &self.extra_body.keys())
+ .field("max_retries", &self.max_retries)
.finish()
}
}
@@ -133,6 +139,11 @@ impl Backend {
&self.config().extra_body
}
+ /// Additional attempts allowed after the initial request.
+ pub fn max_retries(&self) -> u32 {
+ self.config().max_retries
+ }
+
/// Whether this backend speaks the Anthropic Messages wire format — the only
/// one with a `count_tokens` endpoint.
pub fn is_anthropic(&self) -> bool {
@@ -196,6 +207,7 @@ mod tests {
api_key: Some("secret".to_string()),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 0,
}
}
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index eeba631fb..565d3464a 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -5,9 +5,11 @@
//! request, call the configured backend over HTTP, decode the neutral response.
use std::collections::{BTreeMap, HashMap};
+use std::time::{Duration, SystemTime};
use async_trait::async_trait;
use futures_util::StreamExt;
+use reqwest::header::{HeaderMap, RETRY_AFTER};
use reqwest::RequestBuilder;
use serde_json::{Map, Value};
use switchyard_protocol::{
@@ -17,10 +19,11 @@ use switchyard_translation::{
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
encode_request, encode_stream, WireFormat,
};
+use tracing::Instrument;
use crate::backend::Backend;
use crate::error::{LlmClientError, Result};
-use crate::metrics::record_upstream_attempt;
+use crate::metrics::{is_retryable_http_status, record_upstream_attempt};
use crate::raw::RawResponse;
// TODO: Why is this here? What does it do?
@@ -43,6 +46,10 @@ const RESERVED_HEADERS: &[&str] = &[
"content-type",
];
+const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);
+const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2);
+const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
+
/// How one model is served: the `default_backend` used when the request does not
/// pin a wire format, plus any `other_backends` reachable over additional formats.
#[derive(Clone, Debug)]
@@ -137,23 +144,24 @@ impl TranslatingLlmClient {
/// Encode `llm_request` (its model restamped to `model`) for `wire_format`,
/// POST it to `url` with the request's forwarded headers plus the backend's
- /// static headers and auth, and return the successful upstream response and
- /// whether the request asked for a streamed body. A non-success status maps
- /// to a typed error — a 400 is classified as a context-window overflow via
- /// the backend's provider rules. Shared by
+ /// static headers and auth, and return the successful upstream response. A
+ /// buffered response is fully collected within the retry boundary; a streamed
+ /// response is returned as soon as its successful headers arrive. A non-success
+ /// status maps to a typed error — a 400 is classified as a context-window
+ /// overflow via the backend's provider rules. Shared by
/// [`call_rewrite_model`](Self::call_rewrite_model) (which POSTs to the
/// backend's completion URL and decodes a response) and
/// [`count_tokens`](RoutedLlmClient::count_tokens) (which POSTs to the
/// `count_tokens` URL and returns the raw JSON).
async fn send_encoded(
&self,
- url: String,
backend: &Backend,
wire_format: WireFormat,
mut llm_request: LlmRequest,
metadata: Option<&Metadata>,
model: &str,
- ) -> Result<(reqwest::Response, bool)> {
+ endpoint: UpstreamEndpoint,
+ ) -> Result<EncodedResponse> {
// The resolved name is the upstream model id (per the crate contract).
llm_request.model = Some(model.to_string());
let mut body = encode_request(&llm_request, wire_format)
@@ -166,46 +174,142 @@ impl TranslatingLlmClient {
if matches!(backend, Backend::OpenAiChat(_)) {
ensure_openai_stream_usage(&mut body);
}
- let streaming = body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+ let streaming = endpoint.allows_streaming()
+ && body.get("stream").and_then(Value::as_bool).unwrap_or(false);
+ let url = endpoint.url(backend);
+
+ let max_retries = u64::from(backend.max_retries());
+ let max_attempts = max_retries + 1;
+ let mut attempt = 0_u64;
+ loop {
+ let span = tracing::debug_span!(
+ target: "libsy",
+ "libsy.upstream_attempt",
+ model,
+ wire_format = %wire_format,
+ attempt = attempt + 1,
+ max_attempts,
+ retry = attempt > 0,
+ outcome = tracing::field::Empty,
+ status_code = tracing::field::Empty,
+ will_retry = tracing::field::Empty,
+ retry_delay_ms = tracing::field::Empty,
+ );
+ let result = self
+ .send_once(&url, backend, &body, metadata, model, streaming)
+ .instrument(span.clone())
+ .await;
+ // The retained handle updates this same attempt span with its outcome.
+ match result {
+ Ok(response) => {
+ span.record("outcome", "success");
+ span.record("status_code", response.status());
+ span.record("will_retry", false);
+ return Ok(response);
+ }
+ Err(failure) => {
+ let will_retry = attempt < max_retries && failure.is_retryable();
+ span.record("outcome", "error");
+ if let Some(status) = failure.status {
+ span.record("status_code", status);
+ }
+ span.record("will_retry", will_retry);
+ if !will_retry {
+ return Err(failure.error);
+ }
- let builder = self.client.post(url).json(&body);
+ let delay = retry_delay(attempt, failure.retry_after);
+ span.record("retry_delay_ms", duration_millis(delay));
+ // Close the attempt span before sleeping so backoff is not attempt latency.
+ drop(span);
+ tokio::time::sleep(delay).await;
+ attempt += 1;
+ }
+ }
+ }
+ }
+
+ // Performs one HTTP attempt and retains the retry metadata alongside any error.
+ async fn send_once(
+ &self,
+ url: &str,
+ backend: &Backend,
+ body: &Value,
+ metadata: Option<&Metadata>,
+ model: &str,
+ streaming: bool,
+ ) -> std::result::Result<EncodedResponse, AttemptFailure> {
+ let builder = self.client.post(url).json(body);
let builder = forward_metadata_headers(builder, metadata);
let builder = apply_extra_headers(builder, backend);
let builder = backend.apply_auth(builder);
- let http_response = match builder.send().await {
- Ok(response) => {
- record_upstream_attempt(Some(response.status().as_u16()));
- response
- }
+ let response = match builder.send().await {
+ Ok(response) => response,
Err(error) => {
record_upstream_attempt(None);
- return Err(convert_reqwest_error(error));
+ return Err(AttemptFailure {
+ error: convert_reqwest_error(error),
+ status: None,
+ retry_after: None,
+ });
}
};
- let status = http_response.status();
- if !status.is_success() {
- let body = match http_response.text().await {
+ let status = response.status();
+ if status.is_success() {
+ if streaming {
+ // Streaming body failures happen after the retry boundary.
+ record_upstream_attempt(Some(status.as_u16()));
+ return Ok(EncodedResponse::Streaming(response));
+ }
+ let body = match response.bytes().await {
Ok(body) => body,
- Err(error) if error.is_timeout() => {
- return Err(LlmClientError::Timeout {
- source: Box::new(error),
- })
+ Err(error) => {
+ record_upstream_attempt(None);
+ return Err(AttemptFailure {
+ error: convert_reqwest_error(error),
+ status: Some(status.as_u16()),
+ retry_after: None,
+ });
}
- Err(error) => format!("<failed to read error body: {error}>"),
};
- if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
- return Err(LlmClientError::ContextWindowExceeded {
- model: model.to_string(),
- message: body,
- });
- }
- return Err(LlmClientError::UpstreamHttp {
+ record_upstream_attempt(Some(status.as_u16()));
+ return Ok(EncodedResponse::Buffered {
status: status.as_u16(),
- body,
+ body: body.to_vec(),
});
}
- Ok((http_response, streaming))
+
+ let retry_after = retry_after_delay(response.headers());
+ let body = match response.text().await {
+ Ok(body) => body,
+ Err(error) => {
+ record_upstream_attempt(None);
+ return Err(AttemptFailure {
+ error: convert_reqwest_error(error),
+ status: Some(status.as_u16()),
+ retry_after,
+ });
+ }
+ };
+ record_upstream_attempt(Some(status.as_u16()));
+ let error =
+ if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
+ LlmClientError::ContextWindowExceeded {
+ model: model.to_string(),
+ message: body,
+ }
+ } else {
+ LlmClientError::UpstreamHttp {
+ status: status.as_u16(),
+ body,
+ }
+ };
+ Err(AttemptFailure {
+ error,
+ status: Some(status.as_u16()),
+ retry_after,
+ })
}
/// Calls the backend for `model_name` (or the request's own model), over the
@@ -252,43 +356,45 @@ impl TranslatingLlmClient {
message: format!("model {model:?} has no backend for format {wire_format}"),
})?;
- let (http_response, streaming) = self
+ let http_response = self
.send_encoded(
- backend.url(),
backend,
wire_format,
llm_request,
metadata.as_ref(),
&model,
+ UpstreamEndpoint::Completion,
)
.await?;
- let llm_response = if streaming {
- // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
- // transport-agnostic and lives in `switchyard-translation`.
- let bytes = http_response.bytes_stream().map(|chunk| {
- chunk.map(|bytes| bytes.to_vec()).map_err(|error| {
- if error.is_timeout() {
- LlmClientError::Timeout {
- source: Box::new(error),
- }
- } else {
- LlmClientError::Transport {
- source: Box::new(error),
+ let llm_response = match http_response {
+ EncodedResponse::Streaming(http_response) => {
+ // Adapt the reqwest body stream to plain bytes; the SSE-decode itself is
+ // transport-agnostic and lives in `switchyard-translation`.
+ let bytes = http_response.bytes_stream().map(|chunk| {
+ chunk.map(|bytes| bytes.to_vec()).map_err(|error| {
+ if error.is_timeout() {
+ LlmClientError::Timeout {
+ source: Box::new(error),
+ }
+ } else {
+ LlmClientError::Transport {
+ source: Box::new(error),
+ }
}
- }
- })
- });
- let chunks = decode_stream(bytes, wire_format)?;
- LlmResponse::Stream(chunks)
- } else {
- let body = http_response
- .json::<Value>()
- .await
- .map_err(convert_reqwest_error)?;
- let agg = decode_aggregated_response(&body, wire_format)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
- LlmResponse::Agg(agg)
+ })
+ });
+ let chunks = decode_stream(bytes, wire_format)?;
+ LlmResponse::Stream(chunks)
+ }
+ EncodedResponse::Buffered { body, .. } => {
+ let body = serde_json::from_slice::<Value>(&body).map_err(|error| {
+ LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
+ })?;
+ let agg = decode_aggregated_response(&body, wire_format)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ LlmResponse::Agg(agg)
+ }
};
Ok(Response {
@@ -387,38 +493,120 @@ impl RoutedLlmClient for TranslatingLlmClient {
metadata,
..
} = request;
- let (http_response, _) = self
+ let http_response = self
.send_encoded(
- backend.count_tokens_url(),
backend,
WireFormat::AnthropicMessages,
llm_request,
metadata.as_ref(),
model,
+ UpstreamEndpoint::CountTokens,
)
.await?;
- let text = http_response.text().await.map_err(convert_reqwest_error)?;
- serde_json::from_str(&text).map_err(|error| LlmClientError::InvalidResponse {
+ let body = match http_response {
+ EncodedResponse::Buffered { body, .. } => body,
+ EncodedResponse::Streaming(_) => {
+ return Err(LlmClientError::InvalidRequest {
+ message: "count_tokens does not support streaming requests".to_string(),
+ })
+ }
+ };
+ serde_json::from_slice(&body).map_err(|error| LlmClientError::InvalidResponse {
source: Box::new(error),
})
}
}
+#[derive(Clone, Copy)]
+enum UpstreamEndpoint {
+ Completion,
+ CountTokens,
+}
+
+impl UpstreamEndpoint {
+ fn url(self, backend: &Backend) -> String {
+ match self {
+ UpstreamEndpoint::Completion => backend.url(),
+ UpstreamEndpoint::CountTokens => backend.count_tokens_url(),
+ }
+ }
+
+ fn allows_streaming(self) -> bool {
+ matches!(self, UpstreamEndpoint::Completion)
+ }
+}
+
+enum EncodedResponse {
+ Buffered { status: u16, body: Vec<u8> },
+ Streaming(reqwest::Response),
+}
+
+impl EncodedResponse {
+ fn status(&self) -> u16 {
+ match self {
+ EncodedResponse::Buffered { status, .. } => *status,
+ EncodedResponse::Streaming(response) => response.status().as_u16(),
+ }
+ }
+}
+
+// The typed error decides retry eligibility; status and Retry-After feed
+// attempt telemetry and delay selection.
+struct AttemptFailure {
+ error: LlmClientError,
+ status: Option<u16>,
+ retry_after: Option<Duration>,
+}
+
+impl AttemptFailure {
+ fn is_retryable(&self) -> bool {
+ match &self.error {
+ LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
+ LlmClientError::UpstreamHttp { status, .. } => is_retryable_http_status(*status),
+ _ => false,
+ }
+ }
+}
+
+// Uses Retry-After when supplied, capped so an upstream cannot stall a request indefinitely.
+fn retry_after_delay(headers: &HeaderMap) -> Option<Duration> {
+ let value = headers.get(RETRY_AFTER)?.to_str().ok()?;
+ let delay = if let Ok(seconds) = value.parse::<u64>() {
+ Duration::from_secs(seconds)
+ } else {
+ let retry_at = httpdate::parse_http_date(value).ok()?;
+ retry_at
+ .duration_since(SystemTime::now())
+ .unwrap_or(Duration::ZERO)
+ };
+ Some(delay.min(MAX_RETRY_AFTER))
+}
+
+fn retry_delay(retry_number: u64, retry_after: Option<Duration>) -> Duration {
+ // Retry-After wins; otherwise double 250 ms up to the two-second cap.
+ retry_after.unwrap_or_else(|| {
+ let multiplier = 1_u32 << retry_number.min(3);
+ INITIAL_RETRY_DELAY
+ .saturating_mul(multiplier)
+ .min(MAX_RETRY_BACKOFF)
+ })
+}
+
+fn duration_millis(duration: Duration) -> u64 {
+ u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
+}
+
fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
- // `Response::json` labels body collection and JSON parsing failures as
- // decode errors; only the latter is a translation failure.
+ // Reqwest labels truncated or otherwise unreadable response bodies as decode
+ // errors, so distinguish them from serde JSON failures at the call site.
if error.is_timeout() {
LlmClientError::Timeout {
source: Box::new(error),
}
- } else if error.is_body() {
- LlmClientError::Transport {
- source: Box::new(error),
+ } else if error.is_builder() {
+ LlmClientError::Configuration {
+ message: format!("failed to build upstream request: {error}"),
}
- } else if error.is_decode()
- && std::error::Error::source(&error).is_some_and(|source| source.is::<serde_json::Error>())
- {
- LlmClientError::ResponseTranslation(format!("invalid upstream JSON: {error}"))
} else {
LlmClientError::Transport {
source: Box::new(error),
@@ -503,6 +691,8 @@ mod tests {
use std::collections::BTreeMap;
use std::error::Error;
use std::io::{Read, Write};
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::Arc;
use std::thread::JoinHandle;
use serde_json::json;
@@ -519,6 +709,14 @@ mod tests {
api_key: Some("secret".to_string()),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 0,
+ }
+ }
+
+ fn config_with_retries(base_url: &str, max_retries: u32) -> HttpBackendConfig {
+ HttpBackendConfig {
+ max_retries,
+ ..config(base_url)
}
}
@@ -540,31 +738,69 @@ mod tests {
vec![ModelConfig::new("gpt", Backend::OpenAiChat(backend), None)]
}
+ fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec<ModelConfig> {
+ vec![ModelConfig::new(
+ "gpt",
+ Backend::OpenAiChat(config_with_retries(base_url, max_retries)),
+ None,
+ )]
+ }
+
+ fn chat_success_response() -> ResponseTemplate {
+ ResponseTemplate::new(200).set_body_json(json!({
+ "id": "chatcmpl-1",
+ "model": "gpt",
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "recovered"},
+ "finish_reason": "stop"
+ }],
+ "usage": {}
+ }))
+ }
+
fn truncated_response_server(
content_type: &str,
body: &str,
) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
- let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
- let address = listener.local_addr()?;
- let response = format!(
+ response_sequence_server(vec![format!(
"HTTP/1.1 200 OK\r\nContent-Type: {content_type}\r\n\
Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len() + 100
- );
+ )])
+ }
+
+ fn response_sequence_server(
+ responses: Vec<String>,
+ ) -> std::io::Result<(String, JoinHandle<std::io::Result<()>>)> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let address = listener.local_addr()?;
let handle = std::thread::spawn(move || {
- let (mut stream, _) = listener.accept()?;
- let mut request = [0_u8; 1024];
- if stream.read(&mut request)? == 0 {
- return Err(std::io::Error::new(
- std::io::ErrorKind::UnexpectedEof,
- "client closed before sending a request",
- ));
+ for response in responses {
+ let (mut stream, _) = listener.accept()?;
+ let mut request = [0_u8; 1024];
+ if stream.read(&mut request)? == 0 {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::UnexpectedEof,
+ "client closed before sending a request",
+ ));
+ }
+ stream.write_all(response.as_bytes())?;
}
- stream.write_all(response.as_bytes())
+ Ok(())
});
Ok((format!("http://{address}/v1"), handle))
}
+ fn raw_chat_success_response() -> String {
+ let body = r#"{"id":"chatcmpl-1","model":"gpt","choices":[{"index":0,"message":{"role":"assistant","content":"recovered"},"finish_reason":"stop"}],"usage":{}}"#;
+ format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
+ Content-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ )
+ }
+
fn request_for(model: Option<&str>, stream: bool) -> Request {
let mut llm_request = text_request(model.map(str::to_string), "hi");
llm_request.stream = stream;
@@ -719,12 +955,18 @@ mod tests {
async fn invalid_json_is_a_response_translation_error(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let server = MockServer::start().await;
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed_calls = Arc::clone(&calls);
Mock::given(method("POST"))
- .respond_with(ResponseTemplate::new(200).set_body_raw("not json", "application/json"))
+ .respond_with(move |_: &wiremock::Request| {
+ observed_calls.fetch_add(1, Ordering::SeqCst);
+ ResponseTemplate::new(200).set_body_raw("not json", "application/json")
+ })
.mount(&server)
.await;
- let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?;
+ let client =
+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
let Err(error) = client
.call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
.await
@@ -737,6 +979,7 @@ mod tests {
LlmClientError::ResponseTranslation(message)
if message.contains("invalid upstream JSON")
));
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
Ok(())
}
@@ -767,12 +1010,41 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn response_body_transport_failures_are_retried(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let truncated_responses = [
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
+ Content-Length: 102\r\nConnection: close\r\n\r\n{}",
+ "HTTP/1.1 401 Unauthorized\r\nContent-Type: text/plain\r\n\
+ Content-Length: 100\r\nConnection: close\r\n\r\nbad",
+ ];
+
+ for truncated in truncated_responses {
+ let (base_url, server) =
+ response_sequence_server(vec![truncated.to_string(), raw_chat_success_response()])?;
+ let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 1))?;
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await?;
+ server
+ .join()
+ .map_err(|_| std::io::Error::other("response server thread panicked"))??;
+
+ assert_eq!(
+ completion_text(&response.llm_response.into_agg().await?),
+ "recovered"
+ );
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn streaming_body_io_failure_preserves_transport_error(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
let body = "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n";
let (base_url, server) = truncated_response_server("text/event-stream", body)?;
- let client = TranslatingLlmClient::new(&chat_map(&base_url))?;
+ let client = TranslatingLlmClient::new(&chat_map_with_retries(&base_url, 2))?;
let response = client
.call_rewrite_model(Context::default(), request_for(Some("gpt"), true), None)
.await?;
@@ -961,6 +1233,213 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn retryable_http_failure_recovers_within_budget(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed_calls = Arc::clone(&calls);
+ Mock::given(method("POST"))
+ .respond_with(move |_: &wiremock::Request| {
+ if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
+ ResponseTemplate::new(503)
+ .insert_header("retry-after", "0")
+ .set_body_string("temporarily unavailable")
+ } else {
+ chat_success_response()
+ }
+ })
+ .mount(&server)
+ .await;
+
+ let client =
+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await?;
+ let agg = response.llm_response.into_agg().await?;
+
+ assert_eq!(completion_text(&agg), "recovered");
+ assert_eq!(calls.load(Ordering::SeqCst), 2);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn deterministic_http_failure_is_not_retried(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed_calls = Arc::clone(&calls);
+ Mock::given(method("POST"))
+ .respond_with(move |_: &wiremock::Request| {
+ observed_calls.fetch_add(1, Ordering::SeqCst);
+ ResponseTemplate::new(401).set_body_string("invalid key")
+ })
+ .mount(&server)
+ .await;
+
+ let client =
+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected an upstream error");
+ };
+
+ assert!(matches!(
+ error,
+ LlmClientError::UpstreamHttp {
+ status: 401,
+ body
+ } if body == "invalid key"
+ ));
+ assert_eq!(calls.load(Ordering::SeqCst), 1);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn retry_exhaustion_returns_the_final_upstream_error(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed_calls = Arc::clone(&calls);
+ Mock::given(method("POST"))
+ .respond_with(move |_: &wiremock::Request| {
+ let attempt = observed_calls.fetch_add(1, Ordering::SeqCst) + 1;
+ ResponseTemplate::new(500)
+ .insert_header("retry-after", "0")
+ .set_body_string(format!("attempt {attempt}"))
+ })
+ .mount(&server)
+ .await;
+
+ let client =
+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 2))?;
+ let Err(error) = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await
+ else {
+ panic!("expected retry exhaustion");
+ };
+
+ assert!(matches!(
+ error,
+ LlmClientError::UpstreamHttp {
+ status: 500,
+ body
+ } if body == "attempt 3"
+ ));
+ assert_eq!(calls.load(Ordering::SeqCst), 3);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn timeout_is_retried_before_a_response_is_returned(
+ ) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
+ let server = MockServer::start().await;
+ let calls = Arc::new(AtomicUsize::new(0));
+ let observed_calls = Arc::clone(&calls);
+ Mock::given(method("POST"))
+ .respond_with(move |_: &wiremock::Request| {
+ if observed_calls.fetch_add(1, Ordering::SeqCst) == 0 {
+ ResponseTemplate::new(200).set_delay(Duration::from_millis(500))
+ } else {
+ chat_success_response()
+ }
+ })
+ .mount(&server)
+ .await;
+
+ let mut client =
+ TranslatingLlmClient::new(&chat_map_with_retries(&format!("{}/v1", server.uri()), 1))?;
+ client.client = reqwest::Client::builder()
+ .timeout(Duration::from_millis(100))
+ .build()?;
+ let response = client
+ .call_rewrite_model(Context::default(), request_for(Some("gpt"), false), None)
+ .await?;
+
+ assert_eq!(
+ completion_text(&response.llm_response.into_agg().await?),
+ "recovered"
+ );
+ assert_eq!(calls.load(Ordering::SeqCst), 2);
+ Ok(())
+ }
+
+ #[test]
+ fn retryable_error_classes_are_explicit() {
+ let transport = AttemptFailure {
+ error: LlmClientError::Transport {
+ source: std::io::Error::other("disconnected").into(),
+ },
+ status: None,
+ retry_after: None,
+ };
+ assert!(transport.is_retryable());
+
+ for status in [408, 429, 500, 503, 599] {
+ let failure = AttemptFailure {
+ error: LlmClientError::UpstreamHttp {
+ status,
+ body: String::new(),
+ },
+ status: Some(status),
+ retry_after: None,
+ };
+ assert!(failure.is_retryable(), "HTTP {status} should retry");
+ }
+ for status in [400, 401, 409, 600] {
+ let failure = AttemptFailure {
+ error: LlmClientError::UpstreamHttp {
+ status,
+ body: String::new(),
+ },
+ status: Some(status),
+ retry_after: None,
+ };
+ assert!(!failure.is_retryable(), "HTTP {status} should fail fast");
+ }
+
+ let configuration = AttemptFailure {
+ error: LlmClientError::Configuration {
+ message: "invalid header".to_string(),
+ },
+ status: None,
+ retry_after: None,
+ };
+ assert!(!configuration.is_retryable());
+
+ let context_window = AttemptFailure {
+ error: LlmClientError::ContextWindowExceeded {
+ model: "gpt".to_string(),
+ message: "too long".to_string(),
+ },
+ status: Some(400),
+ retry_after: None,
+ };
+ assert!(!context_window.is_retryable());
+ }
+
+ #[test]
+ fn retry_after_supports_seconds_and_http_dates() {
+ let mut headers = HeaderMap::new();
+ headers.insert(RETRY_AFTER, reqwest::header::HeaderValue::from_static("3"));
+ assert_eq!(retry_after_delay(&headers), Some(Duration::from_secs(3)));
+
+ let retry_at = SystemTime::now() + Duration::from_secs(2);
+ let value = httpdate::fmt_http_date(retry_at);
+ let Ok(value) = reqwest::header::HeaderValue::from_str(&value) else {
+ panic!("formatted HTTP date should be a valid header");
+ };
+ headers.insert(RETRY_AFTER, value);
+ let Some(delay) = retry_after_delay(&headers) else {
+ panic!("HTTP date should produce a retry delay");
+ };
+ assert!(delay <= Duration::from_secs(2));
+ }
+
#[tokio::test]
async fn routed_llm_client_exposes_timeout_variant(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index 79fb84e3a..8fa213828 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -17,7 +17,7 @@ pub mod error;
pub mod metrics;
pub mod raw;
-pub use backend::{Backend, HttpBackendConfig};
+pub use backend::{Backend, HttpBackendConfig, DEFAULT_MAX_RETRIES};
pub use client::{ModelConfig, TranslatingLlmClient};
pub use error::{LlmClientError, Result};
pub use raw::RawResponse;
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rs
index 67202463d..50985e75a 100644
--- a/crates/libsy-llm-client/src/metrics.rs
+++ b/crates/libsy-llm-client/src/metrics.rs
@@ -5,10 +5,15 @@
use opentelemetry::{global, KeyValue};
+pub(crate) const fn is_retryable_http_status(status: u16) -> bool {
+ status == 408 || status == 429 || (status >= 500 && status <= 599)
+}
+
pub const fn http_outcome_label(status: Option<u16>) -> &'static str {
match status {
Some(200..=299) => "success",
- Some(429 | 500 | 504) | None => "retryable_error",
+ Some(status) if is_retryable_http_status(status) => "retryable_error",
+ None => "retryable_error",
Some(_) => "other_error",
}
}
@@ -53,3 +58,21 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {
],
);
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn outcome_labels_match_the_retry_policy() {
+ for status in [408, 429, 500, 502, 599] {
+ assert!(is_retryable_http_status(status));
+ assert_eq!(http_outcome_label(Some(status)), "retryable_error");
+ }
+ for status in [400, 409, 499, 600] {
+ assert!(!is_retryable_http_status(status));
+ assert_eq!(http_outcome_label(Some(status)), "other_error");
+ }
+ assert_eq!(http_outcome_label(None), "retryable_error");
+ }
+}
diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rs
index 14376fde4..d069e188b 100644
--- a/crates/libsy/examples/ensemble.rs
+++ b/crates/libsy/examples/ensemble.rs
@@ -412,6 +412,7 @@ async fn main() -> Result<()> {
),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 2,
};
let models: Vec<_> = CANDIDATE_MODELS
.iter()
diff --git a/crates/switchyard-server/CONFIGURATION.md b/crates/switchyard-server/CONFIGURATION.md
index ff0ecbba8..44ed47d95 100644
--- a/crates/switchyard-server/CONFIGURATION.md
+++ b/crates/switchyard-server/CONFIGURATION.md
@@ -9,6 +9,7 @@ Define the upstream once under `llm_clients`, then reference it from targets.
format = "openai_chat"
base_url = "https://example.com/v1"
api_key_env = "PROVIDER_API_KEY"
+max_retries = 2
[targets.model]
id = "provider/model"
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index a7060d9cd..34b93fac9 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -12,6 +12,7 @@ schema_version = 1
format = "openai_chat"
base_url = "https://example.com/v1"
api_key_env = "API_KEY"
+max_retries = 2
[targets.model_a]
id = "model/a"
@@ -63,6 +64,8 @@ Each target references an entry under `llm_clients`. All configured clients use
never contains the secret itself. If omitted, the client sends no authentication.
Target-level `extra_body` values are shallow-merged into the upstream request when
the request does not already contain that key.
+`max_retries` defaults to `2` and applies to transport failures, timeouts, HTTP 408/429, and 5xx
+responses.
Random-route `weights` are relative, follow target order, and do not need to sum to one. Omit them
for equal weighting. The optional `seed` reproduces the selection sequence for the same call order.
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index b1524582d..0f4557ea4 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -12,11 +12,14 @@ use libsy::algorithms::{LlmTaskClassifier, Noop, Passthrough, Random, TaskClassi
use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient};
use serde::Deserialize;
use serde_json::Value;
-use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
+use switchyard_llm_client::{
+ Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient, DEFAULT_MAX_RETRIES,
+};
use crate::{ServerError, ServerResult, ServerState};
const SUPPORTED_SCHEMA_VERSION: u32 = 1;
+const MAX_CONFIGURED_RETRIES: u32 = 10;
/// Loads a TOML deployment file and constructs the complete server state.
pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {
@@ -141,6 +144,8 @@ struct LlmClientConfig {
api_key_env: Option<String>,
#[serde(default)]
extra_headers: BTreeMap<String, String>,
+ #[serde(default = "default_max_retries")]
+ max_retries: u32,
}
#[derive(Debug, Deserialize)]
@@ -210,6 +215,11 @@ fn build_backend(
"llm client {client_name} base_url must not be empty"
)));
}
+ if config.max_retries > MAX_CONFIGURED_RETRIES {
+ return Err(ServerError::new(format!(
+ "llm client {client_name} max_retries must be at most {MAX_CONFIGURED_RETRIES}"
+ )));
+ }
let api_key = config
.api_key_env
.as_deref()
@@ -237,6 +247,7 @@ fn build_backend(
api_key,
extra_headers: config.extra_headers.clone(),
extra_body: extra_body.clone(),
+ max_retries: config.max_retries,
};
Ok(match config.format {
ClientFormat::OpenAiChat => Backend::OpenAiChat(http),
@@ -245,6 +256,10 @@ fn build_backend(
})
}
+const fn default_max_retries() -> u32 {
+ DEFAULT_MAX_RETRIES
+}
+
fn build_algorithm(
route_name: &str,
config: &RouteConfig,
@@ -527,6 +542,53 @@ target = "weak"
Ok(())
}
+ #[test]
+ fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> {
+ let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| {
+ ServerError::new(format!("failed to parse default config: {error}"))
+ })?;
+ let Some(primary) = default.llm_clients.get("primary") else {
+ return Err(ServerError::new("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, DEFAULT_MAX_RETRIES);
+
+ let explicit = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = 0",
+ 1,
+ );
+ let config: ServerConfig = toml::from_str(&explicit).map_err(|error| {
+ ServerError::new(format!("failed to parse explicit retry config: {error}"))
+ })?;
+ let Some(primary) = config.llm_clients.get("primary") else {
+ return Err(ServerError::new("primary llm client is missing"));
+ };
+ assert_eq!(primary.max_retries, 0);
+ Ok(())
+ }
+
+ #[test]
+ fn retry_budget_rejects_negative_values() {
+ let invalid = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = -1",
+ 1,
+ );
+ assert!(error_message(&invalid).contains("max_retries"));
+ }
+
+ #[test]
+ fn retry_budget_rejects_excessive_values() {
+ let invalid = VALID_CONFIG.replacen(
+ "base_url = \"https://example.test/v1\"",
+ "base_url = \"https://example.test/v1\"\nmax_retries = 11",
+ 1,
+ );
+ assert!(
+ error_message(&invalid).contains("llm client primary max_retries must be at most 10")
+ );
+ }
+
#[test]
fn api_key_environment_reference_is_validated() {
let missing = VALID_CONFIG.replacen(
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 72fc36088..b173c46db 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -153,6 +153,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
api_key: Some("test-key".to_string()),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 0,
});
let target_models = routes
.iter()
@@ -579,6 +580,7 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<S
api_key: Some("k".to_string()),
extra_headers: BTreeMap::new(),
extra_body: BTreeMap::new(),
+ max_retries: 0,
};
let strong: Arc<dyn RoutedLlmClient> =
Arc::new(TranslatingLlmClient::new(&[ModelConfig::new(