Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(llm-client): allow disabling fallback on rate limits (#1004)
Elias Shieh9ee35cb3ec72
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex e1976de29..5dd586bd7 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -128,6 +128,7 @@ pub struct TranslatingLlmClient {model_to_config: HashMap<ModelId, ModelConfig>,client: reqwest::Client,forward_auth_client: reqwest::Client,+ is_rate_limit_cooldown_enabled: bool,cooldown_epoch: Instant,// Milliseconds since cooldown_epoch. Each atomic is independent, so relaxed ordering suffices.unavailable_until: HashMap<ModelId, AtomicU64>,@@ -166,11 +167,18 @@ impl TranslatingLlmClient {model_to_config,client,forward_auth_client,+ is_rate_limit_cooldown_enabled: true,cooldown_epoch: Instant::now(),unavailable_until,})}+ /// Disable HTTP 429 cooldown so subsequent calls can receive the upstream error.+ pub fn without_rate_limit_cooldown(mut self) -> Self {+ self.is_rate_limit_cooldown_enabled = false;+ self+ }+/// The backend serving `model` over `format` — the default backend when its/// format matches, otherwise a matching entry in `other_backends`; `None` when/// the model is unknown or has no backend for `format`.@@ -302,11 +310,11 @@ impl TranslatingLlmClient {let result = self.send_with_retries(&url, backend, &body, metadata, model, streaming).await;- // Forwarded credentials can hit a user's quota while the backend remains healthy.+ // Keep quota failures request-local when forwarding auth or returning 429 to callers.if let Some(until) = unavailable_until&& let Err(error) = &result&& is_transient_failure(error)- && !(backend.is_forwarding_auth()+ && !((backend.is_forwarding_auth() || !self.is_rate_limit_cooldown_enabled)&& matches!(error, LlmClientError::UpstreamHttp { status, .. } if *status == StatusCode::TOO_MANY_REQUESTS)){let deadline = duration_millis(self.cooldown_epoch.elapsed().saturating_add(cooldown));diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex ab8a3e9cb..a80bde58c 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -18,7 +18,7 @@//! [`ClientRouter`] sends native Responses state back to its provider; for Chat or Anthropic//! targets it retains the canonical history needed to materialize the continuation.-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use std::sync::Arc;use std::time::Instant;@@ -261,6 +261,14 @@ async fn call_first_available({Ok(response) => return Ok(response),Err(error) if index + 1 == models.len() => return Err(error),+ Err(error)+ if clients.rate_limit_fallback_disabled.contains(target)+ && matches!(&error, LibsyError::ClientCall {+ source: LlmClientError::UpstreamHttp { status, .. }, ..+ } if *status == StatusCode::TOO_MANY_REQUESTS) =>+ {+ return Err(error);+ }Err(error) => match fallback_reason(&error) {Some(reason) => tracing::info!(from = %target,@@ -582,6 +590,7 @@ fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {#[derive(Clone)]pub struct ClientRouter {inner: Arc<ClientRouting>,+ rate_limit_fallback_disabled: Arc<HashSet<ModelId>>,}struct ClientRouting {@@ -668,6 +677,7 @@ impl ClientRouter {.next().is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));Self {+ rate_limit_fallback_disabled: Arc::default(),inner: Arc::new(ClientRouting {routing: Routing::ByModel(by_model),decision_clients,@@ -694,6 +704,7 @@ impl ClientRouter {decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,) -> Self {Self {+ rate_limit_fallback_disabled: Arc::default(),inner: Arc::new(ClientRouting {routing: Routing::Single(client),decision_clients,@@ -705,6 +716,16 @@ impl ClientRouter {}}+ /// Stop completion fallback on HTTP 429 for the given models.+ ///+ /// For translating clients, also use+ /// [`TranslatingLlmClient::without_rate_limit_cooldown`](crate::TranslatingLlmClient::without_rate_limit_cooldown)+ /// so later calls are not skipped with a generic cooldown error.+ pub fn without_rate_limit_fallback(mut self, models: HashSet<ModelId>) -> Self {+ self.rate_limit_fallback_disabled = Arc::new(models);+ self+ }+/// The client that serves `model`.////// Errors with [`LlmClientError::Configuration`] when the router maps models and has nodiff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex fb833f2f5..70f472ec9 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -446,11 +446,12 @@ impl DeploymentConfig {let mut clients = BTreeMap::new();for (name, model_configs) in models_by_client {- let client = Arc::new(- TranslatingLlmClient::new(&model_configs)- .map_err(|error| RunnerError::configuration(error.to_string()))?,- );- clients.insert(name, client);+ let mut client = TranslatingLlmClient::new(&model_configs)+ .map_err(|error| RunnerError::configuration(error.to_string()))?;+ if !self.llm_clients[&name].fallback_on_rate_limit {+ client = client.without_rate_limit_cooldown();+ }+ clients.insert(name, Arc::new(client));}Ok(clients)}@@ -588,6 +589,17 @@ impl DeploymentConfig {prompts,routing_answer_target,&completion_targets,+ )+ .without_rate_limit_fallback(+ route+ .routing_target_names()+ .into_iter()+ .filter_map(|name| {+ let target = &self.targets[name];+ (!self.llm_clients[&target.llm_client].fallback_on_rate_limit)+ .then(|| target.id.clone())+ })+ .collect(),);Ok((router, caller_auth))}@@ -758,6 +770,8 @@ struct LlmClientConfig {extra_headers: BTreeMap<String, String>,#[serde(default = "default_max_retries")]max_retries: u32,+ #[serde(default = "default_fallback_on_rate_limit")]+ fallback_on_rate_limit: bool,/// Cooldown after an exhausted transient completion failure. Defaults to 5 seconds; zero disables it.#[serde(default = "default_failure_cooldown_ms")]failure_cooldown_ms: u64,@@ -908,6 +922,10 @@ const fn default_max_retries() -> u32 {DEFAULT_MAX_RETRIES}+const fn default_fallback_on_rate_limit() -> bool {+ true+}+const fn default_failure_cooldown_ms() -> u64 {5000}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex d075fb2b4..ddef98875 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -140,6 +140,17 @@ async fn upstream_chat(}let model = body["model"].as_str().unwrap_or("unknown").to_string();+ if model == "model/rate-limited" {+ return (+ if prompt == "unavailable" {+ StatusCode::SERVICE_UNAVAILABLE+ } else {+ StatusCode::TOO_MANY_REQUESTS+ },+ Json(json!({"error": {"message": "quota exhausted", "code": "usage_limit"}})),+ )+ .into_response();+ }if model == "model/classifier" {let mut pending = HashSet::new();for message in body["messages"].as_array().into_iter().flatten() {@@ -5450,3 +5461,78 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult {);Ok(())}++#[tokio::test]+async fn rate_limit_fallback_policy() -> TestResult {+ for (policy, forward_auth, prompt, is_terminal) in [+ ("", false, "hello", false),+ ("", true, "hello", false),+ ("fallback_on_rate_limit = true", false, "hello", false),+ ("fallback_on_rate_limit = false", false, "hello", true),+ ("fallback_on_rate_limit = false", true, "hello", true),+ (+ "fallback_on_rate_limit = false",+ false,+ "unavailable",+ false,+ ),+ ] {+ let upstream = MockUpstream::start().await?;+ let app = build_switchyard_router(load_test_config(&format!(+ r#"+schema_version = 1+[llm_clients.primary]+format = "openai_chat"+base_url = "{base_url}"+forward_auth = {forward_auth}+{policy}+max_retries = 1+failure_cooldown_ms = 60000+[llm_clients.secondary]+format = "openai_chat"+base_url = "{base_url}"+[targets]+first = {{ id = "model/rate-limited", llm_client = "primary" }}+second = {{ id = "model/fallback", llm_client = "secondary" }}+[routes.test]+id = "{ROUTE_MODEL}"+type = "random"+targets = ["first", "second"]+weights = [1000, 1]+seed = 17+"#,+ base_url = upstream.base_url,+ ))?);+ let mut expected_calls = Vec::new();+ // The second request exercises cooldown as well as streaming HTTP errors.+ for stream in [false, true] {+ let response = send_with_headers(+ &app,+ "POST",+ "/v1/chat/completions",+ Some(json!({+ "model": ROUTE_MODEL, "stream": stream,+ "messages": [{"role": "user", "content": prompt}]+ })),+ &[("authorization", "Bearer test-caller")],+ )+ .await?;+ if !stream || is_terminal || forward_auth {+ expected_calls.extend(["model/rate-limited"; 2]);+ }+ if is_terminal {+ assert_eq!(response.status, StatusCode::TOO_MANY_REQUESTS);+ assert_eq!(response.json()?["error"]["code"], "usage_limit");+ } else {+ assert_eq!(response.status, StatusCode::OK);+ expected_calls.push("model/fallback");+ }+ assert_eq!(+ upstream.models().await,+ expected_calls,+ "{policy}, forward_auth={forward_auth}, {prompt}, stream={stream}"+ );+ }+ }+ Ok(())+}diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 50e9258be..d17259916 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -53,6 +53,7 @@ route reaches no upstream. A file without a `[targets]` table is rejected with| `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. || `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. A route's forwarding clients must use one credential family unless they all use the same scheme, host, and port, such as one LLM gateway. || `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. |+| `fallback_on_rate_limit` | No | `true` | Try another completion candidate after an upstream HTTP 429 exhausts retries. Set to `false` to return the 429 instead. || `max_retries` | No | `2` | Retry budget, `0`–`10`. || `failure_cooldown_ms` | No | `5000` (5 seconds) | Skip a backend for this many milliseconds after an exhausted transient completion failure. Zero disables it. || `timeout_ms` | No | unset | Deadline in milliseconds for all attempts, retry delays, and the complete response, including stream reads. Must be at least `1`. Unset leaves the wait unbounded. |@@ -66,12 +67,17 @@ The cooldown starts after retries are exhausted.`failure_cooldown_ms` tracks transport failures, timeouts, HTTP 408/429, and 5xxresponses after retries. State is shared across callers per model within the client,including callers using forwarded credentials. With `forward_auth = true`, HTTP 429-only triggers request-local retries and fallback; other callers keep trying the model.-For shared credentials, HTTP 429 also triggers cooldown. During cooldown,+triggers request-local retries and, by default, fallback; other callers keep trying the model.+For shared credentials, HTTP 429 also triggers cooldown by default. During cooldown,ordered fallback tries the next candidate. A terminal cooldown error returns HTTP503. Calls resume together after expiry. Auxiliary calls and errors after a streamis returned leave cooldown state unchanged.+With `fallback_on_rate_limit = false`, HTTP 429 does not start cooldown, so later+requests can receive the upstream response too. Retry budgets and deadlines still+apply. Other errors keep their existing fallback and cooldown behavior. This option+controls completion candidates; judge and advisor error recovery is unchanged.+`timeout_ms` applies separately to every call through the client, including judgeverdicts and answers. To give a judge a short deadline without limiting theanswering models, put the judge on its own `[llm_clients]` entry; two entries may