Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(llm-client): allow disabling fallback on rate limits (#1004)

Elias Shieh9ee35cb3ec72
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index e1976de29..5dd586bd7 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -128,6 +128,7 @@ pub struct TranslatingLlmClient {
model_to_config: HashMap<ModelId, ModelConfig>,
client: reqwest::Client,
forward_auth_client: reqwest::Client,
+ is_rate_limit_cooldown_enabled: bool,
cooldown_epoch: Instant,
// Milliseconds since cooldown_epoch. Each atomic is independent, so relaxed ordering suffices.
unavailable_until: HashMap<ModelId, AtomicU64>,
@@ -166,11 +167,18 @@ impl TranslatingLlmClient {
model_to_config,
client,
forward_auth_client,
+ is_rate_limit_cooldown_enabled: true,
cooldown_epoch: Instant::now(),
unavailable_until,
})
}
+ /// Disable HTTP 429 cooldown so subsequent calls can receive the upstream error.
+ pub fn without_rate_limit_cooldown(mut self) -> Self {
+ self.is_rate_limit_cooldown_enabled = false;
+ self
+ }
+
/// The backend serving `model` over `format` — the default backend when its
/// format matches, otherwise a matching entry in `other_backends`; `None` when
/// the model is unknown or has no backend for `format`.
@@ -302,11 +310,11 @@ impl TranslatingLlmClient {
let result = self
.send_with_retries(&url, backend, &body, metadata, model, streaming)
.await;
- // Forwarded credentials can hit a user's quota while the backend remains healthy.
+ // Keep quota failures request-local when forwarding auth or returning 429 to callers.
if let Some(until) = unavailable_until
&& let Err(error) = &result
&& is_transient_failure(error)
- && !(backend.is_forwarding_auth()
+ && !((backend.is_forwarding_auth() || !self.is_rate_limit_cooldown_enabled)
&& matches!(error, LlmClientError::UpstreamHttp { status, .. } if *status == StatusCode::TOO_MANY_REQUESTS))
{
let deadline = duration_millis(self.cooldown_epoch.elapsed().saturating_add(cooldown));
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index ab8a3e9cb..a80bde58c 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -18,7 +18,7 @@
//! [`ClientRouter`] sends native Responses state back to its provider; for Chat or Anthropic
//! targets it retains the canonical history needed to materialize the continuation.
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Instant;
@@ -261,6 +261,14 @@ async fn call_first_available(
{
Ok(response) => return Ok(response),
Err(error) if index + 1 == models.len() => return Err(error),
+ Err(error)
+ if clients.rate_limit_fallback_disabled.contains(target)
+ && matches!(&error, LibsyError::ClientCall {
+ source: LlmClientError::UpstreamHttp { status, .. }, ..
+ } if *status == StatusCode::TOO_MANY_REQUESTS) =>
+ {
+ return Err(error);
+ }
Err(error) => match fallback_reason(&error) {
Some(reason) => tracing::info!(
from = %target,
@@ -582,6 +590,7 @@ fn conversation_id(fields: &serde_json::Map<String, Value>) -> Option<&str> {
#[derive(Clone)]
pub struct ClientRouter {
inner: Arc<ClientRouting>,
+ rate_limit_fallback_disabled: Arc<HashSet<ModelId>>,
}
struct ClientRouting {
@@ -668,6 +677,7 @@ impl ClientRouter {
.next()
.is_some_and(|first| clients.any(|client| !Arc::ptr_eq(first, client)));
Self {
+ rate_limit_fallback_disabled: Arc::default(),
inner: Arc::new(ClientRouting {
routing: Routing::ByModel(by_model),
decision_clients,
@@ -694,6 +704,7 @@ impl ClientRouter {
decision_clients: HashMap<ModelId, Arc<dyn RoutedDecisionClient>>,
) -> Self {
Self {
+ rate_limit_fallback_disabled: Arc::default(),
inner: Arc::new(ClientRouting {
routing: Routing::Single(client),
decision_clients,
@@ -705,6 +716,16 @@ impl ClientRouter {
}
}
+ /// Stop completion fallback on HTTP 429 for the given models.
+ ///
+ /// For translating clients, also use
+ /// [`TranslatingLlmClient::without_rate_limit_cooldown`](crate::TranslatingLlmClient::without_rate_limit_cooldown)
+ /// so later calls are not skipped with a generic cooldown error.
+ pub fn without_rate_limit_fallback(mut self, models: HashSet<ModelId>) -> Self {
+ self.rate_limit_fallback_disabled = Arc::new(models);
+ self
+ }
+
/// The client that serves `model`.
///
/// Errors with [`LlmClientError::Configuration`] when the router maps models and has no
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index fb833f2f5..70f472ec9 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -446,11 +446,12 @@ impl DeploymentConfig {
let mut clients = BTreeMap::new();
for (name, model_configs) in models_by_client {
- let client = Arc::new(
- TranslatingLlmClient::new(&model_configs)
- .map_err(|error| RunnerError::configuration(error.to_string()))?,
- );
- clients.insert(name, client);
+ let mut client = TranslatingLlmClient::new(&model_configs)
+ .map_err(|error| RunnerError::configuration(error.to_string()))?;
+ if !self.llm_clients[&name].fallback_on_rate_limit {
+ client = client.without_rate_limit_cooldown();
+ }
+ clients.insert(name, Arc::new(client));
}
Ok(clients)
}
@@ -588,6 +589,17 @@ impl DeploymentConfig {
prompts,
routing_answer_target,
&completion_targets,
+ )
+ .without_rate_limit_fallback(
+ route
+ .routing_target_names()
+ .into_iter()
+ .filter_map(|name| {
+ let target = &self.targets[name];
+ (!self.llm_clients[&target.llm_client].fallback_on_rate_limit)
+ .then(|| target.id.clone())
+ })
+ .collect(),
);
Ok((router, caller_auth))
}
@@ -758,6 +770,8 @@ struct LlmClientConfig {
extra_headers: BTreeMap<String, String>,
#[serde(default = "default_max_retries")]
max_retries: u32,
+ #[serde(default = "default_fallback_on_rate_limit")]
+ fallback_on_rate_limit: bool,
/// Cooldown after an exhausted transient completion failure. Defaults to 5 seconds; zero disables it.
#[serde(default = "default_failure_cooldown_ms")]
failure_cooldown_ms: u64,
@@ -908,6 +922,10 @@ const fn default_max_retries() -> u32 {
DEFAULT_MAX_RETRIES
}
+const fn default_fallback_on_rate_limit() -> bool {
+ true
+}
+
const fn default_failure_cooldown_ms() -> u64 {
5000
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index d075fb2b4..ddef98875 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -140,6 +140,17 @@ async fn upstream_chat(
}
let model = body["model"].as_str().unwrap_or("unknown").to_string();
+ if model == "model/rate-limited" {
+ return (
+ if prompt == "unavailable" {
+ StatusCode::SERVICE_UNAVAILABLE
+ } else {
+ StatusCode::TOO_MANY_REQUESTS
+ },
+ Json(json!({"error": {"message": "quota exhausted", "code": "usage_limit"}})),
+ )
+ .into_response();
+ }
if model == "model/classifier" {
let mut pending = HashSet::new();
for message in body["messages"].as_array().into_iter().flatten() {
@@ -5450,3 +5461,78 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult {
);
Ok(())
}
+
+#[tokio::test]
+async fn rate_limit_fallback_policy() -> TestResult {
+ for (policy, forward_auth, prompt, is_terminal) in [
+ ("", false, "hello", false),
+ ("", true, "hello", false),
+ ("fallback_on_rate_limit = true", false, "hello", false),
+ ("fallback_on_rate_limit = false", false, "hello", true),
+ ("fallback_on_rate_limit = false", true, "hello", true),
+ (
+ "fallback_on_rate_limit = false",
+ false,
+ "unavailable",
+ false,
+ ),
+ ] {
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "{base_url}"
+forward_auth = {forward_auth}
+{policy}
+max_retries = 1
+failure_cooldown_ms = 60000
+[llm_clients.secondary]
+format = "openai_chat"
+base_url = "{base_url}"
+[targets]
+first = {{ id = "model/rate-limited", llm_client = "primary" }}
+second = {{ id = "model/fallback", llm_client = "secondary" }}
+[routes.test]
+id = "{ROUTE_MODEL}"
+type = "random"
+targets = ["first", "second"]
+weights = [1000, 1]
+seed = 17
+"#,
+ base_url = upstream.base_url,
+ ))?);
+ let mut expected_calls = Vec::new();
+ // The second request exercises cooldown as well as streaming HTTP errors.
+ for stream in [false, true] {
+ let response = send_with_headers(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": ROUTE_MODEL, "stream": stream,
+ "messages": [{"role": "user", "content": prompt}]
+ })),
+ &[("authorization", "Bearer test-caller")],
+ )
+ .await?;
+ if !stream || is_terminal || forward_auth {
+ expected_calls.extend(["model/rate-limited"; 2]);
+ }
+ if is_terminal {
+ assert_eq!(response.status, StatusCode::TOO_MANY_REQUESTS);
+ assert_eq!(response.json()?["error"]["code"], "usage_limit");
+ } else {
+ assert_eq!(response.status, StatusCode::OK);
+ expected_calls.push("model/fallback");
+ }
+ assert_eq!(
+ upstream.models().await,
+ expected_calls,
+ "{policy}, forward_auth={forward_auth}, {prompt}, stream={stream}"
+ );
+ }
+ }
+ Ok(())
+}
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 50e9258be..d17259916 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -53,6 +53,7 @@ route reaches no upstream. A file without a `[targets]` table is rejected with
| `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. |
| `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. A route's forwarding clients must use one credential family unless they all use the same scheme, host, and port, such as one LLM gateway. |
| `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. |
+| `fallback_on_rate_limit` | No | `true` | Try another completion candidate after an upstream HTTP 429 exhausts retries. Set to `false` to return the 429 instead. |
| `max_retries` | No | `2` | Retry budget, `0`–`10`. |
| `failure_cooldown_ms` | No | `5000` (5 seconds) | Skip a backend for this many milliseconds after an exhausted transient completion failure. Zero disables it. |
| `timeout_ms` | No | unset | Deadline in milliseconds for all attempts, retry delays, and the complete response, including stream reads. Must be at least `1`. Unset leaves the wait unbounded. |
@@ -66,12 +67,17 @@ The cooldown starts after retries are exhausted.
`failure_cooldown_ms` tracks transport failures, timeouts, HTTP 408/429, and 5xx
responses after retries. State is shared across callers per model within the client,
including callers using forwarded credentials. With `forward_auth = true`, HTTP 429
-only triggers request-local retries and fallback; other callers keep trying the model.
-For shared credentials, HTTP 429 also triggers cooldown. During cooldown,
+triggers request-local retries and, by default, fallback; other callers keep trying the model.
+For shared credentials, HTTP 429 also triggers cooldown by default. During cooldown,
ordered fallback tries the next candidate. A terminal cooldown error returns HTTP
503. Calls resume together after expiry. Auxiliary calls and errors after a stream
is returned leave cooldown state unchanged.
+With `fallback_on_rate_limit = false`, HTTP 429 does not start cooldown, so later
+requests can receive the upstream response too. Retry budgets and deadlines still
+apply. Other errors keep their existing fallback and cooldown behavior. This option
+controls completion candidates; judge and advisor error recovery is unchanged.
+
`timeout_ms` applies separately to every call through the client, including judge
verdicts and answers. To give a judge a short deadline without limiting the
answering models, put the judge on its own `[llm_clients]` entry; two entries may