Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(llm): honor provider retry hints

AkitaOnRailsa736b4b74eb9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa58c11c..1a490594 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,12 @@ version in [`docs/changelog/`](docs/changelog/):
### Fixed
+- Fallback-provider circuits now honor bounded retry hints instead of retrying
+ a quota-exhausted candidate every 30 seconds. HTTP `Retry-After` takes
+ precedence; Gemini's `google.rpc.RetryInfo.retryDelay` is read from the full
+ scrubbed error body before its diagnostic text is cut to 1 KiB. Hints are
+ clamped to 30 seconds through one hour, remain scoped to one chain entry, and
+ never enlarge the logged provider body (#1619).
- Fixed the PowerShell Docker wrapper leaving native managed commands without
the root bearer stored in a named volume. Native `install-hooks` now refreshes
a host copy under the user's local app-data directory once, and `run`,
diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rs
index 78d99b62..b82d1a91 100644
--- a/crates/ai-memory-cli/src/commands/llm_test.rs
+++ b/crates/ai-memory-cli/src/commands/llm_test.rs
@@ -150,6 +150,7 @@ mod tests {
LlmError::Provider {
status: 429,
body: "rate_limit_error".into(),
+ retry_after: None,
},
);
assert!(format!("{err:#}").contains("not a real rate limit"));
@@ -166,6 +167,7 @@ mod tests {
LlmError::Provider {
status,
body: String::new(),
+ retry_after: None,
},
);
assert!(!format!("{err:#}").contains("not a real rate limit"));
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index ab9cafe3..8bf886b7 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -4951,6 +4951,7 @@ mod tests {
Err(ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
})
}
@@ -4968,6 +4969,7 @@ mod tests {
Err(ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
})
}
@@ -5053,10 +5055,12 @@ mod tests {
ConsolidationFailure::MalformedRequest => LlmError::Provider {
status: 400,
body: "bad request".into(),
+ retry_after: None,
},
ConsolidationFailure::RateLimited => LlmError::Provider {
status: 429,
body: "rate limited".into(),
+ retry_after: None,
},
};
Err(error)
@@ -5095,10 +5099,12 @@ mod tests {
ConsolidatorError::Llm(LlmError::Provider {
status: 401,
body: "invalid api key".into(),
+ retry_after: None,
}),
ConsolidatorError::Llm(LlmError::Provider {
status: 400,
body: "bad request".into(),
+ retry_after: None,
}),
];
for error in &terminal {
@@ -5127,14 +5133,17 @@ mod tests {
ConsolidatorError::Llm(LlmError::Provider {
status: 429,
body: "rate limited".into(),
+ retry_after: None,
}),
ConsolidatorError::Llm(LlmError::Provider {
status: 503,
body: "capacity".into(),
+ retry_after: None,
}),
ConsolidatorError::Llm(LlmError::Provider {
status: 502,
body: "bad gateway".into(),
+ retry_after: None,
}),
ConsolidatorError::Llm(LlmError::Http(connect)),
];
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rs
index 829e11b7..4c815ae0 100644
--- a/crates/ai-memory-consolidate/src/auto_improve.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve.rs
@@ -2308,6 +2308,7 @@ mod tests {
let error = AutoImproveError::Llm(LlmError::Provider {
status: 400,
body: AUTO_IMPROVE_REDACTION_SENTINEL.into(),
+ retry_after: None,
});
let summary = redacted_auto_improve_summary(&error);
assert_eq!(summary, "auto-improve failed: class=provider status=400");
diff --git a/crates/ai-memory-consolidate/src/auto_improve_schedule.rs b/crates/ai-memory-consolidate/src/auto_improve_schedule.rs
index 387d34b1..ca7e2a29 100644
--- a/crates/ai-memory-consolidate/src/auto_improve_schedule.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve_schedule.rs
@@ -731,6 +731,7 @@ mod tests {
Err(LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
})
}
@@ -750,6 +751,7 @@ mod tests {
Err(LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
})
}
@@ -798,6 +800,7 @@ mod tests {
Err(LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
})
}
@@ -1072,6 +1075,7 @@ mod tests {
let provider_400 = anyhow::Error::new(AutoImproveError::Llm(LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
}));
assert_eq!(
redacted_scheduler_error_summary(&provider_400),
@@ -1082,6 +1086,7 @@ mod tests {
let bare = anyhow::Error::new(LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
});
assert_eq!(
redacted_scheduler_error_summary(&bare),
diff --git a/crates/ai-memory-consolidate/src/bootstrap.rs b/crates/ai-memory-consolidate/src/bootstrap.rs
index aaed0e9c..9875fa54 100644
--- a/crates/ai-memory-consolidate/src/bootstrap.rs
+++ b/crates/ai-memory-consolidate/src/bootstrap.rs
@@ -1731,6 +1731,7 @@ mod tests {
return Err(LlmError::Provider {
status: 503,
body: "busy".into(),
+ retry_after: None,
});
}
Ok(serde_json::json!({ "pages": [], "rationale": "ok" }))
diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rs
index abed2a44..e2129ce5 100644
--- a/crates/ai-memory-consolidate/src/consolidator.rs
+++ b/crates/ai-memory-consolidate/src/consolidator.rs
@@ -2314,6 +2314,7 @@ mod tests {
let error = ConsolidatorError::Llm(LlmError::Provider {
status: 400,
body: REDACTION_SENTINEL.into(),
+ retry_after: None,
});
let summary = redacted_error_summary(&error);
assert_eq!(summary, "consolidation failed: class=provider status=400");
@@ -2328,6 +2329,7 @@ mod tests {
let provider = ConsolidatorError::Llm(LlmError::Provider {
status: 400,
body: REDACTION_SENTINEL.into(),
+ retry_after: None,
});
let serde_error = ConsolidatorError::Serde(REDACTION_SENTINEL.into());
let provider_summary = redacted_error_summary(&provider);
@@ -4670,6 +4672,7 @@ mod tests {
ScriptedFailure::Transient => LlmError::Provider {
status: 503,
body: "This model is currently experiencing high demand.".into(),
+ retry_after: None,
},
ScriptedFailure::Deterministic => LlmError::Auth("expired".into()),
});
diff --git a/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs b/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs
index 3864e4d8..82148777 100644
--- a/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs
+++ b/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs
@@ -198,6 +198,7 @@ impl Embedder for ConfigurableEmbedder {
return Err(ai_memory_llm::LlmError::Provider {
status: 500,
body: "synthetic configured failure".into(),
+ retry_after: None,
});
}
Ok(vec![0.5_f32; self.dim as usize])
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rs
index 25c6afce..c51af6a3 100644
--- a/crates/ai-memory-llm/src/anthropic.rs
+++ b/crates/ai-memory-llm/src/anthropic.rs
@@ -12,7 +12,7 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::openai_compat::{first_json_object, strip_reasoning_blocks};
use crate::provider::LlmProvider;
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
use crate::types::{
ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, ReasoningEffort, Usage,
};
@@ -448,13 +448,8 @@ impl AnthropicProvider {
);
}
let resp = headers.apply(builder).json(body).send().await?;
- let status = resp.status();
- if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ if !resp.status().is_success() {
+ return Err(provider_error(resp).await);
}
response_json_limited::<R>(resp).await
}
diff --git a/crates/ai-memory-llm/src/codex_responses.rs b/crates/ai-memory-llm/src/codex_responses.rs
index 3b8504c0..2c7a8611 100644
--- a/crates/ai-memory-llm/src/codex_responses.rs
+++ b/crates/ai-memory-llm/src/codex_responses.rs
@@ -7,7 +7,7 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::openai_oauth::{CodexResponsesRequest, CodexResponsesResponse, parse_sse_response};
-use crate::response::{provider_error_body, response_json_limited, response_text_limited};
+use crate::response::{provider_error, response_json_limited, response_text_limited};
use crate::types::ExtraHeaders;
/// ChatGPT/Codex Responses backend.
@@ -49,12 +49,8 @@ pub(crate) async fn post_codex_responses(
request = request.header("chatgpt-account-id", account_id);
}
let response = request.send().await.map_err(LlmError::from)?;
- let status = response.status();
- if !status.is_success() {
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body: provider_error_body(response).await,
- });
+ if !response.status().is_success() {
+ return Err(provider_error(response).await);
}
if body.stream {
parse_sse_response(&response_text_limited(response).await?, body.model)
diff --git a/crates/ai-memory-llm/src/copilot.rs b/crates/ai-memory-llm/src/copilot.rs
index c41335fc..a58f901c 100644
--- a/crates/ai-memory-llm/src/copilot.rs
+++ b/crates/ai-memory-llm/src/copilot.rs
@@ -23,7 +23,7 @@ use crate::error::{LlmError, LlmResult};
use crate::openai::{STRUCTURED_OUTPUT_SCHEMA_NAME, enforce_strict_object_schemas};
use crate::openai_oauth::model_uses_default_temperature;
use crate::provider::LlmProvider;
-use crate::response::{provider_error_body, response_json_limited, response_text_limited};
+use crate::response::{provider_error, response_json_limited, response_text_limited};
use crate::text::truncate_for_embedding;
use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, Usage};
@@ -391,13 +391,8 @@ impl CopilotProvider {
.headers(copilot_runtime_headers()),
);
let resp = request.json(body).send().await.map_err(LlmError::from)?;
- let status = resp.status();
- if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ if !resp.status().is_success() {
+ return Err(provider_error(resp).await);
}
response_json_limited::<CopilotChatResponse>(resp).await
}
@@ -418,12 +413,8 @@ impl CopilotProvider {
.headers(copilot_runtime_headers()),
);
let resp = request.json(body).send().await.map_err(LlmError::from)?;
- let status = resp.status();
- if !status.is_success() {
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body: provider_error_body(resp).await,
- });
+ if !resp.status().is_success() {
+ return Err(provider_error(resp).await);
}
response_json_limited::<CopilotResponsesResponse>(resp).await
}
@@ -617,18 +608,10 @@ impl Embedder for CopilotEmbedder {
}
// Client errors (e.g. input too long) are not retried.
if status.as_u16() == 400 {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
let body = response_text_limited(resp).await?;
let values = parse_openai_embedding_values(&body, status.as_u16())?;
@@ -666,7 +649,13 @@ async fn exchange_copilot_token(
.map_err(LlmError::from)?;
let status = resp.status();
if !status.is_success() {
- let body = provider_error_body(resp).await;
+ let error = provider_error(resp).await;
+ let LlmError::Provider {
+ body, retry_after, ..
+ } = error
+ else {
+ return Err(error);
+ };
// GitHub rate-limits this endpoint per account. A 429/5xx here says
// nothing about the credential, and `LlmError::Auth` is not transient,
// so classifying it as auth stops the fallback chain and tells the
@@ -675,6 +664,7 @@ async fn exchange_copilot_token(
return Err(LlmError::Provider {
status: status.as_u16(),
body: format!("copilot token exchange: {body}"),
+ retry_after,
});
}
return Err(LlmError::Auth(format!(
diff --git a/crates/ai-memory-llm/src/cursor_agent.rs b/crates/ai-memory-llm/src/cursor_agent.rs
index 056b16f9..e3e26f68 100644
--- a/crates/ai-memory-llm/src/cursor_agent.rs
+++ b/crates/ai-memory-llm/src/cursor_agent.rs
@@ -215,6 +215,7 @@ impl CursorAgentProvider {
.map_err(|_| LlmError::Provider {
status: 504,
body: "cursor agent timed out".into(),
+ retry_after: None,
})??;
let (status, stdout, stderr) = collected;
@@ -224,6 +225,7 @@ impl CursorAgentProvider {
return Err(LlmError::Provider {
status: u16::try_from(code).unwrap_or(1),
body: truncate_chars(&body, 500),
+ retry_after: None,
});
}
let text = String::from_utf8_lossy(&stdout).trim().to_string();
diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rs
index 608d8485..babf2816 100644
--- a/crates/ai-memory-llm/src/embedding.rs
+++ b/crates/ai-memory-llm/src/embedding.rs
@@ -23,7 +23,7 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::openai::normalize_openai_base;
-use crate::response::{provider_error_body, response_json_limited, response_text_limited};
+use crate::response::{provider_error, response_json_limited, response_text_limited};
use crate::text::{truncate_for_embedding, truncate_with_ellipsis};
/// Conservative per-request input cap for OpenAI-compatible embedding APIs
@@ -222,6 +222,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
let v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Provider {
status,
body: truncate_with_ellipsis(&format!("openai embeddings json: {e}; body={body}"), 1024),
+ retry_after: None,
})?;
if let Some(err) = v.get("error") {
let msg = err
@@ -232,6 +233,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
return Err(LlmError::Provider {
status,
body: truncate_with_ellipsis(msg, 1024),
+ retry_after: None,
});
}
if let Some(data) = v.get("data").and_then(|d| d.as_array()) {
@@ -241,6 +243,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
&format!("openai embeddings data[] empty; body={body}"),
512,
),
+ retry_after: None,
})?;
let emb = first
.get("embedding")
@@ -251,6 +254,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
&format!("missing embedding array in data[0]; body={body}"),
512,
),
+ retry_after: None,
})?;
let mut out = Vec::with_capacity(emb.len());
for n in emb {
@@ -260,6 +264,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
&format!("non-numeric embedding value in data[0]; body={body}"),
512,
),
+ retry_after: None,
})?;
out.push(f as f32);
}
@@ -275,6 +280,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
&format!("openai embeddings shape: {e}; body={body}"),
1024,
),
+ retry_after: None,
})?;
let first = parsed.data.into_iter().next().ok_or_else(|| {
LlmError::UnexpectedShape(format!(
@@ -386,18 +392,10 @@ async fn openai_style_embed(
}
// Client errors (e.g. input > 8192 tokens) are not retried.
if status.as_u16() == 400 {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
let body = response_text_limited(resp).await?;
let values = parse_openai_embedding_values(&body, status.as_u16())?;
@@ -713,11 +711,7 @@ impl Embedder for VoyageEmbedder {
};
let status = resp.status();
if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
let parsed: VoyageResponse = response_json_limited(resp).await?;
let first =
diff --git a/crates/ai-memory-llm/src/error.rs b/crates/ai-memory-llm/src/error.rs
index 2dc3b90a..66e0acd6 100644
--- a/crates/ai-memory-llm/src/error.rs
+++ b/crates/ai-memory-llm/src/error.rs
@@ -1,5 +1,7 @@
//! LLM error type.
+use std::time::Duration;
+
use thiserror::Error;
/// Result alias used throughout the LLM crate.
@@ -20,6 +22,8 @@ pub enum LlmError {
status: u16,
/// Response body (truncated).
body: String,
+ /// Provider-requested delay before another equivalent request.
+ retry_after: Option<Duration>,
},
/// JSON (de)serialization failure.
@@ -129,6 +133,16 @@ impl LlmError {
}
}
+ /// Provider-requested delay before another equivalent request, when the
+ /// failed HTTP response carried a trustworthy, boundedly parsed hint.
+ #[must_use]
+ pub const fn retry_after(&self) -> Option<Duration> {
+ match self {
+ Self::Provider { retry_after, .. } => *retry_after,
+ _ => None,
+ }
+ }
+
/// Redacted error category used for health reporting and the fallback
/// chain's aggregate error — never a response body or secret.
#[must_use]
@@ -164,7 +178,8 @@ mod tests {
assert!(
LlmError::Provider {
status: 429,
- body: String::new()
+ body: String::new(),
+ retry_after: None,
}
.is_transient()
);
@@ -172,7 +187,8 @@ mod tests {
assert!(
LlmError::Provider {
status,
- body: String::new()
+ body: String::new(),
+ retry_after: None,
}
.is_transient(),
"status {status} should be transient"
@@ -183,7 +199,8 @@ mod tests {
assert!(
!LlmError::Provider {
status,
- body: String::new()
+ body: String::new(),
+ retry_after: None,
}
.is_transient(),
"status {status} must not be treated as transient"
@@ -255,7 +272,8 @@ mod tests {
assert_eq!(
LlmError::Provider {
status: 503,
- body: "boom".into()
+ body: "boom".into(),
+ retry_after: None,
}
.http_status(),
Some(503)
@@ -270,6 +288,7 @@ mod tests {
LlmError::Provider {
status: 500,
body: "secret upstream body".into(),
+ retry_after: None,
},
"provider",
),
@@ -314,7 +333,8 @@ mod tests {
assert!(
LlmError::Provider {
status,
- body: String::new()
+ body: String::new(),
+ retry_after: None,
}
.is_transient(),
"{status} must agree with is_transient"
@@ -325,7 +345,8 @@ mod tests {
assert!(
!LlmError::Provider {
status,
- body: String::new()
+ body: String::new(),
+ retry_after: None,
}
.is_transient(),
"{status} must agree with is_transient"
diff --git a/crates/ai-memory-llm/src/fallback.rs b/crates/ai-memory-llm/src/fallback.rs
index 4186d6d1..a433781a 100644
--- a/crates/ai-memory-llm/src/fallback.rs
+++ b/crates/ai-memory-llm/src/fallback.rs
@@ -26,6 +26,7 @@ use crate::types::{ChatRequest, ChatResponse, LlmOperationId};
/// before it is eligible again. Restarting the process clears all circuit
/// state; there is no durable or cross-process cooldown.
pub const CIRCUIT_COOLDOWN: Duration = Duration::from_secs(30);
+const MAX_CIRCUIT_COOLDOWN: Duration = Duration::from_secs(60 * 60);
/// One provider in an ordered fallback chain.
///
@@ -129,7 +130,12 @@ impl FallbackLlmProvider {
state.last_error_status = err.http_status();
state.last_error_class = Some(err.class());
if err.is_transient() {
- state.circuit_open_until = Some(Instant::now() + CIRCUIT_COOLDOWN);
+ let cooldown = err
+ .retry_after()
+ .unwrap_or(CIRCUIT_COOLDOWN)
+ .max(CIRCUIT_COOLDOWN)
+ .min(MAX_CIRCUIT_COOLDOWN);
+ state.circuit_open_until = Some(Instant::now() + cooldown);
}
}
@@ -416,9 +422,14 @@ mod tests {
fn clone_err(err: &LlmError) -> LlmError {
match err {
- LlmError::Provider { status, body } => LlmError::Provider {
+ LlmError::Provider {
+ status,
+ body,
+ retry_after,
+ } => LlmError::Provider {
status: *status,
body: body.clone(),
+ retry_after: *retry_after,
},
LlmError::Serde(msg) => LlmError::Serde(msg.clone()),
LlmError::UnexpectedShape(msg) => LlmError::UnexpectedShape(msg.clone()),
@@ -481,6 +492,7 @@ mod tests {
LlmError::Provider {
status,
body: "upstream body that must never leak".to_string(),
+ retry_after: None,
}
}
@@ -569,22 +581,27 @@ mod tests {
LlmError::Provider {
status: 400,
body: String::new(),
+ retry_after: None,
},
LlmError::Provider {
status: 401,
body: String::new(),
+ retry_after: None,
},
LlmError::Provider {
status: 403,
body: String::new(),
+ retry_after: None,
},
LlmError::Provider {
status: 404,
body: String::new(),
+ retry_after: None,
},
LlmError::Provider {
status: 422,
body: String::new(),
+ retry_after: None,
},
LlmError::Schema("bad schema".into()),
LlmError::Serde("deserialize failed".into()),
@@ -649,6 +666,40 @@ mod tests {
LlmError::Provider {
status,
body: body.to_string(),
+ retry_after: None,
+ }
+ }
+
+ #[test]
+ fn provider_retry_hints_are_clamped_to_the_circuit_bounds() {
+ let chain = FallbackLlmProvider::new(vec![Candidate::new(
+ "primary",
+ "m1",
+ Arc::new(ScriptedLlm::ok("primary", "m1", "unused")),
+ )]);
+
+ for (hint, expected) in [(5, 30), (90, 90), (7_200, 3_600)] {
+ let before = Instant::now();
+ chain.record_error(
+ 0,
+ &LlmError::Provider {
+ status: 429,
+ body: String::new(),
+ retry_after: Some(Duration::from_secs(hint)),
+ },
+ );
+ let until = chain.entries[0]
+ .state
+ .lock()
+ .unwrap()
+ .circuit_open_until
+ .unwrap();
+ let actual = until.duration_since(before);
+ assert!(
+ actual >= Duration::from_secs(expected)
+ && actual < Duration::from_secs(expected + 1),
+ "hint {hint}s produced {actual:?}"
+ );
}
}
diff --git a/crates/ai-memory-llm/src/gemini.rs b/crates/ai-memory-llm/src/gemini.rs
index cfe625ed..49dd0439 100644
--- a/crates/ai-memory-llm/src/gemini.rs
+++ b/crates/ai-memory-llm/src/gemini.rs
@@ -18,7 +18,7 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::provider::LlmProvider;
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, Role, Usage};
/// Default Gemini API base.
@@ -253,13 +253,8 @@ impl GeminiProvider {
.header("content-type", "application/json"),
);
let resp = request.json(body).send().await?;
- let status = resp.status();
- if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ if !resp.status().is_success() {
+ return Err(provider_error(resp).await);
}
response_json_limited::<R>(resp).await
}
diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rs
index c9133844..ca50465b 100644
--- a/crates/ai-memory-llm/src/google.rs
+++ b/crates/ai-memory-llm/src/google.rs
@@ -11,7 +11,7 @@ use tracing::debug;
use crate::embedding::{Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, retry_delay};
use crate::error::{LlmError, LlmResult};
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
use crate::text::truncate_for_embedding;
/// Default Gemini API host.
@@ -118,11 +118,7 @@ impl GoogleEmbedder {
continue;
}
if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ return Err(provider_error(resp).await);
}
let parsed: GeminiEmbedResponse = response_json_limited(resp).await?;
let values = parsed.embedding.values;
diff --git a/crates/ai-memory-llm/src/health.rs b/crates/ai-memory-llm/src/health.rs
index 17b9527c..5642bf3d 100644
--- a/crates/ai-memory-llm/src/health.rs
+++ b/crates/ai-memory-llm/src/health.rs
@@ -755,7 +755,7 @@ fn error_status_and_message(err: &LlmError) -> (Option<u16>, String) {
e.status().map(|status| status.as_u16()),
truncate_error_message(&e.to_string()),
),
- LlmError::Provider { status, body } => (Some(*status), truncate_error_message(body)),
+ LlmError::Provider { status, body, .. } => (Some(*status), truncate_error_message(body)),
_ => (None, truncate_error_message(&err.to_string())),
}
}
@@ -793,6 +793,7 @@ mod tests {
return Err(LlmError::Provider {
status: 401,
body: "bad token".to_string(),
+ retry_after: None,
});
}
Ok(ChatResponse {
@@ -1022,6 +1023,7 @@ mod tests {
let failure: LlmResult<ChatResponse> = Err(LlmError::Provider {
status: 429,
body: "secret upstream body".into(),
+ retry_after: None,
});
workloads.record(
LlmWorkload::Rerank,
diff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rs
index 1bb2fb92..47c652c6 100644
--- a/crates/ai-memory-llm/src/lib.rs
+++ b/crates/ai-memory-llm/src/lib.rs
@@ -154,7 +154,7 @@ pub use opencode::{
};
pub use provider::{LlmProvider, complete_structured, complete_structured_with_operation_id};
pub use reranker::{LlmReranker, RerankCandidate, RerankScore, RerankStage, Reranker};
-pub use response::provider_error_body;
+pub use response::{provider_error, provider_error_body};
pub use stored_token::StoredOAuthToken;
pub use types::{
ChatMessage, ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, LlmWorkload,
diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rs
index 2e9ed387..2b867795 100644
--- a/crates/ai-memory-llm/src/openai.rs
+++ b/crates/ai-memory-llm/src/openai.rs
@@ -10,7 +10,7 @@ use tracing::debug;
use crate::error::{LlmError, LlmResult};
use crate::provider::LlmProvider;
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
use reqwest::header::{HeaderName, HeaderValue};
use crate::types::{
@@ -635,13 +635,8 @@ impl OpenAiProvider {
}
};
let resp = request.json(body).send().await?;
- let status = resp.status();
- if !status.is_success() {
- let body = provider_error_body(resp).await;
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body,
- });
+ if !resp.status().is_success() {
+ return Err(provider_error(resp).await);
}
response_json_limited::<OpenAiResponse>(resp).await
}
diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rs
index 8b600262..cf5791ba 100644
--- a/crates/ai-memory-llm/src/openai_compat.rs
+++ b/crates/ai-memory-llm/src/openai_compat.rs
@@ -383,7 +383,7 @@ fn is_parse_shape_error(err: &LlmError) -> bool {
/// before generation. Generic 4xx errors are deliberately excluded so auth,
/// quota, model, and malformed-request failures retain their original cause.
fn is_response_format_rejection(err: &LlmError) -> bool {
- let LlmError::Provider { status, body } = err else {
+ let LlmError::Provider { status, body, .. } = err else {
return false;
};
if !matches!(status, 400 | 422) {
@@ -655,18 +655,22 @@ mod tests {
assert!(is_response_format_rejection(&LlmError::Provider {
status: 400,
body: "unsupported parameter: response_format".into(),
+ retry_after: None,
}));
assert!(is_response_format_rejection(&LlmError::Provider {
status: 422,
body: "json_schema is not supported".into(),
+ retry_after: None,
}));
assert!(!is_response_format_rejection(&LlmError::Provider {
status: 400,
body: "unknown model".into(),
+ retry_after: None,
}));
assert!(!is_response_format_rejection(&LlmError::Provider {
status: 401,
body: "response_format requires authentication".into(),
+ retry_after: None,
}));
}
@@ -684,15 +688,18 @@ mod tests {
// Transport / HTTP / auth → propagate.
assert!(!is_parse_shape_error(&LlmError::Provider {
status: 500,
- body: "engine boom".into()
+ body: "engine boom".into(),
+ retry_after: None,
}));
assert!(!is_parse_shape_error(&LlmError::Provider {
status: 401,
- body: "unauthorized".into()
+ body: "unauthorized".into(),
+ retry_after: None,
}));
assert!(!is_parse_shape_error(&LlmError::Provider {
status: 429,
- body: "rate limited".into()
+ body: "rate limited".into(),
+ retry_after: None,
}));
}
diff --git a/crates/ai-memory-llm/src/openai_oauth.rs b/crates/ai-memory-llm/src/openai_oauth.rs
index 17c1ff13..4a7fbd62 100644
--- a/crates/ai-memory-llm/src/openai_oauth.rs
+++ b/crates/ai-memory-llm/src/openai_oauth.rs
@@ -366,6 +366,7 @@ pub(crate) fn parse_sse_response(body: &str, model: &str) -> LlmResult<CodexResp
return Err(LlmError::Provider {
status: SSE_TERMINAL_ERROR_STATUS,
body: truncate_with_ellipsis(trimmed, SSE_ERROR_BODY_TRIM),
+ retry_after: None,
});
}
let kind = value
@@ -408,6 +409,7 @@ pub(crate) fn parse_sse_response(body: &str, model: &str) -> LlmResult<CodexResp
return Err(LlmError::Provider {
status: SSE_TERMINAL_ERROR_STATUS,
body: truncate_with_ellipsis(trimmed, SSE_ERROR_BODY_TRIM),
+ retry_after: None,
});
}
_ => {}
@@ -828,7 +830,7 @@ mod tests {
let err = parse_sse_response(&body, "gpt-5.5").expect_err("terminal event must fail");
match err {
- LlmError::Provider { status, body } => {
+ LlmError::Provider { status, body, .. } => {
assert_eq!(status, 502);
assert!(body.contains(event), "body should include event: {body}");
assert!(
@@ -886,7 +888,7 @@ mod tests {
let err = parse_sse_response(body, "gpt-5.5").expect_err("terminal event must fail");
match err {
- LlmError::Provider { status, body } => {
+ LlmError::Provider { status, body, .. } => {
assert_eq!(status, SSE_TERMINAL_ERROR_STATUS);
assert!(
body.contains("content_filter"),
@@ -925,7 +927,7 @@ mod tests {
let err = parse_sse_response(body, "gpt-5.5")
.expect_err("top-level error payload must surface as a terminal error");
match err {
- LlmError::Provider { status, body } => {
+ LlmError::Provider { status, body, .. } => {
assert_eq!(status, super::SSE_TERMINAL_ERROR_STATUS);
assert!(
body.contains("context length exceeded"),
diff --git a/crates/ai-memory-llm/src/opencode.rs b/crates/ai-memory-llm/src/opencode.rs
index 8482af68..71c6e3a5 100644
--- a/crates/ai-memory-llm/src/opencode.rs
+++ b/crates/ai-memory-llm/src/opencode.rs
@@ -22,7 +22,7 @@ use crate::openai::{
};
use crate::openai_compat::OpenAiCompatProvider;
use crate::provider::LlmProvider;
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
use crate::types::{
ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, ReasoningEffort, Usage,
};
@@ -470,12 +470,8 @@ impl OpenCodeResponsesProvider {
headers.set_default(name, value);
}
let response = headers.apply(builder).send().await?;
- let status = response.status();
- if !status.is_success() {
- return Err(LlmError::Provider {
- status: status.as_u16(),
- body: provider_error_body(response).await,
- });
+ if !response.status().is_success() {
+ return Err(provider_error(response).await);
}
response_json_limited(response).await
}
@@ -1051,7 +1047,7 @@ mod tests {
.unwrap_err(),
] {
assert!(
- matches!(error, LlmError::Provider { status: code, body } if code == status && body == "gateway error")
+ matches!(error, LlmError::Provider { status: code, body, .. } if code == status && body == "gateway error")
);
}
assert_eq!(server.received_requests().await.unwrap().len(), 2);
diff --git a/crates/ai-memory-llm/src/response.rs b/crates/ai-memory-llm/src/response.rs
index 3b144e6d..5a46fbbf 100644
--- a/crates/ai-memory-llm/src/response.rs
+++ b/crates/ai-memory-llm/src/response.rs
@@ -1,3 +1,5 @@
+use std::time::Duration;
+
use ai_memory_core::Sanitizer;
use serde::de::DeserializeOwned;
@@ -22,6 +24,7 @@ pub(crate) async fn response_bytes_limited(
return Err(LlmError::Provider {
status,
body: format!("provider response exceeded {max_bytes} bytes"),
+ retry_after: None,
});
}
bytes.extend_from_slice(&chunk);
@@ -50,14 +53,97 @@ pub(crate) async fn response_json_limited<T: DeserializeOwned>(
/// Scrubbing runs before the cut so a secret straddling the 1 KiB boundary
/// is still redacted rather than left as an unrecognisable prefix.
pub async fn provider_error_body(resp: reqwest::Response) -> String {
+ match provider_error(resp).await {
+ LlmError::Provider { body, .. } => body,
+ error => format!("<failed to read provider error body: {error}>"),
+ }
+}
+
+/// Build a typed provider error while preserving a retry hint that may live
+/// beyond the bounded display prefix. The body is still read at most 1 MiB,
+/// scrubbed before parsing, and exposed at most 1 KiB.
+pub async fn provider_error(resp: reqwest::Response) -> LlmError {
+ let status = resp.status().as_u16();
+ let header_retry = resp
+ .headers()
+ .get(reqwest::header::RETRY_AFTER)
+ .and_then(|value| value.to_str().ok())
+ .and_then(parse_delta_seconds);
match response_bytes_limited(resp, MAX_PROVIDER_ERROR_BYTES).await {
Ok(bytes) => {
- let body = Sanitizer::builtin().scrub(&String::from_utf8_lossy(&bytes));
- truncate_with_ellipsis(&body, DISPLAY_ERROR_BYTES)
+ let scrubbed = Sanitizer::builtin().scrub(&String::from_utf8_lossy(&bytes));
+ let retry_after = header_retry.or_else(|| google_retry_delay(&scrubbed));
+ LlmError::Provider {
+ status,
+ body: truncate_with_ellipsis(&scrubbed, DISPLAY_ERROR_BYTES),
+ retry_after,
+ }
+ }
+ Err(LlmError::Provider { body, .. }) => LlmError::Provider {
+ status,
+ body,
+ retry_after: header_retry,
+ },
+ Err(error) => LlmError::Provider {
+ status,
+ body: format!("<failed to read provider error body: {error}>"),
+ retry_after: header_retry,
+ },
+ }
+}
+
+fn parse_delta_seconds(raw: &str) -> Option<Duration> {
+ raw.trim().parse::<u64>().ok().map(Duration::from_secs)
+}
+
+fn google_retry_delay(body: &str) -> Option<Duration> {
+ fn visit(value: &serde_json::Value) -> Option<Duration> {
+ match value {
+ serde_json::Value::Object(object) => {
+ let is_retry = object
+ .get("@type")
+ .and_then(serde_json::Value::as_str)
+ .is_some_and(|kind| {
+ matches!(
+ kind,
+ "google.rpc.RetryInfo" | "type.googleapis.com/google.rpc.RetryInfo"
+ )
+ });
+ if is_retry
+ && let Some(delay) = object
+ .get("retryDelay")
+ .and_then(serde_json::Value::as_str)
+ .and_then(parse_google_duration)
+ {
+ return Some(delay);
+ }
+ object.values().find_map(visit)
+ }
+ serde_json::Value::Array(values) => values.iter().find_map(visit),
+ _ => None,
}
- Err(LlmError::Provider { body, .. }) => body,
- Err(e) => format!("<failed to read provider error body: {e}>"),
}
+
+ serde_json::from_str(body)
+ .ok()
+ .and_then(|value| visit(&value))
+}
+
+fn parse_google_duration(raw: &str) -> Option<Duration> {
+ let seconds = raw.strip_suffix('s')?;
+ let (whole, fraction) = seconds.split_once('.').unwrap_or((seconds, ""));
+ if whole.starts_with('-') || !fraction.bytes().all(|byte| byte.is_ascii_digit()) {
+ return None;
+ }
+ let whole = whole.parse::<u64>().ok()?;
+ let nanos = if fraction.is_empty() {
+ 0
+ } else {
+ let kept = &fraction[..fraction.len().min(9)];
+ let value = kept.parse::<u32>().ok()?;
+ value.checked_mul(10_u32.pow(9 - u32::try_from(kept.len()).ok()?))?
+ };
+ Some(Duration::new(whole, nanos))
}
#[cfg(test)]
@@ -65,7 +151,11 @@ mod tests {
use super::*;
/// Answer one request on a loopback port with `status` and `body`.
- async fn serve_once(status: &'static str, body: String) -> String {
+ async fn serve_once_with_headers(
+ status: &'static str,
+ headers: &'static str,
+ body: String,
+ ) -> String {
use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -77,7 +167,7 @@ mod tests {
let mut buf = [0_u8; 4096];
let _ = stream.read(&mut buf).await;
let head = format!(
- "HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = stream.write_all(head.as_bytes()).await;
@@ -86,12 +176,22 @@ mod tests {
format!("http://{addr}/")
}
+ async fn serve_once(status: &'static str, body: String) -> String {
+ serve_once_with_headers(status, "", body).await
+ }
+
async fn error_body_for(body: String) -> String {
let url = serve_once("400 Bad Request", body).await;
let resp = reqwest::Client::new().get(url).send().await.unwrap();
provider_error_body(resp).await
}
+ async fn provider_error_for(headers: &'static str, body: String) -> LlmError {
+ let url = serve_once_with_headers("429 Too Many Requests", headers, body).await;
+ let resp = reqwest::Client::new().get(url).send().await.unwrap();
+ provider_error(resp).await
+ }
+
#[tokio::test]
async fn provider_error_body_redacts_a_secret_straddling_the_display_cut() {
// The cut keeps `ghp_` plus 12 characters: too short for the GitHub
@@ -126,4 +226,72 @@ mod tests {
let oversized = error_body_for("a".repeat(MAX_PROVIDER_ERROR_BYTES + 1)).await;
assert!(oversized.contains("exceeded"), "unexpected: {oversized}");
}
+
+ #[tokio::test]
+ async fn provider_error_reads_gemini_retry_info_before_the_display_cut() {
+ let body = serde_json::json!({
+ "error": {
+ "message": "x".repeat(DISPLAY_ERROR_BYTES * 2),
+ "details": [
+ {"@type": "type.googleapis.com/google.rpc.QuotaFailure"},
+ {
+ "@type": "type.googleapis.com/google.rpc.RetryInfo",
+ "retryDelay": "32993s"
+ }
+ ]
+ }
+ })
+ .to_string();
+
+ let error = provider_error_for("", body).await;
+
+ assert_eq!(error.retry_after(), Some(Duration::from_secs(32_993)));
+ let LlmError::Provider { body, .. } = error else {
+ panic!("expected provider error");
+ };
+ assert!(body.len() <= DISPLAY_ERROR_BYTES + "…".len());
+ assert!(
+ !body.contains("RetryInfo"),
+ "the control must be beyond the cut"
+ );
+ }
+
+ #[tokio::test]
+ async fn provider_error_finds_nested_compat_retry_info_and_parses_fractional_seconds() {
+ let body = serde_json::json!({
+ "error": {
+ "details": [[{
+ "@type": "type.googleapis.com/google.rpc.RetryInfo",
+ "retryDelay": "12.345s"
+ }]]
+ }
+ })
+ .to_string();
+
+ let error = provider_error_for("", body).await;
+
+ assert_eq!(error.retry_after(), Some(Duration::new(12, 345_000_000)));
+ }
+
+ #[tokio::test]
+ async fn retry_after_header_precedes_the_body_and_invalid_hints_are_ignored() {
+ let body = serde_json::json!({
+ "details": [{
+ "@type": "type.googleapis.com/google.rpc.RetryInfo",
+ "retryDelay": "10s"
+ }]
+ })
+ .to_string();
+ let error = provider_error_for("Retry-After: 42\r\n", body).await;
+ assert_eq!(error.retry_after(), Some(Duration::from_secs(42)));
+
+ for raw in ["-1s", "NaNs", "1.second"] {
+ let body = serde_json::json!({
+ "@type": "type.googleapis.com/google.rpc.RetryInfo",
+ "retryDelay": raw
+ })
+ .to_string();
+ assert_eq!(provider_error_for("", body).await.retry_after(), None);
+ }
+ }
}
diff --git a/crates/ai-memory-llm/src/stored_token.rs b/crates/ai-memory-llm/src/stored_token.rs
index cfc8afc7..5f620987 100644
--- a/crates/ai-memory-llm/src/stored_token.rs
+++ b/crates/ai-memory-llm/src/stored_token.rs
@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use crate::auth_file::{load_entry, now_ms, save_entry};
use crate::error::{LlmError, LlmResult};
-use crate::response::{provider_error_body, response_json_limited};
+use crate::response::{provider_error, response_json_limited};
/// A credential-bearing HTTP endpoint validated as TLS-protected, except for
/// explicit loopback development servers. Keeping the parsed URL behind this
@@ -176,7 +176,13 @@ where
.map_err(LlmError::from)?;
let status = resp.status();
if !status.is_success() {
- let body = provider_error_body(resp).await;
+ let error = provider_error(resp).await;
+ let LlmError::Provider {
+ body, retry_after, ..
+ } = error
+ else {
+ return Err(error);
+ };
// A rate-limited or overloaded token endpoint has not rejected the
// grant, so it must stay transient and must not print a re-login hint
// for a refresh token that is still valid.
@@ -184,6 +190,7 @@ where
return Err(LlmError::Provider {
status: status.as_u16(),
body: format!("{failure_label}: {body}"),
+ retry_after,
});
}
return Err(LlmError::Auth(format!(
diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rs
index c2ab38f0..1ce10e5d 100644
--- a/crates/ai-memory-mcp/src/server.rs
+++ b/crates/ai-memory-mcp/src/server.rs
@@ -8099,6 +8099,7 @@ mod tests {
StubRerankOutcome::ProviderFail => Err(ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
}),
}
}
@@ -15862,6 +15863,7 @@ mod tests {
Err(ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
}
@@ -15873,6 +15875,7 @@ mod tests {
Err(ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
})
}
}
@@ -21828,6 +21831,7 @@ mod tests {
ai_memory_llm::LlmError::Provider {
status: 400,
body: "SENTINEL_PRIVATE_BODY".into(),
+ retry_after: None,
}
}
diff --git a/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs b/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs
index df2c0d6d..27520456 100644
--- a/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs
+++ b/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs
@@ -55,6 +55,7 @@ fn provider_404() -> LlmError {
LlmError::Provider {
status: 404,
body: "404 page not found".into(),
+ retry_after: None,
}
}
diff --git a/crates/ai-memory-web/tests/suite/health.rs b/crates/ai-memory-web/tests/suite/health.rs
index 183a24bb..1de70661 100644
--- a/crates/ai-memory-web/tests/suite/health.rs
+++ b/crates/ai-memory-web/tests/suite/health.rs
@@ -49,6 +49,7 @@ impl LlmProvider for FailingLlm {
Err(LlmError::Provider {
status: 401,
body: format!("invalid api key {PLANTED_PROVIDER}"),
+ retry_after: None,
})
}
async fn complete_structured_raw(
@@ -59,6 +60,7 @@ impl LlmProvider for FailingLlm {
Err(LlmError::Provider {
status: 401,
body: PLANTED_PROVIDER.into(),
+ retry_after: None,
})
}
}
@@ -80,6 +82,7 @@ impl Embedder for FailingEmbedder {
Err(LlmError::Provider {
status: 503,
body: format!("upstream said {PLANTED_EMBED}"),
+ retry_after: None,
})
}
}
diff --git a/docs/llm-provider-fallback.md b/docs/llm-provider-fallback.md
index 108e9987..d2ae0cde 100644
--- a/docs/llm-provider-fallback.md
+++ b/docs/llm-provider-fallback.md
@@ -118,12 +118,14 @@ Copilot token exchange and the `openai-oauth`/OIDC refresh. A 429 or 5xx from
those token endpoints is transient, because the grant was not rejected; any
other non-2xx stays an auth failure with its re-login hint.
-Before a candidate is called, the wrapper checks an in-memory circuit keyed by
-`(provider, model)`. A transient error opens that candidate's circuit for a
-bounded cooldown; other candidates remain eligible. A successful response
-closes that candidate's circuit. Restarting the server clears circuits. The
-initial implementation uses no durable circuit state and no separate forced
-request deadline.
+Before a candidate is called, the wrapper checks that chain entry's in-memory
+circuit. Two entries with the same provider and model but different credentials
+therefore rotate independently. A transient error opens only that entry's
+circuit. The default cooldown is 30 seconds; a delta-seconds `Retry-After`
+header or Google `RetryInfo.retryDelay` response detail extends it, clamped to
+one hour. Other candidates remain eligible. A successful response closes that
+candidate's circuit. Restarting the server clears circuits. The implementation
+uses no durable circuit state and no separate forced request deadline.
If every eligible candidate fails, the wrapper returns a bounded aggregate
error containing provider/model labels and error classes, never response bodies