Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(llm): honor provider retry hints
AkitaOnRailsa736b4b74eb9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex aa58c11c..1a490594 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -13,6 +13,12 @@ version in [`docs/changelog/`](docs/changelog/):### Fixed+- Fallback-provider circuits now honor bounded retry hints instead of retrying+ a quota-exhausted candidate every 30 seconds. HTTP `Retry-After` takes+ precedence; Gemini's `google.rpc.RetryInfo.retryDelay` is read from the full+ scrubbed error body before its diagnostic text is cut to 1 KiB. Hints are+ clamped to 30 seconds through one hour, remain scoped to one chain entry, and+ never enlarge the logged provider body (#1619).- Fixed the PowerShell Docker wrapper leaving native managed commands withoutthe root bearer stored in a named volume. Native `install-hooks` now refreshesa host copy under the user's local app-data directory once, and `run`,diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rsindex 78d99b62..b82d1a91 100644--- a/crates/ai-memory-cli/src/commands/llm_test.rs+++ b/crates/ai-memory-cli/src/commands/llm_test.rs@@ -150,6 +150,7 @@ mod tests {LlmError::Provider {status: 429,body: "rate_limit_error".into(),+ retry_after: None,},);assert!(format!("{err:#}").contains("not a real rate limit"));@@ -166,6 +167,7 @@ mod tests {LlmError::Provider {status,body: String::new(),+ retry_after: None,},);assert!(!format!("{err:#}").contains("not a real rate limit"));diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex ab9cafe3..8bf886b7 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -4951,6 +4951,7 @@ mod tests {Err(ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})})}@@ -4968,6 +4969,7 @@ mod tests {Err(ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})})}@@ -5053,10 +5055,12 @@ mod tests {ConsolidationFailure::MalformedRequest => LlmError::Provider {status: 400,body: "bad request".into(),+ retry_after: None,},ConsolidationFailure::RateLimited => LlmError::Provider {status: 429,body: "rate limited".into(),+ retry_after: None,},};Err(error)@@ -5095,10 +5099,12 @@ mod tests {ConsolidatorError::Llm(LlmError::Provider {status: 401,body: "invalid api key".into(),+ retry_after: None,}),ConsolidatorError::Llm(LlmError::Provider {status: 400,body: "bad request".into(),+ retry_after: None,}),];for error in &terminal {@@ -5127,14 +5133,17 @@ mod tests {ConsolidatorError::Llm(LlmError::Provider {status: 429,body: "rate limited".into(),+ retry_after: None,}),ConsolidatorError::Llm(LlmError::Provider {status: 503,body: "capacity".into(),+ retry_after: None,}),ConsolidatorError::Llm(LlmError::Provider {status: 502,body: "bad gateway".into(),+ retry_after: None,}),ConsolidatorError::Llm(LlmError::Http(connect)),];diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rsindex 829e11b7..4c815ae0 100644--- a/crates/ai-memory-consolidate/src/auto_improve.rs+++ b/crates/ai-memory-consolidate/src/auto_improve.rs@@ -2308,6 +2308,7 @@ mod tests {let error = AutoImproveError::Llm(LlmError::Provider {status: 400,body: AUTO_IMPROVE_REDACTION_SENTINEL.into(),+ retry_after: None,});let summary = redacted_auto_improve_summary(&error);assert_eq!(summary, "auto-improve failed: class=provider status=400");diff --git a/crates/ai-memory-consolidate/src/auto_improve_schedule.rs b/crates/ai-memory-consolidate/src/auto_improve_schedule.rsindex 387d34b1..ca7e2a29 100644--- a/crates/ai-memory-consolidate/src/auto_improve_schedule.rs+++ b/crates/ai-memory-consolidate/src/auto_improve_schedule.rs@@ -731,6 +731,7 @@ mod tests {Err(LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})})}@@ -750,6 +751,7 @@ mod tests {Err(LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})})}@@ -798,6 +800,7 @@ mod tests {Err(LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})})}@@ -1072,6 +1075,7 @@ mod tests {let provider_400 = anyhow::Error::new(AutoImproveError::Llm(LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,}));assert_eq!(redacted_scheduler_error_summary(&provider_400),@@ -1082,6 +1086,7 @@ mod tests {let bare = anyhow::Error::new(LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,});assert_eq!(redacted_scheduler_error_summary(&bare),diff --git a/crates/ai-memory-consolidate/src/bootstrap.rs b/crates/ai-memory-consolidate/src/bootstrap.rsindex aaed0e9c..9875fa54 100644--- a/crates/ai-memory-consolidate/src/bootstrap.rs+++ b/crates/ai-memory-consolidate/src/bootstrap.rs@@ -1731,6 +1731,7 @@ mod tests {return Err(LlmError::Provider {status: 503,body: "busy".into(),+ retry_after: None,});}Ok(serde_json::json!({ "pages": [], "rationale": "ok" }))diff --git a/crates/ai-memory-consolidate/src/consolidator.rs b/crates/ai-memory-consolidate/src/consolidator.rsindex abed2a44..e2129ce5 100644--- a/crates/ai-memory-consolidate/src/consolidator.rs+++ b/crates/ai-memory-consolidate/src/consolidator.rs@@ -2314,6 +2314,7 @@ mod tests {let error = ConsolidatorError::Llm(LlmError::Provider {status: 400,body: REDACTION_SENTINEL.into(),+ retry_after: None,});let summary = redacted_error_summary(&error);assert_eq!(summary, "consolidation failed: class=provider status=400");@@ -2328,6 +2329,7 @@ mod tests {let provider = ConsolidatorError::Llm(LlmError::Provider {status: 400,body: REDACTION_SENTINEL.into(),+ retry_after: None,});let serde_error = ConsolidatorError::Serde(REDACTION_SENTINEL.into());let provider_summary = redacted_error_summary(&provider);@@ -4670,6 +4672,7 @@ mod tests {ScriptedFailure::Transient => LlmError::Provider {status: 503,body: "This model is currently experiencing high demand.".into(),+ retry_after: None,},ScriptedFailure::Deterministic => LlmError::Auth("expired".into()),});diff --git a/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs b/crates/ai-memory-consolidate/tests/suite/embed_backfill.rsindex 3864e4d8..82148777 100644--- a/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs+++ b/crates/ai-memory-consolidate/tests/suite/embed_backfill.rs@@ -198,6 +198,7 @@ impl Embedder for ConfigurableEmbedder {return Err(ai_memory_llm::LlmError::Provider {status: 500,body: "synthetic configured failure".into(),+ retry_after: None,});}Ok(vec![0.5_f32; self.dim as usize])diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rsindex 25c6afce..c51af6a3 100644--- a/crates/ai-memory-llm/src/anthropic.rs+++ b/crates/ai-memory-llm/src/anthropic.rs@@ -12,7 +12,7 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::openai_compat::{first_json_object, strip_reasoning_blocks};use crate::provider::LlmProvider;-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, ReasoningEffort, Usage,};@@ -448,13 +448,8 @@ impl AnthropicProvider {);}let resp = headers.apply(builder).json(body).send().await?;- let status = resp.status();- if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ if !resp.status().is_success() {+ return Err(provider_error(resp).await);}response_json_limited::<R>(resp).await}diff --git a/crates/ai-memory-llm/src/codex_responses.rs b/crates/ai-memory-llm/src/codex_responses.rsindex 3b8504c0..2c7a8611 100644--- a/crates/ai-memory-llm/src/codex_responses.rs+++ b/crates/ai-memory-llm/src/codex_responses.rs@@ -7,7 +7,7 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::openai_oauth::{CodexResponsesRequest, CodexResponsesResponse, parse_sse_response};-use crate::response::{provider_error_body, response_json_limited, response_text_limited};+use crate::response::{provider_error, response_json_limited, response_text_limited};use crate::types::ExtraHeaders;/// ChatGPT/Codex Responses backend.@@ -49,12 +49,8 @@ pub(crate) async fn post_codex_responses(request = request.header("chatgpt-account-id", account_id);}let response = request.send().await.map_err(LlmError::from)?;- let status = response.status();- if !status.is_success() {- return Err(LlmError::Provider {- status: status.as_u16(),- body: provider_error_body(response).await,- });+ if !response.status().is_success() {+ return Err(provider_error(response).await);}if body.stream {parse_sse_response(&response_text_limited(response).await?, body.model)diff --git a/crates/ai-memory-llm/src/copilot.rs b/crates/ai-memory-llm/src/copilot.rsindex c41335fc..a58f901c 100644--- a/crates/ai-memory-llm/src/copilot.rs+++ b/crates/ai-memory-llm/src/copilot.rs@@ -23,7 +23,7 @@ use crate::error::{LlmError, LlmResult};use crate::openai::{STRUCTURED_OUTPUT_SCHEMA_NAME, enforce_strict_object_schemas};use crate::openai_oauth::model_uses_default_temperature;use crate::provider::LlmProvider;-use crate::response::{provider_error_body, response_json_limited, response_text_limited};+use crate::response::{provider_error, response_json_limited, response_text_limited};use crate::text::truncate_for_embedding;use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, Usage};@@ -391,13 +391,8 @@ impl CopilotProvider {.headers(copilot_runtime_headers()),);let resp = request.json(body).send().await.map_err(LlmError::from)?;- let status = resp.status();- if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ if !resp.status().is_success() {+ return Err(provider_error(resp).await);}response_json_limited::<CopilotChatResponse>(resp).await}@@ -418,12 +413,8 @@ impl CopilotProvider {.headers(copilot_runtime_headers()),);let resp = request.json(body).send().await.map_err(LlmError::from)?;- let status = resp.status();- if !status.is_success() {- return Err(LlmError::Provider {- status: status.as_u16(),- body: provider_error_body(resp).await,- });+ if !resp.status().is_success() {+ return Err(provider_error(resp).await);}response_json_limited::<CopilotResponsesResponse>(resp).await}@@ -617,18 +608,10 @@ impl Embedder for CopilotEmbedder {}// Client errors (e.g. input too long) are not retried.if status.as_u16() == 400 {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}let body = response_text_limited(resp).await?;let values = parse_openai_embedding_values(&body, status.as_u16())?;@@ -666,7 +649,13 @@ async fn exchange_copilot_token(.map_err(LlmError::from)?;let status = resp.status();if !status.is_success() {- let body = provider_error_body(resp).await;+ let error = provider_error(resp).await;+ let LlmError::Provider {+ body, retry_after, ..+ } = error+ else {+ return Err(error);+ };// GitHub rate-limits this endpoint per account. A 429/5xx here says// nothing about the credential, and `LlmError::Auth` is not transient,// so classifying it as auth stops the fallback chain and tells the@@ -675,6 +664,7 @@ async fn exchange_copilot_token(return Err(LlmError::Provider {status: status.as_u16(),body: format!("copilot token exchange: {body}"),+ retry_after,});}return Err(LlmError::Auth(format!(diff --git a/crates/ai-memory-llm/src/cursor_agent.rs b/crates/ai-memory-llm/src/cursor_agent.rsindex 056b16f9..e3e26f68 100644--- a/crates/ai-memory-llm/src/cursor_agent.rs+++ b/crates/ai-memory-llm/src/cursor_agent.rs@@ -215,6 +215,7 @@ impl CursorAgentProvider {.map_err(|_| LlmError::Provider {status: 504,body: "cursor agent timed out".into(),+ retry_after: None,})??;let (status, stdout, stderr) = collected;@@ -224,6 +225,7 @@ impl CursorAgentProvider {return Err(LlmError::Provider {status: u16::try_from(code).unwrap_or(1),body: truncate_chars(&body, 500),+ retry_after: None,});}let text = String::from_utf8_lossy(&stdout).trim().to_string();diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rsindex 608d8485..babf2816 100644--- a/crates/ai-memory-llm/src/embedding.rs+++ b/crates/ai-memory-llm/src/embedding.rs@@ -23,7 +23,7 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::openai::normalize_openai_base;-use crate::response::{provider_error_body, response_json_limited, response_text_limited};+use crate::response::{provider_error, response_json_limited, response_text_limited};use crate::text::{truncate_for_embedding, truncate_with_ellipsis};/// Conservative per-request input cap for OpenAI-compatible embedding APIs@@ -222,6 +222,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResullet v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Provider {status,body: truncate_with_ellipsis(&format!("openai embeddings json: {e}; body={body}"), 1024),+ retry_after: None,})?;if let Some(err) = v.get("error") {let msg = err@@ -232,6 +233,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResulreturn Err(LlmError::Provider {status,body: truncate_with_ellipsis(msg, 1024),+ retry_after: None,});}if let Some(data) = v.get("data").and_then(|d| d.as_array()) {@@ -241,6 +243,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul&format!("openai embeddings data[] empty; body={body}"),512,),+ retry_after: None,})?;let emb = first.get("embedding")@@ -251,6 +254,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul&format!("missing embedding array in data[0]; body={body}"),512,),+ retry_after: None,})?;let mut out = Vec::with_capacity(emb.len());for n in emb {@@ -260,6 +264,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul&format!("non-numeric embedding value in data[0]; body={body}"),512,),+ retry_after: None,})?;out.push(f as f32);}@@ -275,6 +280,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul&format!("openai embeddings shape: {e}; body={body}"),1024,),+ retry_after: None,})?;let first = parsed.data.into_iter().next().ok_or_else(|| {LlmError::UnexpectedShape(format!(@@ -386,18 +392,10 @@ async fn openai_style_embed(}// Client errors (e.g. input > 8192 tokens) are not retried.if status.as_u16() == 400 {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}let body = response_text_limited(resp).await?;let values = parse_openai_embedding_values(&body, status.as_u16())?;@@ -713,11 +711,7 @@ impl Embedder for VoyageEmbedder {};let status = resp.status();if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}let parsed: VoyageResponse = response_json_limited(resp).await?;let first =diff --git a/crates/ai-memory-llm/src/error.rs b/crates/ai-memory-llm/src/error.rsindex 2dc3b90a..66e0acd6 100644--- a/crates/ai-memory-llm/src/error.rs+++ b/crates/ai-memory-llm/src/error.rs@@ -1,5 +1,7 @@//! LLM error type.+use std::time::Duration;+use thiserror::Error;/// Result alias used throughout the LLM crate.@@ -20,6 +22,8 @@ pub enum LlmError {status: u16,/// Response body (truncated).body: String,+ /// Provider-requested delay before another equivalent request.+ retry_after: Option<Duration>,},/// JSON (de)serialization failure.@@ -129,6 +133,16 @@ impl LlmError {}}+ /// Provider-requested delay before another equivalent request, when the+ /// failed HTTP response carried a trustworthy, boundedly parsed hint.+ #[must_use]+ pub const fn retry_after(&self) -> Option<Duration> {+ match self {+ Self::Provider { retry_after, .. } => *retry_after,+ _ => None,+ }+ }+/// Redacted error category used for health reporting and the fallback/// chain's aggregate error — never a response body or secret.#[must_use]@@ -164,7 +178,8 @@ mod tests {assert!(LlmError::Provider {status: 429,- body: String::new()+ body: String::new(),+ retry_after: None,}.is_transient());@@ -172,7 +187,8 @@ mod tests {assert!(LlmError::Provider {status,- body: String::new()+ body: String::new(),+ retry_after: None,}.is_transient(),"status {status} should be transient"@@ -183,7 +199,8 @@ mod tests {assert!(!LlmError::Provider {status,- body: String::new()+ body: String::new(),+ retry_after: None,}.is_transient(),"status {status} must not be treated as transient"@@ -255,7 +272,8 @@ mod tests {assert_eq!(LlmError::Provider {status: 503,- body: "boom".into()+ body: "boom".into(),+ retry_after: None,}.http_status(),Some(503)@@ -270,6 +288,7 @@ mod tests {LlmError::Provider {status: 500,body: "secret upstream body".into(),+ retry_after: None,},"provider",),@@ -314,7 +333,8 @@ mod tests {assert!(LlmError::Provider {status,- body: String::new()+ body: String::new(),+ retry_after: None,}.is_transient(),"{status} must agree with is_transient"@@ -325,7 +345,8 @@ mod tests {assert!(!LlmError::Provider {status,- body: String::new()+ body: String::new(),+ retry_after: None,}.is_transient(),"{status} must agree with is_transient"diff --git a/crates/ai-memory-llm/src/fallback.rs b/crates/ai-memory-llm/src/fallback.rsindex 4186d6d1..a433781a 100644--- a/crates/ai-memory-llm/src/fallback.rs+++ b/crates/ai-memory-llm/src/fallback.rs@@ -26,6 +26,7 @@ use crate::types::{ChatRequest, ChatResponse, LlmOperationId};/// before it is eligible again. Restarting the process clears all circuit/// state; there is no durable or cross-process cooldown.pub const CIRCUIT_COOLDOWN: Duration = Duration::from_secs(30);+const MAX_CIRCUIT_COOLDOWN: Duration = Duration::from_secs(60 * 60);/// One provider in an ordered fallback chain.///@@ -129,7 +130,12 @@ impl FallbackLlmProvider {state.last_error_status = err.http_status();state.last_error_class = Some(err.class());if err.is_transient() {- state.circuit_open_until = Some(Instant::now() + CIRCUIT_COOLDOWN);+ let cooldown = err+ .retry_after()+ .unwrap_or(CIRCUIT_COOLDOWN)+ .max(CIRCUIT_COOLDOWN)+ .min(MAX_CIRCUIT_COOLDOWN);+ state.circuit_open_until = Some(Instant::now() + cooldown);}}@@ -416,9 +422,14 @@ mod tests {fn clone_err(err: &LlmError) -> LlmError {match err {- LlmError::Provider { status, body } => LlmError::Provider {+ LlmError::Provider {+ status,+ body,+ retry_after,+ } => LlmError::Provider {status: *status,body: body.clone(),+ retry_after: *retry_after,},LlmError::Serde(msg) => LlmError::Serde(msg.clone()),LlmError::UnexpectedShape(msg) => LlmError::UnexpectedShape(msg.clone()),@@ -481,6 +492,7 @@ mod tests {LlmError::Provider {status,body: "upstream body that must never leak".to_string(),+ retry_after: None,}}@@ -569,22 +581,27 @@ mod tests {LlmError::Provider {status: 400,body: String::new(),+ retry_after: None,},LlmError::Provider {status: 401,body: String::new(),+ retry_after: None,},LlmError::Provider {status: 403,body: String::new(),+ retry_after: None,},LlmError::Provider {status: 404,body: String::new(),+ retry_after: None,},LlmError::Provider {status: 422,body: String::new(),+ retry_after: None,},LlmError::Schema("bad schema".into()),LlmError::Serde("deserialize failed".into()),@@ -649,6 +666,40 @@ mod tests {LlmError::Provider {status,body: body.to_string(),+ retry_after: None,+ }+ }++ #[test]+ fn provider_retry_hints_are_clamped_to_the_circuit_bounds() {+ let chain = FallbackLlmProvider::new(vec![Candidate::new(+ "primary",+ "m1",+ Arc::new(ScriptedLlm::ok("primary", "m1", "unused")),+ )]);++ for (hint, expected) in [(5, 30), (90, 90), (7_200, 3_600)] {+ let before = Instant::now();+ chain.record_error(+ 0,+ &LlmError::Provider {+ status: 429,+ body: String::new(),+ retry_after: Some(Duration::from_secs(hint)),+ },+ );+ let until = chain.entries[0]+ .state+ .lock()+ .unwrap()+ .circuit_open_until+ .unwrap();+ let actual = until.duration_since(before);+ assert!(+ actual >= Duration::from_secs(expected)+ && actual < Duration::from_secs(expected + 1),+ "hint {hint}s produced {actual:?}"+ );}}diff --git a/crates/ai-memory-llm/src/gemini.rs b/crates/ai-memory-llm/src/gemini.rsindex cfe625ed..49dd0439 100644--- a/crates/ai-memory-llm/src/gemini.rs+++ b/crates/ai-memory-llm/src/gemini.rs@@ -18,7 +18,7 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::provider::LlmProvider;-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, Role, Usage};/// Default Gemini API base.@@ -253,13 +253,8 @@ impl GeminiProvider {.header("content-type", "application/json"),);let resp = request.json(body).send().await?;- let status = resp.status();- if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ if !resp.status().is_success() {+ return Err(provider_error(resp).await);}response_json_limited::<R>(resp).await}diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rsindex c9133844..ca50465b 100644--- a/crates/ai-memory-llm/src/google.rs+++ b/crates/ai-memory-llm/src/google.rs@@ -11,7 +11,7 @@ use tracing::debug;use crate::embedding::{Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, retry_delay};use crate::error::{LlmError, LlmResult};-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};use crate::text::truncate_for_embedding;/// Default Gemini API host.@@ -118,11 +118,7 @@ impl GoogleEmbedder {continue;}if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ return Err(provider_error(resp).await);}let parsed: GeminiEmbedResponse = response_json_limited(resp).await?;let values = parsed.embedding.values;diff --git a/crates/ai-memory-llm/src/health.rs b/crates/ai-memory-llm/src/health.rsindex 17b9527c..5642bf3d 100644--- a/crates/ai-memory-llm/src/health.rs+++ b/crates/ai-memory-llm/src/health.rs@@ -755,7 +755,7 @@ fn error_status_and_message(err: &LlmError) -> (Option<u16>, String) {e.status().map(|status| status.as_u16()),truncate_error_message(&e.to_string()),),- LlmError::Provider { status, body } => (Some(*status), truncate_error_message(body)),+ LlmError::Provider { status, body, .. } => (Some(*status), truncate_error_message(body)),_ => (None, truncate_error_message(&err.to_string())),}}@@ -793,6 +793,7 @@ mod tests {return Err(LlmError::Provider {status: 401,body: "bad token".to_string(),+ retry_after: None,});}Ok(ChatResponse {@@ -1022,6 +1023,7 @@ mod tests {let failure: LlmResult<ChatResponse> = Err(LlmError::Provider {status: 429,body: "secret upstream body".into(),+ retry_after: None,});workloads.record(LlmWorkload::Rerank,diff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rsindex 1bb2fb92..47c652c6 100644--- a/crates/ai-memory-llm/src/lib.rs+++ b/crates/ai-memory-llm/src/lib.rs@@ -154,7 +154,7 @@ pub use opencode::{};pub use provider::{LlmProvider, complete_structured, complete_structured_with_operation_id};pub use reranker::{LlmReranker, RerankCandidate, RerankScore, RerankStage, Reranker};-pub use response::provider_error_body;+pub use response::{provider_error, provider_error_body};pub use stored_token::StoredOAuthToken;pub use types::{ChatMessage, ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, LlmWorkload,diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rsindex 2e9ed387..2b867795 100644--- a/crates/ai-memory-llm/src/openai.rs+++ b/crates/ai-memory-llm/src/openai.rs@@ -10,7 +10,7 @@ use tracing::debug;use crate::error::{LlmError, LlmResult};use crate::provider::LlmProvider;-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};use reqwest::header::{HeaderName, HeaderValue};use crate::types::{@@ -635,13 +635,8 @@ impl OpenAiProvider {}};let resp = request.json(body).send().await?;- let status = resp.status();- if !status.is_success() {- let body = provider_error_body(resp).await;- return Err(LlmError::Provider {- status: status.as_u16(),- body,- });+ if !resp.status().is_success() {+ return Err(provider_error(resp).await);}response_json_limited::<OpenAiResponse>(resp).await}diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rsindex 8b600262..cf5791ba 100644--- a/crates/ai-memory-llm/src/openai_compat.rs+++ b/crates/ai-memory-llm/src/openai_compat.rs@@ -383,7 +383,7 @@ fn is_parse_shape_error(err: &LlmError) -> bool {/// before generation. Generic 4xx errors are deliberately excluded so auth,/// quota, model, and malformed-request failures retain their original cause.fn is_response_format_rejection(err: &LlmError) -> bool {- let LlmError::Provider { status, body } = err else {+ let LlmError::Provider { status, body, .. } = err else {return false;};if !matches!(status, 400 | 422) {@@ -655,18 +655,22 @@ mod tests {assert!(is_response_format_rejection(&LlmError::Provider {status: 400,body: "unsupported parameter: response_format".into(),+ retry_after: None,}));assert!(is_response_format_rejection(&LlmError::Provider {status: 422,body: "json_schema is not supported".into(),+ retry_after: None,}));assert!(!is_response_format_rejection(&LlmError::Provider {status: 400,body: "unknown model".into(),+ retry_after: None,}));assert!(!is_response_format_rejection(&LlmError::Provider {status: 401,body: "response_format requires authentication".into(),+ retry_after: None,}));}@@ -684,15 +688,18 @@ mod tests {// Transport / HTTP / auth → propagate.assert!(!is_parse_shape_error(&LlmError::Provider {status: 500,- body: "engine boom".into()+ body: "engine boom".into(),+ retry_after: None,}));assert!(!is_parse_shape_error(&LlmError::Provider {status: 401,- body: "unauthorized".into()+ body: "unauthorized".into(),+ retry_after: None,}));assert!(!is_parse_shape_error(&LlmError::Provider {status: 429,- body: "rate limited".into()+ body: "rate limited".into(),+ retry_after: None,}));}diff --git a/crates/ai-memory-llm/src/openai_oauth.rs b/crates/ai-memory-llm/src/openai_oauth.rsindex 17c1ff13..4a7fbd62 100644--- a/crates/ai-memory-llm/src/openai_oauth.rs+++ b/crates/ai-memory-llm/src/openai_oauth.rs@@ -366,6 +366,7 @@ pub(crate) fn parse_sse_response(body: &str, model: &str) -> LlmResult<CodexRespreturn Err(LlmError::Provider {status: SSE_TERMINAL_ERROR_STATUS,body: truncate_with_ellipsis(trimmed, SSE_ERROR_BODY_TRIM),+ retry_after: None,});}let kind = value@@ -408,6 +409,7 @@ pub(crate) fn parse_sse_response(body: &str, model: &str) -> LlmResult<CodexRespreturn Err(LlmError::Provider {status: SSE_TERMINAL_ERROR_STATUS,body: truncate_with_ellipsis(trimmed, SSE_ERROR_BODY_TRIM),+ retry_after: None,});}_ => {}@@ -828,7 +830,7 @@ mod tests {let err = parse_sse_response(&body, "gpt-5.5").expect_err("terminal event must fail");match err {- LlmError::Provider { status, body } => {+ LlmError::Provider { status, body, .. } => {assert_eq!(status, 502);assert!(body.contains(event), "body should include event: {body}");assert!(@@ -886,7 +888,7 @@ mod tests {let err = parse_sse_response(body, "gpt-5.5").expect_err("terminal event must fail");match err {- LlmError::Provider { status, body } => {+ LlmError::Provider { status, body, .. } => {assert_eq!(status, SSE_TERMINAL_ERROR_STATUS);assert!(body.contains("content_filter"),@@ -925,7 +927,7 @@ mod tests {let err = parse_sse_response(body, "gpt-5.5").expect_err("top-level error payload must surface as a terminal error");match err {- LlmError::Provider { status, body } => {+ LlmError::Provider { status, body, .. } => {assert_eq!(status, super::SSE_TERMINAL_ERROR_STATUS);assert!(body.contains("context length exceeded"),diff --git a/crates/ai-memory-llm/src/opencode.rs b/crates/ai-memory-llm/src/opencode.rsindex 8482af68..71c6e3a5 100644--- a/crates/ai-memory-llm/src/opencode.rs+++ b/crates/ai-memory-llm/src/opencode.rs@@ -22,7 +22,7 @@ use crate::openai::{};use crate::openai_compat::OpenAiCompatProvider;use crate::provider::LlmProvider;-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};use crate::types::{ChatRequest, ChatResponse, ExtraHeaders, LlmOperationId, ReasoningEffort, Usage,};@@ -470,12 +470,8 @@ impl OpenCodeResponsesProvider {headers.set_default(name, value);}let response = headers.apply(builder).send().await?;- let status = response.status();- if !status.is_success() {- return Err(LlmError::Provider {- status: status.as_u16(),- body: provider_error_body(response).await,- });+ if !response.status().is_success() {+ return Err(provider_error(response).await);}response_json_limited(response).await}@@ -1051,7 +1047,7 @@ mod tests {.unwrap_err(),] {assert!(- matches!(error, LlmError::Provider { status: code, body } if code == status && body == "gateway error")+ matches!(error, LlmError::Provider { status: code, body, .. } if code == status && body == "gateway error"));}assert_eq!(server.received_requests().await.unwrap().len(), 2);diff --git a/crates/ai-memory-llm/src/response.rs b/crates/ai-memory-llm/src/response.rsindex 3b144e6d..5a46fbbf 100644--- a/crates/ai-memory-llm/src/response.rs+++ b/crates/ai-memory-llm/src/response.rs@@ -1,3 +1,5 @@+use std::time::Duration;+use ai_memory_core::Sanitizer;use serde::de::DeserializeOwned;@@ -22,6 +24,7 @@ pub(crate) async fn response_bytes_limited(return Err(LlmError::Provider {status,body: format!("provider response exceeded {max_bytes} bytes"),+ retry_after: None,});}bytes.extend_from_slice(&chunk);@@ -50,14 +53,97 @@ pub(crate) async fn response_json_limited<T: DeserializeOwned>(/// Scrubbing runs before the cut so a secret straddling the 1 KiB boundary/// is still redacted rather than left as an unrecognisable prefix.pub async fn provider_error_body(resp: reqwest::Response) -> String {+ match provider_error(resp).await {+ LlmError::Provider { body, .. } => body,+ error => format!("<failed to read provider error body: {error}>"),+ }+}++/// Build a typed provider error while preserving a retry hint that may live+/// beyond the bounded display prefix. The body is still read at most 1 MiB,+/// scrubbed before parsing, and exposed at most 1 KiB.+pub async fn provider_error(resp: reqwest::Response) -> LlmError {+ let status = resp.status().as_u16();+ let header_retry = resp+ .headers()+ .get(reqwest::header::RETRY_AFTER)+ .and_then(|value| value.to_str().ok())+ .and_then(parse_delta_seconds);match response_bytes_limited(resp, MAX_PROVIDER_ERROR_BYTES).await {Ok(bytes) => {- let body = Sanitizer::builtin().scrub(&String::from_utf8_lossy(&bytes));- truncate_with_ellipsis(&body, DISPLAY_ERROR_BYTES)+ let scrubbed = Sanitizer::builtin().scrub(&String::from_utf8_lossy(&bytes));+ let retry_after = header_retry.or_else(|| google_retry_delay(&scrubbed));+ LlmError::Provider {+ status,+ body: truncate_with_ellipsis(&scrubbed, DISPLAY_ERROR_BYTES),+ retry_after,+ }+ }+ Err(LlmError::Provider { body, .. }) => LlmError::Provider {+ status,+ body,+ retry_after: header_retry,+ },+ Err(error) => LlmError::Provider {+ status,+ body: format!("<failed to read provider error body: {error}>"),+ retry_after: header_retry,+ },+ }+}++fn parse_delta_seconds(raw: &str) -> Option<Duration> {+ raw.trim().parse::<u64>().ok().map(Duration::from_secs)+}++fn google_retry_delay(body: &str) -> Option<Duration> {+ fn visit(value: &serde_json::Value) -> Option<Duration> {+ match value {+ serde_json::Value::Object(object) => {+ let is_retry = object+ .get("@type")+ .and_then(serde_json::Value::as_str)+ .is_some_and(|kind| {+ matches!(+ kind,+ "google.rpc.RetryInfo" | "type.googleapis.com/google.rpc.RetryInfo"+ )+ });+ if is_retry+ && let Some(delay) = object+ .get("retryDelay")+ .and_then(serde_json::Value::as_str)+ .and_then(parse_google_duration)+ {+ return Some(delay);+ }+ object.values().find_map(visit)+ }+ serde_json::Value::Array(values) => values.iter().find_map(visit),+ _ => None,}- Err(LlmError::Provider { body, .. }) => body,- Err(e) => format!("<failed to read provider error body: {e}>"),}++ serde_json::from_str(body)+ .ok()+ .and_then(|value| visit(&value))+}++fn parse_google_duration(raw: &str) -> Option<Duration> {+ let seconds = raw.strip_suffix('s')?;+ let (whole, fraction) = seconds.split_once('.').unwrap_or((seconds, ""));+ if whole.starts_with('-') || !fraction.bytes().all(|byte| byte.is_ascii_digit()) {+ return None;+ }+ let whole = whole.parse::<u64>().ok()?;+ let nanos = if fraction.is_empty() {+ 0+ } else {+ let kept = &fraction[..fraction.len().min(9)];+ let value = kept.parse::<u32>().ok()?;+ value.checked_mul(10_u32.pow(9 - u32::try_from(kept.len()).ok()?))?+ };+ Some(Duration::new(whole, nanos))}#[cfg(test)]@@ -65,7 +151,11 @@ mod tests {use super::*;/// Answer one request on a loopback port with `status` and `body`.- async fn serve_once(status: &'static str, body: String) -> String {+ async fn serve_once_with_headers(+ status: &'static str,+ headers: &'static str,+ body: String,+ ) -> String {use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();@@ -77,7 +167,7 @@ mod tests {let mut buf = [0_u8; 4096];let _ = stream.read(&mut buf).await;let head = format!(- "HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n",body.len());let _ = stream.write_all(head.as_bytes()).await;@@ -86,12 +176,22 @@ mod tests {format!("http://{addr}/")}+ async fn serve_once(status: &'static str, body: String) -> String {+ serve_once_with_headers(status, "", body).await+ }+async fn error_body_for(body: String) -> String {let url = serve_once("400 Bad Request", body).await;let resp = reqwest::Client::new().get(url).send().await.unwrap();provider_error_body(resp).await}+ async fn provider_error_for(headers: &'static str, body: String) -> LlmError {+ let url = serve_once_with_headers("429 Too Many Requests", headers, body).await;+ let resp = reqwest::Client::new().get(url).send().await.unwrap();+ provider_error(resp).await+ }+#[tokio::test]async fn provider_error_body_redacts_a_secret_straddling_the_display_cut() {// The cut keeps `ghp_` plus 12 characters: too short for the GitHub@@ -126,4 +226,72 @@ mod tests {let oversized = error_body_for("a".repeat(MAX_PROVIDER_ERROR_BYTES + 1)).await;assert!(oversized.contains("exceeded"), "unexpected: {oversized}");}++ #[tokio::test]+ async fn provider_error_reads_gemini_retry_info_before_the_display_cut() {+ let body = serde_json::json!({+ "error": {+ "message": "x".repeat(DISPLAY_ERROR_BYTES * 2),+ "details": [+ {"@type": "type.googleapis.com/google.rpc.QuotaFailure"},+ {+ "@type": "type.googleapis.com/google.rpc.RetryInfo",+ "retryDelay": "32993s"+ }+ ]+ }+ })+ .to_string();++ let error = provider_error_for("", body).await;++ assert_eq!(error.retry_after(), Some(Duration::from_secs(32_993)));+ let LlmError::Provider { body, .. } = error else {+ panic!("expected provider error");+ };+ assert!(body.len() <= DISPLAY_ERROR_BYTES + "…".len());+ assert!(+ !body.contains("RetryInfo"),+ "the control must be beyond the cut"+ );+ }++ #[tokio::test]+ async fn provider_error_finds_nested_compat_retry_info_and_parses_fractional_seconds() {+ let body = serde_json::json!({+ "error": {+ "details": [[{+ "@type": "type.googleapis.com/google.rpc.RetryInfo",+ "retryDelay": "12.345s"+ }]]+ }+ })+ .to_string();++ let error = provider_error_for("", body).await;++ assert_eq!(error.retry_after(), Some(Duration::new(12, 345_000_000)));+ }++ #[tokio::test]+ async fn retry_after_header_precedes_the_body_and_invalid_hints_are_ignored() {+ let body = serde_json::json!({+ "details": [{+ "@type": "type.googleapis.com/google.rpc.RetryInfo",+ "retryDelay": "10s"+ }]+ })+ .to_string();+ let error = provider_error_for("Retry-After: 42\r\n", body).await;+ assert_eq!(error.retry_after(), Some(Duration::from_secs(42)));++ for raw in ["-1s", "NaNs", "1.second"] {+ let body = serde_json::json!({+ "@type": "type.googleapis.com/google.rpc.RetryInfo",+ "retryDelay": raw+ })+ .to_string();+ assert_eq!(provider_error_for("", body).await.retry_after(), None);+ }+ }}diff --git a/crates/ai-memory-llm/src/stored_token.rs b/crates/ai-memory-llm/src/stored_token.rsindex cfc8afc7..5f620987 100644--- a/crates/ai-memory-llm/src/stored_token.rs+++ b/crates/ai-memory-llm/src/stored_token.rs@@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};use crate::auth_file::{load_entry, now_ms, save_entry};use crate::error::{LlmError, LlmResult};-use crate::response::{provider_error_body, response_json_limited};+use crate::response::{provider_error, response_json_limited};/// A credential-bearing HTTP endpoint validated as TLS-protected, except for/// explicit loopback development servers. Keeping the parsed URL behind this@@ -176,7 +176,13 @@ where.map_err(LlmError::from)?;let status = resp.status();if !status.is_success() {- let body = provider_error_body(resp).await;+ let error = provider_error(resp).await;+ let LlmError::Provider {+ body, retry_after, ..+ } = error+ else {+ return Err(error);+ };// A rate-limited or overloaded token endpoint has not rejected the// grant, so it must stay transient and must not print a re-login hint// for a refresh token that is still valid.@@ -184,6 +190,7 @@ wherereturn Err(LlmError::Provider {status: status.as_u16(),body: format!("{failure_label}: {body}"),+ retry_after,});}return Err(LlmError::Auth(format!(diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rsindex c2ab38f0..1ce10e5d 100644--- a/crates/ai-memory-mcp/src/server.rs+++ b/crates/ai-memory-mcp/src/server.rs@@ -8099,6 +8099,7 @@ mod tests {StubRerankOutcome::ProviderFail => Err(ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,}),}}@@ -15862,6 +15863,7 @@ mod tests {Err(ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})}@@ -15873,6 +15875,7 @@ mod tests {Err(ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,})}}@@ -21828,6 +21831,7 @@ mod tests {ai_memory_llm::LlmError::Provider {status: 400,body: "SENTINEL_PRIVATE_BODY".into(),+ retry_after: None,}}diff --git a/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs b/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rsindex df2c0d6d..27520456 100644--- a/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs+++ b/crates/ai-memory-mcp/tests/suite/admin_provider_error_logging.rs@@ -55,6 +55,7 @@ fn provider_404() -> LlmError {LlmError::Provider {status: 404,body: "404 page not found".into(),+ retry_after: None,}}diff --git a/crates/ai-memory-web/tests/suite/health.rs b/crates/ai-memory-web/tests/suite/health.rsindex 183a24bb..1de70661 100644--- a/crates/ai-memory-web/tests/suite/health.rs+++ b/crates/ai-memory-web/tests/suite/health.rs@@ -49,6 +49,7 @@ impl LlmProvider for FailingLlm {Err(LlmError::Provider {status: 401,body: format!("invalid api key {PLANTED_PROVIDER}"),+ retry_after: None,})}async fn complete_structured_raw(@@ -59,6 +60,7 @@ impl LlmProvider for FailingLlm {Err(LlmError::Provider {status: 401,body: PLANTED_PROVIDER.into(),+ retry_after: None,})}}@@ -80,6 +82,7 @@ impl Embedder for FailingEmbedder {Err(LlmError::Provider {status: 503,body: format!("upstream said {PLANTED_EMBED}"),+ retry_after: None,})}}diff --git a/docs/llm-provider-fallback.md b/docs/llm-provider-fallback.mdindex 108e9987..d2ae0cde 100644--- a/docs/llm-provider-fallback.md+++ b/docs/llm-provider-fallback.md@@ -118,12 +118,14 @@ Copilot token exchange and the `openai-oauth`/OIDC refresh. A 429 or 5xx fromthose token endpoints is transient, because the grant was not rejected; anyother non-2xx stays an auth failure with its re-login hint.-Before a candidate is called, the wrapper checks an in-memory circuit keyed by-`(provider, model)`. A transient error opens that candidate's circuit for a-bounded cooldown; other candidates remain eligible. A successful response-closes that candidate's circuit. Restarting the server clears circuits. The-initial implementation uses no durable circuit state and no separate forced-request deadline.+Before a candidate is called, the wrapper checks that chain entry's in-memory+circuit. Two entries with the same provider and model but different credentials+therefore rotate independently. A transient error opens only that entry's+circuit. The default cooldown is 30 seconds; a delta-seconds `Retry-After`+header or Google `RetryInfo.retryDelay` response detail extends it, clamped to+one hour. Other candidates remain eligible. A successful response closes that+candidate's circuit. Restarting the server clears circuits. The implementation+uses no durable circuit state and no separate forced request deadline.If every eligible candidate fails, the wrapper returns a bounded aggregateerror containing provider/model labels and error classes, never response bodies