Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(embed): apply the 429/503 retry policy to every HTTP embedder
Vinícius DS Andrade9adfbdce10f5
diff --git a/crates/ai-memory-llm/src/copilot.rs b/crates/ai-memory-llm/src/copilot.rsindex ae081c26..1e991876 100644--- a/crates/ai-memory-llm/src/copilot.rs+++ b/crates/ai-memory-llm/src/copilot.rs@@ -17,7 +17,7 @@ use tracing::{debug, info, warn};use crate::auth::CopilotAuth;use crate::auth_file::{load_entry, now_ms, save_entry};use crate::embedding::{- Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, parse_openai_embedding_values,+ Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, parse_openai_embedding_values, retry_delay,};use crate::error::{LlmError, LlmResult};use crate::openai::{STRUCTURED_OUTPUT_SCHEMA_NAME, enforce_strict_object_schemas};@@ -580,6 +580,7 @@ impl Embedder for CopilotEmbedder {model: &self.model,};let mut attempt = 0u32;+ let mut waited = Duration::ZERO;loop {let resp = self.state@@ -593,10 +594,16 @@ impl Embedder for CopilotEmbedder {.await.map_err(LlmError::from)?;let status = resp.status();- if status.as_u16() == 429 && attempt < 5 {+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {attempt += 1;- let delay = Duration::from_secs(2u64.saturating_pow(attempt));- debug!(attempt, ?delay, "copilot embeddings rate-limited; retrying");+ waited += delay;+ debug!(+ attempt,+ ?delay,+ status = status.as_u16(),+ "copilot embeddings 429/503; retrying"+ );tokio::time::sleep(delay).await;continue;}diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rsindex ac8c699a..08d22ceb 100644--- a/crates/ai-memory-llm/src/embedding.rs+++ b/crates/ai-memory-llm/src/embedding.rs@@ -276,7 +276,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResulOk(first.embedding)}-/// Retried statuses for [`openai_style_embed`]: a rate limit and a+/// Retried statuses for every remote embedder's request: a rate limit and a/// transient "out of capacity" answer. Everything else — other 4xx and/// 5xx — is handled exactly as before: no retry.fn is_retryable_embedding_status(status: u16) -> bool {@@ -294,8 +294,8 @@ const MAX_RETRY_AFTER_SECS: u64 = 32;/// write-page embed never wait longer than a 429 storm already could.const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);-/// Decide the wait before the next attempt of [`openai_style_embed`], or-/// give up.+/// Decide the wait before the next attempt of a remote embedding request+/// (every HTTP embedder shares this policy), or give up.////// Only 429 and 503 are retriable ([`is_retryable_embedding_status`])./// `attempt` is the 1-based index of the attempt being decided: up to@@ -308,7 +308,7 @@ const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);/// stop and report the last status: a non-retriable status, more than/// five new attempts, or `waited + delay` exceeding/// [`MAX_RETRY_WAIT_BUDGET`].-fn retry_delay(+pub(crate) fn retry_delay(status: reqwest::StatusCode,retry_after: Option<&reqwest::header::HeaderValue>,attempt: u32,@@ -673,13 +673,31 @@ impl Embedder for VoyageEmbedder {input: [text],model: &self.model,};- let resp = self- .client- .post(&url)- .bearer_auth(self.api_key.expose_secret())- .json(&req)- .send()- .await?;+ let mut attempt = 0u32;+ let mut waited = Duration::ZERO;+ let resp = loop {+ let resp = self+ .client+ .post(&url)+ .bearer_auth(self.api_key.expose_secret())+ .json(&req)+ .send()+ .await?;+ let status = resp.status();+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);+ let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) else {+ break resp;+ };+ attempt += 1;+ waited += delay;+ debug!(+ attempt,+ ?delay,+ status = status.as_u16(),+ "voyage embeddings 429/503; retrying"+ );+ tokio::time::sleep(delay).await;+ };let status = resp.status();if !status.is_success() {let body = provider_error_body(resp).await;diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rsindex 84a1053f..0bdb3e5a 100644--- a/crates/ai-memory-llm/src/google.rs+++ b/crates/ai-memory-llm/src/google.rs@@ -9,7 +9,7 @@ use secrecy::{ExposeSecret, SecretString};use serde::{Deserialize, Serialize};use tracing::debug;-use crate::embedding::{Embedder, normalise};+use crate::embedding::{Embedder, normalise, retry_delay};use crate::error::{LlmError, LlmResult};use crate::response::{provider_error_body, response_json_limited};@@ -86,6 +86,7 @@ impl GoogleEmbedder {debug!(url, model = %self.model, ?task_type, "POST google/embedContent");let mut attempt = 0u32;+ let mut waited = Duration::ZERO;loop {let resp = self.client@@ -95,13 +96,15 @@ impl GoogleEmbedder {.send().await?;let status = resp.status();- if status.as_u16() == 429 && attempt < 5 {+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {attempt += 1;- let delay = Duration::from_secs(2u64.saturating_pow(attempt));+ waited += delay;debug!(attempt,?delay,- "google embedContent rate-limited; retrying"+ status = status.as_u16(),+ "google embedContent 429/503; retrying");tokio::time::sleep(delay).await;continue;