Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(embed): apply the 429/503 retry policy to every HTTP embedder

Vinícius DS Andrade9adfbdce10f5
diff --git a/crates/ai-memory-llm/src/copilot.rs b/crates/ai-memory-llm/src/copilot.rs
index ae081c26..1e991876 100644
--- a/crates/ai-memory-llm/src/copilot.rs
+++ b/crates/ai-memory-llm/src/copilot.rs
@@ -17,7 +17,7 @@ use tracing::{debug, info, warn};
use crate::auth::CopilotAuth;
use crate::auth_file::{load_entry, now_ms, save_entry};
use crate::embedding::{
- Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, parse_openai_embedding_values,
+ Embedder, OPENAI_EMBED_MAX_TOKENS, normalise, parse_openai_embedding_values, retry_delay,
};
use crate::error::{LlmError, LlmResult};
use crate::openai::{STRUCTURED_OUTPUT_SCHEMA_NAME, enforce_strict_object_schemas};
@@ -580,6 +580,7 @@ impl Embedder for CopilotEmbedder {
model: &self.model,
};
let mut attempt = 0u32;
+ let mut waited = Duration::ZERO;
loop {
let resp = self
.state
@@ -593,10 +594,16 @@ impl Embedder for CopilotEmbedder {
.await
.map_err(LlmError::from)?;
let status = resp.status();
- if status.as_u16() == 429 && attempt < 5 {
+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);
+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {
attempt += 1;
- let delay = Duration::from_secs(2u64.saturating_pow(attempt));
- debug!(attempt, ?delay, "copilot embeddings rate-limited; retrying");
+ waited += delay;
+ debug!(
+ attempt,
+ ?delay,
+ status = status.as_u16(),
+ "copilot embeddings 429/503; retrying"
+ );
tokio::time::sleep(delay).await;
continue;
}
diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rs
index ac8c699a..08d22ceb 100644
--- a/crates/ai-memory-llm/src/embedding.rs
+++ b/crates/ai-memory-llm/src/embedding.rs
@@ -276,7 +276,7 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
Ok(first.embedding)
}
-/// Retried statuses for [`openai_style_embed`]: a rate limit and a
+/// Retried statuses for every remote embedder's request: a rate limit and a
/// transient "out of capacity" answer. Everything else — other 4xx and
/// 5xx — is handled exactly as before: no retry.
fn is_retryable_embedding_status(status: u16) -> bool {
@@ -294,8 +294,8 @@ const MAX_RETRY_AFTER_SECS: u64 = 32;
/// write-page embed never wait longer than a 429 storm already could.
const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);
-/// Decide the wait before the next attempt of [`openai_style_embed`], or
-/// give up.
+/// Decide the wait before the next attempt of a remote embedding request
+/// (every HTTP embedder shares this policy), or give up.
///
/// Only 429 and 503 are retriable ([`is_retryable_embedding_status`]).
/// `attempt` is the 1-based index of the attempt being decided: up to
@@ -308,7 +308,7 @@ const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);
/// stop and report the last status: a non-retriable status, more than
/// five new attempts, or `waited + delay` exceeding
/// [`MAX_RETRY_WAIT_BUDGET`].
-fn retry_delay(
+pub(crate) fn retry_delay(
status: reqwest::StatusCode,
retry_after: Option<&reqwest::header::HeaderValue>,
attempt: u32,
@@ -673,13 +673,31 @@ impl Embedder for VoyageEmbedder {
input: [text],
model: &self.model,
};
- let resp = self
- .client
- .post(&url)
- .bearer_auth(self.api_key.expose_secret())
- .json(&req)
- .send()
- .await?;
+ let mut attempt = 0u32;
+ let mut waited = Duration::ZERO;
+ let resp = loop {
+ let resp = self
+ .client
+ .post(&url)
+ .bearer_auth(self.api_key.expose_secret())
+ .json(&req)
+ .send()
+ .await?;
+ let status = resp.status();
+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);
+ let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) else {
+ break resp;
+ };
+ attempt += 1;
+ waited += delay;
+ debug!(
+ attempt,
+ ?delay,
+ status = status.as_u16(),
+ "voyage embeddings 429/503; retrying"
+ );
+ tokio::time::sleep(delay).await;
+ };
let status = resp.status();
if !status.is_success() {
let body = provider_error_body(resp).await;
diff --git a/crates/ai-memory-llm/src/google.rs b/crates/ai-memory-llm/src/google.rs
index 84a1053f..0bdb3e5a 100644
--- a/crates/ai-memory-llm/src/google.rs
+++ b/crates/ai-memory-llm/src/google.rs
@@ -9,7 +9,7 @@ use secrecy::{ExposeSecret, SecretString};
use serde::{Deserialize, Serialize};
use tracing::debug;
-use crate::embedding::{Embedder, normalise};
+use crate::embedding::{Embedder, normalise, retry_delay};
use crate::error::{LlmError, LlmResult};
use crate::response::{provider_error_body, response_json_limited};
@@ -86,6 +86,7 @@ impl GoogleEmbedder {
debug!(url, model = %self.model, ?task_type, "POST google/embedContent");
let mut attempt = 0u32;
+ let mut waited = Duration::ZERO;
loop {
let resp = self
.client
@@ -95,13 +96,15 @@ impl GoogleEmbedder {
.send()
.await?;
let status = resp.status();
- if status.as_u16() == 429 && attempt < 5 {
+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);
+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {
attempt += 1;
- let delay = Duration::from_secs(2u64.saturating_pow(attempt));
+ waited += delay;
debug!(
attempt,
?delay,
- "google embedContent rate-limited; retrying"
+ status = status.as_u16(),
+ "google embedContent 429/503; retrying"
);
tokio::time::sleep(delay).await;
continue;