Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(embed): retry 503 like 429, honor Retry-After, fail on failed pages

Djalma Júnior9c85104a950b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fee7fa0b..4f2e0cde 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,16 @@ version in [`docs/changelog/`](docs/changelog/):
## [Unreleased]
### Fixed
+- Retried OpenAI and OpenAI-compatible embedding requests on HTTP 503 in
+ addition to 429, and both waits now honor a whole-second `Retry-After`
+ response header (capped at 32 seconds per wait, with the previous
+ 62-second total wait budget unchanged), so a gateway that answers 503
+ with `Retry-After` while temporarily out of capacity no longer drops
+ most of the pages of an `ai-memory embed` backfill.
+- `ai-memory embed` now exits with a non-zero status when the report
+ shows pages that failed to embed, while keeping the full JSON report
+ on stdout; a run in which every page failed previously returned
+ success.
- Fixed `memory_lint` reporting findings for TTL-expired pages. The lint pass
now omits expired pages before evaluating episodic staleness, rule suggestions,
duplicate titles, and vector or LLM contradiction candidates, preventing
diff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rs
index 92b08ddb..35bf955a 100644
--- a/crates/ai-memory-cli/src/commands/embed.rs
+++ b/crates/ai-memory-cli/src/commands/embed.rs
@@ -17,6 +17,25 @@ struct EmbedRequest {
all_projects: bool,
}
+/// Turn a live-mode `/admin/embed` report into an exit status.
+///
+/// The server answers 200 even when pages failed so the operator can
+/// read the counts out of the body; the exit code must carry that
+/// signal too, or a run in which every page failed still reads as
+/// success to a script. Absent fields count as zero, so a partial or
+/// older report with no `failed` key cannot fail the command.
+fn check_report(report: &serde_json::Value) -> Result<()> {
+ let embedded = report["embedded"].as_u64().unwrap_or(0);
+ let failed = report["failed"].as_u64().unwrap_or(0);
+ let skipped = report["skipped"].as_u64().unwrap_or(0);
+ if failed == 0 {
+ return Ok(());
+ }
+ anyhow::bail!(
+ "{failed} page(s) failed to embed ({embedded} embedded, {skipped} skipped); run `ai-memory embed` again to retry them"
+ )
+}
+
/// Run the `embed` subcommand.
///
/// Sends the request to the server over HTTP and prints the JSON
@@ -25,7 +44,8 @@ struct EmbedRequest {
///
/// # Errors
/// Returns an error if the server is unreachable or returns a non-2xx
-/// response.
+/// response, or (live mode only) if the report shows pages that failed
+/// to embed — the full report is still printed to stdout first.
pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {
let endpoint = ServerEndpoint::from_config_resolving_auth(config).await;
// Model migrations must reach stale rows in every project namespace.
@@ -66,6 +86,8 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {
println!("dry-run: would embed {would} page(s), {skipped} already up-to-date");
} else {
println!("{}", serde_json::to_string_pretty(&report)?);
+ // Live mode: the report is on stdout, the exit code must agree.
+ check_report(&report)?;
}
Ok(())
}
@@ -75,6 +97,35 @@ mod tests {
use crate::cli::{Cli, Command};
use crate::config::{Config, RuntimeEnv};
use clap::Parser;
+ use serde_json::json;
+
+ #[test]
+ fn check_report_accepts_a_clean_live_report() {
+ let report = json!({ "embedded": 7, "failed": 0, "skipped": 2 });
+ super::check_report(&report).expect("failed == 0 must succeed");
+ }
+
+ #[test]
+ fn check_report_fails_with_the_counts_when_pages_failed() {
+ let report = json!({ "embedded": 3, "failed": 2, "skipped": 1 });
+ let err = super::check_report(&report).expect_err("failed > 0 must fail");
+ let msg = err.to_string();
+ assert!(msg.contains("2 page(s) failed to embed"), "message: {msg}");
+ assert!(msg.contains("3 embedded"), "message: {msg}");
+ assert!(msg.contains("1 skipped"), "message: {msg}");
+ assert!(
+ msg.contains("run `ai-memory embed` again"),
+ "message: {msg}"
+ );
+ }
+
+ #[test]
+ fn check_report_treats_absent_fields_as_zero() {
+ let empty = json!({});
+ super::check_report(&empty).expect("absent failed must count as 0");
+ let embedded_only = json!({ "embedded": 4, "skipped": 1 });
+ super::check_report(&embedded_only).expect("a report without a failed key must succeed");
+ }
#[test]
fn force_without_project_enables_all_projects() {
diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rs
index 6d39274a..125e8fd6 100644
--- a/crates/ai-memory-llm/src/embedding.rs
+++ b/crates/ai-memory-llm/src/embedding.rs
@@ -275,10 +275,61 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResul
Ok(first.embedding)
}
+/// Retried statuses for [`openai_style_embed`]: a rate limit and a
+/// transient "out of capacity" answer. Everything else — other 4xx and
+/// 5xx — is handled exactly as before: no retry.
+fn is_retryable_embedding_status(status: u16) -> bool {
+ matches!(status, 429 | 503)
+}
+
+/// Cap for a single wait taken from `Retry-After`; larger values are
+/// clamped so one bad header cannot stall a call for minutes.
+const MAX_RETRY_AFTER_SECS: u64 = 32;
+
+/// Total wait budget for one embedding call: the worst case of the
+/// previous fixed backoff (2+4+8+16+32). When the next wait would push
+/// the call past this, the loop gives up and returns the error of the
+/// last status, so search (`embed_search_query`) and the inline
+/// write-page embed never wait longer than a 429 storm already could.
+const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);
+
+/// Decide the wait before the next attempt of [`openai_style_embed`], or
+/// give up.
+///
+/// Only 429 and 503 are retriable ([`is_retryable_embedding_status`]).
+/// `attempt` is the 1-based index of the attempt being decided: up to
+/// five new attempts are allowed, and attempt 6 is already past the
+/// budget of retries. A whole-second `Retry-After` header (e.g.
+/// `Retry-After: 5`) is honoured, clamped to [`MAX_RETRY_AFTER_SECS`];
+/// a missing header, a non-whole-second value (the HTTP-date form is not
+/// supported) or an unparseable one falls back to the historical
+/// `2^attempt` seconds (2, 4, 8, 16, 32). `None` means the call should
+/// stop and report the last status: a non-retriable status, more than
+/// five new attempts, or `waited + delay` exceeding
+/// [`MAX_RETRY_WAIT_BUDGET`].
+fn retry_delay(
+ status: reqwest::StatusCode,
+ retry_after: Option<&reqwest::header::HeaderValue>,
+ attempt: u32,
+ waited: Duration,
+) -> Option<Duration> {
+ if !is_retryable_embedding_status(status.as_u16()) || attempt > 5 {
+ return None;
+ }
+ let delay = match retry_after
+ .and_then(|h| h.to_str().ok())
+ .and_then(|s| s.parse::<u64>().ok())
+ {
+ Some(secs) => Duration::from_secs(secs.min(MAX_RETRY_AFTER_SECS)),
+ None => Duration::from_secs(2u64.saturating_pow(attempt)),
+ };
+ (waited + delay <= MAX_RETRY_WAIT_BUDGET).then_some(delay)
+}
+
/// Shared OpenAI-shape embedding request: head-truncate, POST
/// `/embeddings` (bearer auth only when a key is present — local
-/// engines such as Ollama and LM Studio run keyless), retry 429 with
-/// backoff, parse, dim-check, unit-normalise.
+/// engines such as Ollama and LM Studio run keyless), retry 429 and 503
+/// honouring `Retry-After`, parse, dim-check, unit-normalise.
async fn openai_style_embed(
client: &reqwest::Client,
base_url: &str,
@@ -300,6 +351,7 @@ async fn openai_style_embed(
model,
};
let mut attempt = 0u32;
+ let mut waited = Duration::ZERO;
loop {
let mut builder = client.post(&url);
if let Some(key) = api_key {
@@ -307,10 +359,18 @@ async fn openai_style_embed(
}
let resp = builder.json(&req).send().await?;
let status = resp.status();
- if status.as_u16() == 429 && attempt < 5 {
+ // Read the header while the response is still owned here: nothing
+ // below may consume the body before the wait is decided.
+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);
+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {
attempt += 1;
- let delay = Duration::from_secs(2u64.saturating_pow(attempt));
- debug!(attempt, ?delay, "openai embeddings rate-limited; retrying");
+ waited += delay;
+ debug!(
+ attempt,
+ ?delay,
+ status = status.as_u16(),
+ "openai embeddings 429/503; retrying"
+ );
tokio::time::sleep(delay).await;
continue;
}
@@ -762,6 +822,144 @@ pub fn cosine(a: &[f32], b: &[f32]) -> f32 {
mod tests {
use super::*;
+ fn header(value: &'static str) -> reqwest::header::HeaderValue {
+ reqwest::header::HeaderValue::from_static(value)
+ }
+
+ #[test]
+ fn retry_delay_honours_whole_second_retry_after_on_429_and_503() {
+ let five = Duration::from_secs(5);
+ assert_eq!(
+ retry_delay(
+ reqwest::StatusCode::from_u16(429).unwrap(),
+ Some(&header("5")),
+ 1,
+ Duration::ZERO
+ ),
+ Some(five)
+ );
+ assert_eq!(
+ retry_delay(
+ reqwest::StatusCode::from_u16(503).unwrap(),
+ Some(&header("5")),
+ 1,
+ Duration::ZERO
+ ),
+ Some(five)
+ );
+ }
+
+ #[test]
+ fn retry_delay_clamps_retry_after_to_the_per_wait_cap() {
+ let capped = Duration::from_secs(32);
+ assert_eq!(
+ retry_delay(
+ reqwest::StatusCode::from_u16(429).unwrap(),
+ Some(&header("120")),
+ 1,
+ Duration::ZERO
+ ),
+ Some(capped)
+ );
+ assert_eq!(
+ retry_delay(
+ reqwest::StatusCode::from_u16(503).unwrap(),
+ Some(&header("120")),
+ 1,
+ Duration::ZERO
+ ),
+ Some(capped)
+ );
+ }
+
+ #[test]
+ fn retry_delay_falls_back_to_exponential_backoff_without_a_valid_header() {
+ let status = reqwest::StatusCode::from_u16(503).unwrap();
+ // Missing header: the historical 2^attempt seconds.
+ assert_eq!(
+ retry_delay(status, None, 1, Duration::ZERO),
+ Some(Duration::from_secs(2))
+ );
+ assert_eq!(
+ retry_delay(status, None, 3, Duration::ZERO),
+ Some(Duration::from_secs(8))
+ );
+ assert_eq!(
+ retry_delay(status, None, 5, Duration::ZERO),
+ Some(Duration::from_secs(32))
+ );
+ // A non-whole-second value is not supported (HTTP-date form):
+ // same fallback as an absent header.
+ assert_eq!(
+ retry_delay(status, Some(&header("abc")), 1, Duration::ZERO),
+ Some(Duration::from_secs(2))
+ );
+ assert_eq!(
+ retry_delay(
+ status,
+ Some(&header("Wed, 21 Oct 2015 07:28:00 GMT")),
+ 1,
+ Duration::ZERO
+ ),
+ Some(Duration::from_secs(2))
+ );
+ }
+
+ #[test]
+ fn retry_delay_refuses_non_retriable_statuses() {
+ let header5 = Some(&header("5"));
+ for status in [400u16, 404, 500, 502] {
+ let code = reqwest::StatusCode::from_u16(status).unwrap();
+ assert_eq!(
+ retry_delay(code, header5, 1, Duration::ZERO),
+ None,
+ "status {status}"
+ );
+ assert_eq!(
+ retry_delay(code, None, 1, Duration::ZERO),
+ None,
+ "status {status}"
+ );
+ }
+ }
+
+ #[test]
+ fn retry_delay_stops_after_five_new_attempts() {
+ let status = reqwest::StatusCode::from_u16(503).unwrap();
+ assert_eq!(
+ retry_delay(status, Some(&header("0")), 5, Duration::from_secs(30)),
+ Some(Duration::ZERO),
+ "attempt 5 stays inside the attempt cap and the 62 s budget"
+ );
+ assert_eq!(
+ retry_delay(status, Some(&header("0")), 6, Duration::ZERO),
+ None
+ );
+ assert_eq!(retry_delay(status, None, 6, Duration::ZERO), None);
+ assert_eq!(retry_delay(status, None, 7, Duration::ZERO), None);
+ }
+
+ #[test]
+ fn retry_delay_gives_up_when_the_wait_budget_is_exhausted() {
+ let status = reqwest::StatusCode::from_u16(503).unwrap();
+ // 35 s already waited plus a 30 s Retry-After would reach 65 s.
+ assert_eq!(
+ retry_delay(status, Some(&header("30")), 1, Duration::from_secs(35)),
+ None
+ );
+ // A clamped 120 s header (32 s) after 32 s already waited: 64 s > 62 s.
+ assert_eq!(
+ retry_delay(status, Some(&header("120")), 2, Duration::from_secs(32)),
+ None
+ );
+ // Boundary control: the previous worst case (30 s waited + 32 s)
+ // still fits the budget, so behaviour is unchanged at the limit.
+ assert_eq!(
+ retry_delay(status, None, 5, Duration::from_secs(30)),
+ Some(Duration::from_secs(32))
+ );
+ }
+
#[test]
fn document_prefix_identity_is_unchanged_when_the_prefix_is_empty() {
// The legacy-identity guarantee: an install with no document
diff --git a/crates/ai-memory-llm/tests/suite/mod.rs b/crates/ai-memory-llm/tests/suite/mod.rs
index cd14007b..b8973465 100644
--- a/crates/ai-memory-llm/tests/suite/mod.rs
+++ b/crates/ai-memory-llm/tests/suite/mod.rs
@@ -10,4 +10,5 @@ mod extra_headers_on_the_wire;
mod fallback_provider;
mod openai_compat_embedder;
mod openai_compat_guardrails;
+mod openai_compat_retry_after;
mod openai_compat_strict;
diff --git a/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rs b/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rs
new file mode 100644
index 00000000..67688fde
--- /dev/null
+++ b/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rs
@@ -0,0 +1,123 @@
+//! Retry behaviour of the OpenAI-shape embedding request against an
+//! in-process wiremock: 429 and 503 are retried (and nothing else), a
+//! whole-second `Retry-After` is honoured, and the retry budget is
+//! bounded both in attempts (five new attempts, six requests total) and
+//! in total wait time.
+//!
+//! Every mock answers with `Retry-After: 0`, so the waits are zero and
+//! no test sleeps for more than an instant.
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use ai_memory_llm::{Embedder, LlmError, OpenAiCompatEmbedder};
+use wiremock::matchers::{method, path};
+use wiremock::{Mock, MockServer, ResponseTemplate};
+
+fn success_body(dim: usize) -> serde_json::Value {
+ serde_json::json!({
+ "object": "list",
+ "data": [{ "object": "embedding", "index": 0, "embedding": vec![0.5_f32; dim] }],
+ "model": "nomic-embed-text",
+ "usage": { "prompt_tokens": 1, "total_tokens": 1 },
+ })
+}
+
+#[tokio::test]
+async fn retry_after_0_503_twice_then_success_succeeds_in_exactly_three_requests() {
+ let server = MockServer::start().await;
+ let hits = std::sync::Arc::new(AtomicUsize::new(0));
+ let counter = hits.clone();
+ Mock::given(method("POST"))
+ .and(path("/v1/embeddings"))
+ .respond_with(move |_req: &wiremock::Request| {
+ let n = counter.fetch_add(1, Ordering::SeqCst) + 1;
+ if n <= 2 {
+ // A gateway out of capacity: 503 with a zero wait so the
+ // test does not sleep.
+ ResponseTemplate::new(503)
+ .insert_header("Retry-After", "0")
+ .set_body_string("gateway unavailable")
+ } else {
+ ResponseTemplate::new(200).set_body_json(success_body(4))
+ }
+ })
+ .expect(3)
+ .mount(&server)
+ .await;
+
+ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)
+ .expect("embedder builds");
+ let v = e
+ .embed("retry me")
+ .await
+ .expect("embed succeeds after two 503s");
+ assert_eq!(v.len(), 4);
+ assert_eq!(
+ hits.load(Ordering::SeqCst),
+ 3,
+ "503, 503, 200 — exactly three requests"
+ );
+}
+
+#[tokio::test]
+async fn endless_503_fails_after_at_most_six_requests() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/v1/embeddings"))
+ .respond_with(
+ ResponseTemplate::new(503)
+ .insert_header("Retry-After", "0")
+ .set_body_string("gateway unavailable"),
+ )
+ // One initial request plus the five allowed new attempts.
+ .expect(6)
+ .mount(&server)
+ .await;
+
+ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)
+ .expect("embedder builds");
+ let err = e
+ .embed("retry me")
+ .await
+ .expect_err("a gateway that never recovers must fail");
+ assert!(
+ matches!(err, LlmError::Provider { status: 503, .. }),
+ "the last status is reported: {err:?}"
+ );
+ server.verify().await;
+}
+
+#[tokio::test]
+async fn retry_after_0_429_once_then_success_succeeds_in_exactly_two_requests() {
+ let server = MockServer::start().await;
+ let hits = std::sync::Arc::new(AtomicUsize::new(0));
+ let counter = hits.clone();
+ Mock::given(method("POST"))
+ .and(path("/v1/embeddings"))
+ .respond_with(move |_req: &wiremock::Request| {
+ let n = counter.fetch_add(1, Ordering::SeqCst) + 1;
+ if n == 1 {
+ ResponseTemplate::new(429)
+ .insert_header("Retry-After", "0")
+ .set_body_string("rate limited")
+ } else {
+ ResponseTemplate::new(200).set_body_json(success_body(4))
+ }
+ })
+ .expect(2)
+ .mount(&server)
+ .await;
+
+ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)
+ .expect("embedder builds");
+ let v = e
+ .embed("rate limited")
+ .await
+ .expect("embed succeeds after one 429");
+ assert_eq!(v.len(), 4);
+ assert_eq!(
+ hits.load(Ordering::SeqCst),
+ 2,
+ "429, 200 — exactly two requests"
+ );
+}