Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(embed): retry 503 like 429, honor Retry-After, fail on failed pages
Djalma Júnior9c85104a950b
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fee7fa0b..4f2e0cde 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -12,6 +12,16 @@ version in [`docs/changelog/`](docs/changelog/):## [Unreleased]### Fixed+- Retried OpenAI and OpenAI-compatible embedding requests on HTTP 503 in+ addition to 429, and both waits now honor a whole-second `Retry-After`+ response header (capped at 32 seconds per wait, with the previous+ 62-second total wait budget unchanged), so a gateway that answers 503+ with `Retry-After` while temporarily out of capacity no longer drops+ most of the pages of an `ai-memory embed` backfill.+- `ai-memory embed` now exits with a non-zero status when the report+ shows pages that failed to embed, while keeping the full JSON report+ on stdout; a run in which every page failed previously returned+ success.- Fixed `memory_lint` reporting findings for TTL-expired pages. The lint passnow omits expired pages before evaluating episodic staleness, rule suggestions,duplicate titles, and vector or LLM contradiction candidates, preventingdiff --git a/crates/ai-memory-cli/src/commands/embed.rs b/crates/ai-memory-cli/src/commands/embed.rsindex 92b08ddb..35bf955a 100644--- a/crates/ai-memory-cli/src/commands/embed.rs+++ b/crates/ai-memory-cli/src/commands/embed.rs@@ -17,6 +17,25 @@ struct EmbedRequest {all_projects: bool,}+/// Turn a live-mode `/admin/embed` report into an exit status.+///+/// The server answers 200 even when pages failed so the operator can+/// read the counts out of the body; the exit code must carry that+/// signal too, or a run in which every page failed still reads as+/// success to a script. Absent fields count as zero, so a partial or+/// older report with no `failed` key cannot fail the command.+fn check_report(report: &serde_json::Value) -> Result<()> {+ let embedded = report["embedded"].as_u64().unwrap_or(0);+ let failed = report["failed"].as_u64().unwrap_or(0);+ let skipped = report["skipped"].as_u64().unwrap_or(0);+ if failed == 0 {+ return Ok(());+ }+ anyhow::bail!(+ "{failed} page(s) failed to embed ({embedded} embedded, {skipped} skipped); run `ai-memory embed` again to retry them"+ )+}+/// Run the `embed` subcommand.////// Sends the request to the server over HTTP and prints the JSON@@ -25,7 +44,8 @@ struct EmbedRequest {////// # Errors/// Returns an error if the server is unreachable or returns a non-2xx-/// response.+/// response, or (live mode only) if the report shows pages that failed+/// to embed — the full report is still printed to stdout first.pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {let endpoint = ServerEndpoint::from_config_resolving_auth(config).await;// Model migrations must reach stale rows in every project namespace.@@ -66,6 +86,8 @@ pub async fn run(config: &Config, args: EmbedArgs) -> Result<()> {println!("dry-run: would embed {would} page(s), {skipped} already up-to-date");} else {println!("{}", serde_json::to_string_pretty(&report)?);+ // Live mode: the report is on stdout, the exit code must agree.+ check_report(&report)?;}Ok(())}@@ -75,6 +97,35 @@ mod tests {use crate::cli::{Cli, Command};use crate::config::{Config, RuntimeEnv};use clap::Parser;+ use serde_json::json;++ #[test]+ fn check_report_accepts_a_clean_live_report() {+ let report = json!({ "embedded": 7, "failed": 0, "skipped": 2 });+ super::check_report(&report).expect("failed == 0 must succeed");+ }++ #[test]+ fn check_report_fails_with_the_counts_when_pages_failed() {+ let report = json!({ "embedded": 3, "failed": 2, "skipped": 1 });+ let err = super::check_report(&report).expect_err("failed > 0 must fail");+ let msg = err.to_string();+ assert!(msg.contains("2 page(s) failed to embed"), "message: {msg}");+ assert!(msg.contains("3 embedded"), "message: {msg}");+ assert!(msg.contains("1 skipped"), "message: {msg}");+ assert!(+ msg.contains("run `ai-memory embed` again"),+ "message: {msg}"+ );+ }++ #[test]+ fn check_report_treats_absent_fields_as_zero() {+ let empty = json!({});+ super::check_report(&empty).expect("absent failed must count as 0");+ let embedded_only = json!({ "embedded": 4, "skipped": 1 });+ super::check_report(&embedded_only).expect("a report without a failed key must succeed");+ }#[test]fn force_without_project_enables_all_projects() {diff --git a/crates/ai-memory-llm/src/embedding.rs b/crates/ai-memory-llm/src/embedding.rsindex 6d39274a..125e8fd6 100644--- a/crates/ai-memory-llm/src/embedding.rs+++ b/crates/ai-memory-llm/src/embedding.rs@@ -275,10 +275,61 @@ pub(crate) fn parse_openai_embedding_values(body: &str, status: u16) -> LlmResulOk(first.embedding)}+/// Retried statuses for [`openai_style_embed`]: a rate limit and a+/// transient "out of capacity" answer. Everything else — other 4xx and+/// 5xx — is handled exactly as before: no retry.+fn is_retryable_embedding_status(status: u16) -> bool {+ matches!(status, 429 | 503)+}++/// Cap for a single wait taken from `Retry-After`; larger values are+/// clamped so one bad header cannot stall a call for minutes.+const MAX_RETRY_AFTER_SECS: u64 = 32;++/// Total wait budget for one embedding call: the worst case of the+/// previous fixed backoff (2+4+8+16+32). When the next wait would push+/// the call past this, the loop gives up and returns the error of the+/// last status, so search (`embed_search_query`) and the inline+/// write-page embed never wait longer than a 429 storm already could.+const MAX_RETRY_WAIT_BUDGET: Duration = Duration::from_secs(62);++/// Decide the wait before the next attempt of [`openai_style_embed`], or+/// give up.+///+/// Only 429 and 503 are retriable ([`is_retryable_embedding_status`]).+/// `attempt` is the 1-based index of the attempt being decided: up to+/// five new attempts are allowed, and attempt 6 is already past the+/// budget of retries. A whole-second `Retry-After` header (e.g.+/// `Retry-After: 5`) is honoured, clamped to [`MAX_RETRY_AFTER_SECS`];+/// a missing header, a non-whole-second value (the HTTP-date form is not+/// supported) or an unparseable one falls back to the historical+/// `2^attempt` seconds (2, 4, 8, 16, 32). `None` means the call should+/// stop and report the last status: a non-retriable status, more than+/// five new attempts, or `waited + delay` exceeding+/// [`MAX_RETRY_WAIT_BUDGET`].+fn retry_delay(+ status: reqwest::StatusCode,+ retry_after: Option<&reqwest::header::HeaderValue>,+ attempt: u32,+ waited: Duration,+) -> Option<Duration> {+ if !is_retryable_embedding_status(status.as_u16()) || attempt > 5 {+ return None;+ }+ let delay = match retry_after+ .and_then(|h| h.to_str().ok())+ .and_then(|s| s.parse::<u64>().ok())+ {+ Some(secs) => Duration::from_secs(secs.min(MAX_RETRY_AFTER_SECS)),+ None => Duration::from_secs(2u64.saturating_pow(attempt)),+ };+ (waited + delay <= MAX_RETRY_WAIT_BUDGET).then_some(delay)+}+/// Shared OpenAI-shape embedding request: head-truncate, POST/// `/embeddings` (bearer auth only when a key is present — local-/// engines such as Ollama and LM Studio run keyless), retry 429 with-/// backoff, parse, dim-check, unit-normalise.+/// engines such as Ollama and LM Studio run keyless), retry 429 and 503+/// honouring `Retry-After`, parse, dim-check, unit-normalise.async fn openai_style_embed(client: &reqwest::Client,base_url: &str,@@ -300,6 +351,7 @@ async fn openai_style_embed(model,};let mut attempt = 0u32;+ let mut waited = Duration::ZERO;loop {let mut builder = client.post(&url);if let Some(key) = api_key {@@ -307,10 +359,18 @@ async fn openai_style_embed(}let resp = builder.json(&req).send().await?;let status = resp.status();- if status.as_u16() == 429 && attempt < 5 {+ // Read the header while the response is still owned here: nothing+ // below may consume the body before the wait is decided.+ let retry_after = resp.headers().get(reqwest::header::RETRY_AFTER);+ if let Some(delay) = retry_delay(status, retry_after, attempt + 1, waited) {attempt += 1;- let delay = Duration::from_secs(2u64.saturating_pow(attempt));- debug!(attempt, ?delay, "openai embeddings rate-limited; retrying");+ waited += delay;+ debug!(+ attempt,+ ?delay,+ status = status.as_u16(),+ "openai embeddings 429/503; retrying"+ );tokio::time::sleep(delay).await;continue;}@@ -762,6 +822,144 @@ pub fn cosine(a: &[f32], b: &[f32]) -> f32 {mod tests {use super::*;+ fn header(value: &'static str) -> reqwest::header::HeaderValue {+ reqwest::header::HeaderValue::from_static(value)+ }++ #[test]+ fn retry_delay_honours_whole_second_retry_after_on_429_and_503() {+ let five = Duration::from_secs(5);+ assert_eq!(+ retry_delay(+ reqwest::StatusCode::from_u16(429).unwrap(),+ Some(&header("5")),+ 1,+ Duration::ZERO+ ),+ Some(five)+ );+ assert_eq!(+ retry_delay(+ reqwest::StatusCode::from_u16(503).unwrap(),+ Some(&header("5")),+ 1,+ Duration::ZERO+ ),+ Some(five)+ );+ }++ #[test]+ fn retry_delay_clamps_retry_after_to_the_per_wait_cap() {+ let capped = Duration::from_secs(32);+ assert_eq!(+ retry_delay(+ reqwest::StatusCode::from_u16(429).unwrap(),+ Some(&header("120")),+ 1,+ Duration::ZERO+ ),+ Some(capped)+ );+ assert_eq!(+ retry_delay(+ reqwest::StatusCode::from_u16(503).unwrap(),+ Some(&header("120")),+ 1,+ Duration::ZERO+ ),+ Some(capped)+ );+ }++ #[test]+ fn retry_delay_falls_back_to_exponential_backoff_without_a_valid_header() {+ let status = reqwest::StatusCode::from_u16(503).unwrap();+ // Missing header: the historical 2^attempt seconds.+ assert_eq!(+ retry_delay(status, None, 1, Duration::ZERO),+ Some(Duration::from_secs(2))+ );+ assert_eq!(+ retry_delay(status, None, 3, Duration::ZERO),+ Some(Duration::from_secs(8))+ );+ assert_eq!(+ retry_delay(status, None, 5, Duration::ZERO),+ Some(Duration::from_secs(32))+ );+ // A non-whole-second value is not supported (HTTP-date form):+ // same fallback as an absent header.+ assert_eq!(+ retry_delay(status, Some(&header("abc")), 1, Duration::ZERO),+ Some(Duration::from_secs(2))+ );+ assert_eq!(+ retry_delay(+ status,+ Some(&header("Wed, 21 Oct 2015 07:28:00 GMT")),+ 1,+ Duration::ZERO+ ),+ Some(Duration::from_secs(2))+ );+ }++ #[test]+ fn retry_delay_refuses_non_retriable_statuses() {+ let header5 = Some(&header("5"));+ for status in [400u16, 404, 500, 502] {+ let code = reqwest::StatusCode::from_u16(status).unwrap();+ assert_eq!(+ retry_delay(code, header5, 1, Duration::ZERO),+ None,+ "status {status}"+ );+ assert_eq!(+ retry_delay(code, None, 1, Duration::ZERO),+ None,+ "status {status}"+ );+ }+ }++ #[test]+ fn retry_delay_stops_after_five_new_attempts() {+ let status = reqwest::StatusCode::from_u16(503).unwrap();+ assert_eq!(+ retry_delay(status, Some(&header("0")), 5, Duration::from_secs(30)),+ Some(Duration::ZERO),+ "attempt 5 stays inside the attempt cap and the 62 s budget"+ );+ assert_eq!(+ retry_delay(status, Some(&header("0")), 6, Duration::ZERO),+ None+ );+ assert_eq!(retry_delay(status, None, 6, Duration::ZERO), None);+ assert_eq!(retry_delay(status, None, 7, Duration::ZERO), None);+ }++ #[test]+ fn retry_delay_gives_up_when_the_wait_budget_is_exhausted() {+ let status = reqwest::StatusCode::from_u16(503).unwrap();+ // 35 s already waited plus a 30 s Retry-After would reach 65 s.+ assert_eq!(+ retry_delay(status, Some(&header("30")), 1, Duration::from_secs(35)),+ None+ );+ // A clamped 120 s header (32 s) after 32 s already waited: 64 s > 62 s.+ assert_eq!(+ retry_delay(status, Some(&header("120")), 2, Duration::from_secs(32)),+ None+ );+ // Boundary control: the previous worst case (30 s waited + 32 s)+ // still fits the budget, so behaviour is unchanged at the limit.+ assert_eq!(+ retry_delay(status, None, 5, Duration::from_secs(30)),+ Some(Duration::from_secs(32))+ );+ }+#[test]fn document_prefix_identity_is_unchanged_when_the_prefix_is_empty() {// The legacy-identity guarantee: an install with no documentdiff --git a/crates/ai-memory-llm/tests/suite/mod.rs b/crates/ai-memory-llm/tests/suite/mod.rsindex cd14007b..b8973465 100644--- a/crates/ai-memory-llm/tests/suite/mod.rs+++ b/crates/ai-memory-llm/tests/suite/mod.rs@@ -10,4 +10,5 @@ mod extra_headers_on_the_wire;mod fallback_provider;mod openai_compat_embedder;mod openai_compat_guardrails;+mod openai_compat_retry_after;mod openai_compat_strict;diff --git a/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rs b/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rsnew file mode 100644index 00000000..67688fde--- /dev/null+++ b/crates/ai-memory-llm/tests/suite/openai_compat_retry_after.rs@@ -0,0 +1,123 @@+//! Retry behaviour of the OpenAI-shape embedding request against an+//! in-process wiremock: 429 and 503 are retried (and nothing else), a+//! whole-second `Retry-After` is honoured, and the retry budget is+//! bounded both in attempts (five new attempts, six requests total) and+//! in total wait time.+//!+//! Every mock answers with `Retry-After: 0`, so the waits are zero and+//! no test sleeps for more than an instant.++use std::sync::atomic::{AtomicUsize, Ordering};++use ai_memory_llm::{Embedder, LlmError, OpenAiCompatEmbedder};+use wiremock::matchers::{method, path};+use wiremock::{Mock, MockServer, ResponseTemplate};++fn success_body(dim: usize) -> serde_json::Value {+ serde_json::json!({+ "object": "list",+ "data": [{ "object": "embedding", "index": 0, "embedding": vec![0.5_f32; dim] }],+ "model": "nomic-embed-text",+ "usage": { "prompt_tokens": 1, "total_tokens": 1 },+ })+}++#[tokio::test]+async fn retry_after_0_503_twice_then_success_succeeds_in_exactly_three_requests() {+ let server = MockServer::start().await;+ let hits = std::sync::Arc::new(AtomicUsize::new(0));+ let counter = hits.clone();+ Mock::given(method("POST"))+ .and(path("/v1/embeddings"))+ .respond_with(move |_req: &wiremock::Request| {+ let n = counter.fetch_add(1, Ordering::SeqCst) + 1;+ if n <= 2 {+ // A gateway out of capacity: 503 with a zero wait so the+ // test does not sleep.+ ResponseTemplate::new(503)+ .insert_header("Retry-After", "0")+ .set_body_string("gateway unavailable")+ } else {+ ResponseTemplate::new(200).set_body_json(success_body(4))+ }+ })+ .expect(3)+ .mount(&server)+ .await;++ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)+ .expect("embedder builds");+ let v = e+ .embed("retry me")+ .await+ .expect("embed succeeds after two 503s");+ assert_eq!(v.len(), 4);+ assert_eq!(+ hits.load(Ordering::SeqCst),+ 3,+ "503, 503, 200 — exactly three requests"+ );+}++#[tokio::test]+async fn endless_503_fails_after_at_most_six_requests() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/embeddings"))+ .respond_with(+ ResponseTemplate::new(503)+ .insert_header("Retry-After", "0")+ .set_body_string("gateway unavailable"),+ )+ // One initial request plus the five allowed new attempts.+ .expect(6)+ .mount(&server)+ .await;++ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)+ .expect("embedder builds");+ let err = e+ .embed("retry me")+ .await+ .expect_err("a gateway that never recovers must fail");+ assert!(+ matches!(err, LlmError::Provider { status: 503, .. }),+ "the last status is reported: {err:?}"+ );+ server.verify().await;+}++#[tokio::test]+async fn retry_after_0_429_once_then_success_succeeds_in_exactly_two_requests() {+ let server = MockServer::start().await;+ let hits = std::sync::Arc::new(AtomicUsize::new(0));+ let counter = hits.clone();+ Mock::given(method("POST"))+ .and(path("/v1/embeddings"))+ .respond_with(move |_req: &wiremock::Request| {+ let n = counter.fetch_add(1, Ordering::SeqCst) + 1;+ if n == 1 {+ ResponseTemplate::new(429)+ .insert_header("Retry-After", "0")+ .set_body_string("rate limited")+ } else {+ ResponseTemplate::new(200).set_body_json(success_body(4))+ }+ })+ .expect(2)+ .mount(&server)+ .await;++ let e = OpenAiCompatEmbedder::new(format!("{}/v1", server.uri()), None, "nomic-embed-text", 4)+ .expect("embedder builds");+ let v = e+ .embed("rate limited")+ .await+ .expect("embed succeeds after one 429");+ assert_eq!(v.len(), 4);+ assert_eq!(+ hits.load(Ordering::SeqCst),+ 2,+ "429, 200 — exactly two requests"+ );+}