Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(rerank): make the memory_query rerank budget configurable
Renato Regis597bf0770587
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6609cfa4..d42ddfe4 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -11,6 +11,15 @@ version in [`docs/changelog/`](docs/changelog/):## [Unreleased]+### Added++- Added `AI_MEMORY_RERANK_TIMEOUT_SECS` (`rerank_timeout_secs` in+ `config.toml`) to set the `memory_query` reranker budget, which was fixed at+ 20 seconds. The default stays 20 and zero is rejected at startup. A hosted+ provider behind an LLM fallback chain can spend the whole fixed budget on one+ slow 503 before the next candidate answers, and every such query then keeps+ its pre-rerank order.+## [3.2.0] - 2026-10-21### Addeddiff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rsindex 2a15379c..203516a3 100644--- a/crates/ai-memory-cli/src/commands/serve.rs+++ b/crates/ai-memory-cli/src/commands/serve.rs@@ -3168,9 +3168,13 @@ fn configure_consolidator(info!(provider = llm.name(),model = llm.model(),+ timeout_secs = config.rerank_timeout_secs,"project/scopes memory_query reranking enabled (adds LLM latency)",);- server = server.with_reranker(Arc::new(ai_memory_llm::LlmReranker::new(llm.clone())));+ let stage =+ ai_memory_llm::RerankStage::new(Arc::new(ai_memory_llm::LlmReranker::new(llm.clone())))+ .with_timeout(std::time::Duration::from_secs(config.rerank_timeout_secs));+ server = server.with_rerank_stage(stage);}Ok(ConsolidatorSetup {server,diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rsindex 5dc8670e..4eaf5a0d 100644--- a/crates/ai-memory-cli/src/config.rs+++ b/crates/ai-memory-cli/src/config.rs@@ -519,6 +519,12 @@ pub struct Config {/// preserves the fused, authority-adjusted order. Set with/// `AI_MEMORY_RERANKER=llm`.pub reranker: Option<String>,+ /// Wall-clock budget, in seconds, for one rerank call before the search+ /// answers from the pre-rerank order. Defaults to+ /// `ai_memory_llm::reranker::RERANK_TIMEOUT` (20s). Raise it when the+ /// LLM chain falls back across slow free-tier candidates (a hosted 503+ /// can take ~20s alone). Set with `AI_MEMORY_RERANK_TIMEOUT_SECS`.+ pub rerank_timeout_secs: u64,/// Optional embedding provider (`openai`, `voyage`, `google` / `gemini`,/// or `openai-compat`).pub embedding_provider: Option<String>,@@ -1427,6 +1433,7 @@ impl Default for Config {strip_root_combinators: false,gemini_safe_schemas: false,reranker: None,+ rerank_timeout_secs: ai_memory_llm::reranker::RERANK_TIMEOUT.as_secs(),embedding_provider: None,embedding_model: None,embedding_dim: None,@@ -2630,6 +2637,15 @@ impl Config {config.llm_timeout_secs);}+ // Zero would time out every rerank before the provider is called,+ // silently turning reranking off while it still reports as enabled.+ if config.rerank_timeout_secs == 0 {+ anyhow::bail!(+ "rerank_timeout_secs must be at least 1 second (got {}); \+ AI_MEMORY_RERANK_TIMEOUT_SECS is read in seconds",+ config.rerank_timeout_secs+ );+ }// Parsed (and discarded) here so a malformed header list is rejected// at startup rather than on the first consolidation pass. The typed// value is rebuilt by `llm_provider_config`: `ExtraHeaders` is not@@ -5432,6 +5448,37 @@ input_token_safety_margin = {margin});}+ /// `AI_MEMORY_RERANK_TIMEOUT_SECS` (figment maps it to this field) lets a+ /// slow free-tier fallback chain finish a rerank; unset keeps the 20s+ /// budget the reranker always had.+ #[test]+ fn load_defaults_and_round_trips_the_rerank_timeout() {+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(&config_path, "").unwrap();+ let cfg = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();+ assert_eq!(+ cfg.rerank_timeout_secs,+ ai_memory_llm::reranker::RERANK_TIMEOUT.as_secs()+ );+ std::fs::write(&config_path, "rerank_timeout_secs = 60\n").unwrap();+ let cfg = Config::load(Some(&config_path), Some(tmp.path().to_path_buf())).unwrap();+ assert_eq!(cfg.rerank_timeout_secs, 60);+ }++ #[test]+ fn load_rejects_a_zero_rerank_timeout() {+ let tmp = TempDir::new().unwrap();+ let config_path = tmp.path().join("config.toml");+ std::fs::write(&config_path, "rerank_timeout_secs = 0\n").unwrap();+ let error = Config::load(Some(&config_path), Some(tmp.path().to_path_buf()))+ .expect_err("a zero rerank budget must fail closed");+ assert!(+ error.to_string().contains("rerank_timeout_secs"),+ "unexpected error: {error:#}"+ );+ }+#[test]fn load_round_trips_llm_headers() {let tmp = TempDir::new().unwrap();diff --git a/crates/ai-memory-llm/src/reranker.rs b/crates/ai-memory-llm/src/reranker.rsindex 9b88e0b7..35b43d52 100644--- a/crates/ai-memory-llm/src/reranker.rs+++ b/crates/ai-memory-llm/src/reranker.rs@@ -237,6 +237,7 @@ pub const RERANK_TIMEOUT: Duration = Duration::from_secs(20);pub struct RerankStage {reranker: Arc<dyn Reranker>,gate: Arc<Semaphore>,+ timeout: Duration,}impl RerankStage {@@ -246,9 +247,27 @@ impl RerankStage {Self {reranker,gate: Arc::new(Semaphore::new(RERANK_MAX_IN_FLIGHT)),+ timeout: RERANK_TIMEOUT,}}+ /// Replace the per-call wall-clock budget ([`RERANK_TIMEOUT`] by+ /// default). Free-tier hosted providers can spend most of the default+ /// budget on a single slow 503 before a fallback candidate answers.+ ///+ /// Example: `RerankStage::new(reranker).with_timeout(Duration::from_secs(60))`.+ #[must_use]+ pub fn with_timeout(mut self, timeout: Duration) -> Self {+ self.timeout = timeout;+ self+ }++ /// The per-call wall-clock budget every search surface applies.+ #[must_use]+ pub fn timeout(&self) -> Duration {+ self.timeout+ }+/// The wrapped reranker.#[must_use]pub fn reranker(&self) -> &Arc<dyn Reranker> {diff --git a/crates/ai-memory-mcp/src/server.rs b/crates/ai-memory-mcp/src/server.rsindex 6681f730..79466db5 100644--- a/crates/ai-memory-mcp/src/server.rs+++ b/crates/ai-memory-mcp/src/server.rs@@ -2652,7 +2652,11 @@ impl AiMemoryServer {hits: Vec<(PageHit, Option<ai_memory_store::SearchExplain>)>,limit: usize,) -> Vec<(PageHit, Option<ai_memory_store::SearchExplain>)> {- self.rerank_hits_with_timeout(query, hits, limit, ai_memory_llm::reranker::RERANK_TIMEOUT)+ let timeout = self.rerank.as_ref().map_or(+ ai_memory_llm::reranker::RERANK_TIMEOUT,+ ai_memory_llm::RerankStage::timeout,+ );+ self.rerank_hits_with_timeout(query, hits, limit, timeout).await}@@ -8211,6 +8215,41 @@ mod tests {assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);}+ /// The stage's configured budget (`AI_MEMORY_RERANK_TIMEOUT_SECS`) is the+ /// one `memory_query` applies: a slow reranker is cut off under a short+ /// budget and allowed to finish under a long one.+ #[tokio::test]+ async fn reranker_applies_the_stage_configured_timeout() {+ let (_tmp, _store, server, _ws, _proj) = setup_server().await;+ let hits = rerank_test_hits(4);+ let original_ids: Vec<PageId> = hits.iter().map(|(hit, _)| hit.id).collect();+ let reversed_ids: Vec<PageId> = original_ids.iter().rev().copied().collect();++ let (reranker, _, _) = stub_reranker(StubRerankOutcome::Reverse, Duration::from_millis(50));+ let short =+ ai_memory_llm::RerankStage::new(reranker).with_timeout(Duration::from_millis(1));+ let result = server+ .clone()+ .with_rerank_stage(short)+ .rerank_hits("query", hits.clone(), 4)+ .await;+ assert_eq!(+ result.iter().map(|(hit, _)| hit.id).collect::<Vec<_>>(),+ original_ids+ );++ let (reranker, _, _) = stub_reranker(StubRerankOutcome::Reverse, Duration::from_millis(50));+ let long = ai_memory_llm::RerankStage::new(reranker).with_timeout(Duration::from_secs(5));+ let result = server+ .with_rerank_stage(long)+ .rerank_hits("query", hits, 4)+ .await;+ assert_eq!(+ result.iter().map(|(hit, _)| hit.id).collect::<Vec<_>>(),+ reversed_ids+ );+ }+#[tokio::test]async fn reranker_degrades_without_calling_provider_when_saturated() {let (_tmp, _store, server, _ws, _proj) = setup_server().await;diff --git a/crates/ai-memory-web/src/recall.rs b/crates/ai-memory-web/src/recall.rsindex 09d161bc..a99bdc95 100644--- a/crates/ai-memory-web/src/recall.rs+++ b/crates/ai-memory-web/src/recall.rs@@ -98,7 +98,7 @@ pub(crate) async fn search_scopes(query,hits,limit,- ai_memory_llm::reranker::RERANK_TIMEOUT,+ stage.timeout(),|(hit, _)| RerankCandidate {id: hit.id.to_string(),title: hit.title.clone(),diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 7244bbcc..c2c1e20b 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -1505,6 +1505,7 @@ AI_MEMORY_LLM_HEADERS optional extra HTTP headers on every chat request, ascontain a comma through the env var — use`llm_headers = [...]` in config.toml for that.AI_MEMORY_RERANKER optional `llm`; reranks project/scopes query candidates+AI_MEMORY_RERANK_TIMEOUT_SECS optional rerank budget in seconds (default 20, minimum 1)COPILOT_GITHUB_TOKEN optional GitHub token for copilotAI_MEMORY_CODEX_EXECUTABLE optional Codex executable; defaults to codex on PATHAI_MEMORY_CURSOR_AGENT optional Cursor Agent CLI executable for cursor; defaults to agent on PATHdiff --git a/docs/llm-providers.md b/docs/llm-providers.mdindex 2f5f0e91..cf22e262 100644--- a/docs/llm-providers.md+++ b/docs/llm-providers.md@@ -434,7 +434,9 @@ RRF ranked below the requested cut, at the cost of LLM latency and usage. Therequest sends the query plus at most 30 bounded page titles and search snippetsto the configured provider; all values are JSON-encoded and treated asuntrusted data. A timeout, provider error, or incomplete/invalid score set-preserves the normal order. `global=true` and supplemental global-preference+preserves the normal order. The timeout is 20 seconds per query; raise it with+`AI_MEMORY_RERANK_TIMEOUT_SECS` (or `rerank_timeout_secs` in `config.toml`)+when a slow provider or a fallback chain needs longer to answer. `global=true` and supplemental global-preferencehits keep their existing non-RRF ranking. Concurrent provider calls are cappedat four; saturated queries keep their local ranking without waiting.