AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex,engine): damp bulk AIMD oscillation and memory-breaker drain churn (#1250)
Christian M. Todie757e00daeb5e
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 14ca476eb..3dcb23087 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -150,7 +150,13 @@ pub struct BulkAdmission {/// One congestion event may be reported by every worker that was in flight/// when the server pushed back; they must count once.const SHRINK_COOLDOWN: Duration = Duration::from_secs(1);-/// Clean bulks required before the limit is probed upward by one.+/// While no bulk has succeeded since the last shrink, coalesce repeated 429+/// retries from the same parent-breaker trip into a single halving for up to+/// 15 s (issue #1250) instead of cascading 32 → 16 → 8 → 4 → 2 → 1 once per+/// second while the server drains.+const EPISODE_MAX_COOLDOWN: Duration = Duration::from_secs(15);+/// Clean bulks required before the limit is probed upward by one below+/// `ssthresh` (above `ssthresh`, congestion avoidance scales this by `limit`).const RECOVER_AFTER: usize = 8;/// Minimum gap between two "raising bulk concurrency" lines on stderr.@@ -166,6 +172,14 @@ struct AdmissionState {in_flight: usize,ok_streak: usize,last_shrink: Option<Instant>,+ /// True after a shrink until at least one bulk succeeds, so sustained 429+ /// retries during a single breaker trip do not halve repeatedly every 1 s.+ shrunk_in_episode: bool,+ /// Slow-start threshold: below `ssthresh`, recovery steps every+ /// `RECOVER_AFTER` bulks; at or above `ssthresh` (near the concurrency that+ /// last tripped the breaker), congestion avoidance requires+ /// `RECOVER_AFTER * limit` clean bulks per +1 step.+ ssthresh: usize,/// When the last "raising" line was printed, and the limit it named —/// so the next line can name the whole climb since.last_raise_announce: Option<Instant>,@@ -205,6 +219,8 @@ impl BulkAdmission {in_flight: 0,ok_streak: 0,last_shrink: None,+ shrunk_in_episode: false,+ ssthresh: ceiling,last_raise_announce: None,announced_limit: ceiling,}),@@ -266,11 +282,20 @@ impl BulkAdmission {{return;}+ if state.shrunk_in_episode+ && state+ .last_shrink+ .is_some_and(|last| now.duration_since(last) < EPISODE_MAX_COOLDOWN)+ {+ return;+ }state.last_shrink = Some(now);state.ok_streak = 0;+ state.shrunk_in_episode = true;self.congestion_events.fetch_add(1, Ordering::Relaxed);let previous = state.limit;state.limit = (previous / 2).max(1);+ state.ssthresh = (previous * 3 / 4).max(state.limit + 1);if state.limit != previous && self.announce {eprintln!("autoindex: server pushed back (HTTP 429); lowering bulk concurrency \@@ -290,11 +315,17 @@ impl BulkAdmission {return;}let mut state = self.inner.lock().unwrap();+ state.shrunk_in_episode = false;if state.limit >= self.ceiling {return;}state.ok_streak += 1;- if state.ok_streak < RECOVER_AFTER {+ let needed = if state.limit < state.ssthresh {+ RECOVER_AFTER+ } else {+ RECOVER_AFTER.saturating_mul(state.limit.max(1))+ };+ if state.ok_streak < needed {return;}state.ok_streak = 0;@@ -1412,9 +1443,6 @@ impl Es {}fn bulk_bounded(&self, body: Vec<u8>, max_bytes: usize) -> Result<BulkOutcome> {- // One admission slot for the whole body: its windows go out one after- // another, so this worker never offers more than one request at once.- let _permit = self.admission.acquire();let mut pending = std::collections::VecDeque::from([body]);let mut total = BulkOutcome::default();while let Some(next) = pending.pop_front() {@@ -1619,6 +1647,10 @@ impl Es {// breaker happened to answer with.let throttles = std::cell::Cell::new(0u64);let round = self.retry_loop("_bulk", || {+ // Acquire the admission permit per HTTP attempt rather than across+ // the whole backoff loop (#1250), so throttled workers drop their+ // slot before sleeping and immediately obey a halved window limit.+ let _permit = self.admission.acquire();let resp = match self.send_bulk(body.clone()) {Ok(resp) => resp,Err(e) => return Attempt::Failed(e),@@ -3978,4 +4010,48 @@ mod mapping_refusal_tests {assert_eq!(rendered, format!("PUT /ax/_mapping failed: {status} body"));}}++ #[test]+ fn sustained_429_episode_halves_once_and_congestion_avoidance_slows_climb_near_ceiling() {+ let admission = super::BulkAdmission::new(32, false);+ admission.on_congestion();+ assert_eq!(admission.limit(), 16);+ assert_eq!(admission.congestion_events(), 1);++ // Simulate a 3 s parent-breaker trip where no bulk succeeds: backdate+ // `last_shrink` past `SHRINK_COOLDOWN` (1 s) but within+ // `EPISODE_MAX_COOLDOWN` (15 s). Repeated 429s must NOT cascade 16 → 8 → 4 → 2 → 1.+ admission.inner.lock().unwrap().last_shrink =+ Some(std::time::Instant::now() - std::time::Duration::from_secs(3));+ admission.on_congestion();+ assert_eq!(+ admission.limit(),+ 16,+ "sustained 429s within one breaker episode must not cascade-halve past the first cut"+ );+ assert_eq!(admission.congestion_events(), 1);++ // Below ssthresh (24), each +1 step takes RECOVER_AFTER (8) clean bulks.+ for expected in 17..=24 {+ for _ in 0..super::RECOVER_AFTER {+ admission.on_success();+ }+ assert_eq!(admission.limit(), expected);+ }++ // At ssthresh (24), congestion avoidance kicks in: climbing 24 → 25+ // requires RECOVER_AFTER * 24 clean bulks, not just 8.+ for _ in 0..super::RECOVER_AFTER {+ admission.on_success();+ }+ assert_eq!(+ admission.limit(),+ 24,+ "at or above ssthresh, 8 clean bulks must not immediately raise concurrency"+ );+ for _ in 0..(super::RECOVER_AFTER * 23) {+ admission.on_success();+ }+ assert_eq!(admission.limit(), 25);+ }}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex b70bbe4cf..ddf89c192 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -3908,7 +3908,7 @@ impl Engine {/// released.pub async fn release_rebuildable_memory(&self, indices: &[Arc<Index>]) -> usize {for idx in indices {- if idx.flush().await.is_err() {+ if idx.memtable_bytes() > 0 && idx.flush().await.is_err() {tracing::warn!("memory release: flush failed, releasing caches anyway");}idx.release_memory();diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 15eb61297..231fab6bc 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -362,10 +362,17 @@ impl ResourceGovernor {/// `0` when it isn't available (e.g. from the [`Self::refresh`] test/// wrapper); it only affects that one log line, never admission.pub fn refresh_memory_disk(&self, rss: u64, disk_used_pct: u64, disk_avail_bytes: u64) {- // ── RSS watermark ──+ // ── RSS watermark (5% release hysteresis to prevent rapid 429 flapping, #1250) ──self.rss_bytes.store(rss, Ordering::Relaxed);let watermark = self.memory_watermark_bytes.load(Ordering::Relaxed);- let mem_next = watermark != 0 && rss >= watermark;+ let cur_mem = self.memory_tripped.load(Ordering::Relaxed);+ let release_bytes = watermark - (watermark / 20);+ let mem_next = watermark != 0+ && if cur_mem {+ rss >= release_bytes+ } else {+ rss >= watermark+ };if mem_next != self.memory_tripped.swap(mem_next, Ordering::Relaxed) {if mem_next {tracing::warn!(@@ -2025,4 +2032,31 @@ mod tests {assert_eq!(g.search_inflight_peak(), 2, "concurrency never exceeded 2");drop(p2);}++ #[test]+ fn memory_watermark_releases_with_hysteresis() {+ let mut cfg = Config::default();+ cfg.limits.max_total_memtable_mb = 0;+ cfg.limits.memory_watermark_percent = 95;+ let g = build(&cfg);+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);+ assert!(wm > 0, "this test needs a live RSS watermark");++ g.refresh(0, wm, 0);+ assert!(g.check_ingest_admission().is_err(), "trips at watermark");++ // 98% of watermark is above the 95% release threshold: stays tripped.+ g.refresh(0, wm * 98 / 100, 0);+ assert!(+ g.check_ingest_admission().is_err(),+ "must remain tripped until RSS drops 5% below watermark"+ );++ // 94% of watermark is below the 95% release threshold: releases.+ g.refresh(0, wm * 94 / 100, 0);+ assert!(+ g.check_ingest_admission().is_ok(),+ "must release once RSS drops below 95% of watermark"+ );+ }}