Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex,engine): damp bulk AIMD oscillation and memory-breaker drain churn (#1250)

Christian M. Todie757e00daeb5e
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 14ca476eb..3dcb23087 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -150,7 +150,13 @@ pub struct BulkAdmission {
/// One congestion event may be reported by every worker that was in flight
/// when the server pushed back; they must count once.
const SHRINK_COOLDOWN: Duration = Duration::from_secs(1);
-/// Clean bulks required before the limit is probed upward by one.
+/// While no bulk has succeeded since the last shrink, coalesce repeated 429
+/// retries from the same parent-breaker trip into a single halving for up to
+/// 15 s (issue #1250) instead of cascading 32 → 16 → 8 → 4 → 2 → 1 once per
+/// second while the server drains.
+const EPISODE_MAX_COOLDOWN: Duration = Duration::from_secs(15);
+/// Clean bulks required before the limit is probed upward by one below
+/// `ssthresh` (above `ssthresh`, congestion avoidance scales this by `limit`).
const RECOVER_AFTER: usize = 8;
/// Minimum gap between two "raising bulk concurrency" lines on stderr.
@@ -166,6 +172,14 @@ struct AdmissionState {
in_flight: usize,
ok_streak: usize,
last_shrink: Option<Instant>,
+ /// True after a shrink until at least one bulk succeeds, so sustained 429
+ /// retries during a single breaker trip do not halve repeatedly every 1 s.
+ shrunk_in_episode: bool,
+ /// Slow-start threshold: below `ssthresh`, recovery steps every
+ /// `RECOVER_AFTER` bulks; at or above `ssthresh` (near the concurrency that
+ /// last tripped the breaker), congestion avoidance requires
+ /// `RECOVER_AFTER * limit` clean bulks per +1 step.
+ ssthresh: usize,
/// When the last "raising" line was printed, and the limit it named —
/// so the next line can name the whole climb since.
last_raise_announce: Option<Instant>,
@@ -205,6 +219,8 @@ impl BulkAdmission {
in_flight: 0,
ok_streak: 0,
last_shrink: None,
+ shrunk_in_episode: false,
+ ssthresh: ceiling,
last_raise_announce: None,
announced_limit: ceiling,
}),
@@ -266,11 +282,20 @@ impl BulkAdmission {
{
return;
}
+ if state.shrunk_in_episode
+ && state
+ .last_shrink
+ .is_some_and(|last| now.duration_since(last) < EPISODE_MAX_COOLDOWN)
+ {
+ return;
+ }
state.last_shrink = Some(now);
state.ok_streak = 0;
+ state.shrunk_in_episode = true;
self.congestion_events.fetch_add(1, Ordering::Relaxed);
let previous = state.limit;
state.limit = (previous / 2).max(1);
+ state.ssthresh = (previous * 3 / 4).max(state.limit + 1);
if state.limit != previous && self.announce {
eprintln!(
"autoindex: server pushed back (HTTP 429); lowering bulk concurrency \
@@ -290,11 +315,17 @@ impl BulkAdmission {
return;
}
let mut state = self.inner.lock().unwrap();
+ state.shrunk_in_episode = false;
if state.limit >= self.ceiling {
return;
}
state.ok_streak += 1;
- if state.ok_streak < RECOVER_AFTER {
+ let needed = if state.limit < state.ssthresh {
+ RECOVER_AFTER
+ } else {
+ RECOVER_AFTER.saturating_mul(state.limit.max(1))
+ };
+ if state.ok_streak < needed {
return;
}
state.ok_streak = 0;
@@ -1412,9 +1443,6 @@ impl Es {
}
fn bulk_bounded(&self, body: Vec<u8>, max_bytes: usize) -> Result<BulkOutcome> {
- // One admission slot for the whole body: its windows go out one after
- // another, so this worker never offers more than one request at once.
- let _permit = self.admission.acquire();
let mut pending = std::collections::VecDeque::from([body]);
let mut total = BulkOutcome::default();
while let Some(next) = pending.pop_front() {
@@ -1619,6 +1647,10 @@ impl Es {
// breaker happened to answer with.
let throttles = std::cell::Cell::new(0u64);
let round = self.retry_loop("_bulk", || {
+ // Acquire the admission permit per HTTP attempt rather than across
+ // the whole backoff loop (#1250), so throttled workers drop their
+ // slot before sleeping and immediately obey a halved window limit.
+ let _permit = self.admission.acquire();
let resp = match self.send_bulk(body.clone()) {
Ok(resp) => resp,
Err(e) => return Attempt::Failed(e),
@@ -3978,4 +4010,48 @@ mod mapping_refusal_tests {
assert_eq!(rendered, format!("PUT /ax/_mapping failed: {status} body"));
}
}
+
+ #[test]
+ fn sustained_429_episode_halves_once_and_congestion_avoidance_slows_climb_near_ceiling() {
+ let admission = super::BulkAdmission::new(32, false);
+ admission.on_congestion();
+ assert_eq!(admission.limit(), 16);
+ assert_eq!(admission.congestion_events(), 1);
+
+ // Simulate a 3 s parent-breaker trip where no bulk succeeds: backdate
+ // `last_shrink` past `SHRINK_COOLDOWN` (1 s) but within
+ // `EPISODE_MAX_COOLDOWN` (15 s). Repeated 429s must NOT cascade 16 → 8 → 4 → 2 → 1.
+ admission.inner.lock().unwrap().last_shrink =
+ Some(std::time::Instant::now() - std::time::Duration::from_secs(3));
+ admission.on_congestion();
+ assert_eq!(
+ admission.limit(),
+ 16,
+ "sustained 429s within one breaker episode must not cascade-halve past the first cut"
+ );
+ assert_eq!(admission.congestion_events(), 1);
+
+ // Below ssthresh (24), each +1 step takes RECOVER_AFTER (8) clean bulks.
+ for expected in 17..=24 {
+ for _ in 0..super::RECOVER_AFTER {
+ admission.on_success();
+ }
+ assert_eq!(admission.limit(), expected);
+ }
+
+ // At ssthresh (24), congestion avoidance kicks in: climbing 24 → 25
+ // requires RECOVER_AFTER * 24 clean bulks, not just 8.
+ for _ in 0..super::RECOVER_AFTER {
+ admission.on_success();
+ }
+ assert_eq!(
+ admission.limit(),
+ 24,
+ "at or above ssthresh, 8 clean bulks must not immediately raise concurrency"
+ );
+ for _ in 0..(super::RECOVER_AFTER * 23) {
+ admission.on_success();
+ }
+ assert_eq!(admission.limit(), 25);
+ }
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index b70bbe4cf..ddf89c192 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -3908,7 +3908,7 @@ impl Engine {
/// released.
pub async fn release_rebuildable_memory(&self, indices: &[Arc<Index>]) -> usize {
for idx in indices {
- if idx.flush().await.is_err() {
+ if idx.memtable_bytes() > 0 && idx.flush().await.is_err() {
tracing::warn!("memory release: flush failed, releasing caches anyway");
}
idx.release_memory();
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 15eb61297..231fab6bc 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -362,10 +362,17 @@ impl ResourceGovernor {
/// `0` when it isn't available (e.g. from the [`Self::refresh`] test
/// wrapper); it only affects that one log line, never admission.
pub fn refresh_memory_disk(&self, rss: u64, disk_used_pct: u64, disk_avail_bytes: u64) {
- // ── RSS watermark ──
+ // ── RSS watermark (5% release hysteresis to prevent rapid 429 flapping, #1250) ──
self.rss_bytes.store(rss, Ordering::Relaxed);
let watermark = self.memory_watermark_bytes.load(Ordering::Relaxed);
- let mem_next = watermark != 0 && rss >= watermark;
+ let cur_mem = self.memory_tripped.load(Ordering::Relaxed);
+ let release_bytes = watermark - (watermark / 20);
+ let mem_next = watermark != 0
+ && if cur_mem {
+ rss >= release_bytes
+ } else {
+ rss >= watermark
+ };
if mem_next != self.memory_tripped.swap(mem_next, Ordering::Relaxed) {
if mem_next {
tracing::warn!(
@@ -2025,4 +2032,31 @@ mod tests {
assert_eq!(g.search_inflight_peak(), 2, "concurrency never exceeded 2");
drop(p2);
}
+
+ #[test]
+ fn memory_watermark_releases_with_hysteresis() {
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 0;
+ cfg.limits.memory_watermark_percent = 95;
+ let g = build(&cfg);
+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);
+ assert!(wm > 0, "this test needs a live RSS watermark");
+
+ g.refresh(0, wm, 0);
+ assert!(g.check_ingest_admission().is_err(), "trips at watermark");
+
+ // 98% of watermark is above the 95% release threshold: stays tripped.
+ g.refresh(0, wm * 98 / 100, 0);
+ assert!(
+ g.check_ingest_admission().is_err(),
+ "must remain tripped until RSS drops 5% below watermark"
+ );
+
+ // 94% of watermark is below the 95% release threshold: releases.
+ g.refresh(0, wm * 94 / 100, 0);
+ assert!(
+ g.check_ingest_admission().is_ok(),
+ "must release once RSS drops below 95% of watermark"
+ );
+ }
}