AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(governor): let a small merge write run after a yield streak below the watermark (#2722)
xerj-orgb1678d1f130a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a1bb8a10d..8ddf475bc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- A node whose resting memory sits between the merge-pool ceiling and the+ memory watermark merges again. Every background merge's output write was+ refused there, so small batches were retried every 5 s without ever+ completing, and each retry decoded its inputs again. After three yields in+ a row with nothing merged, a batch of at most 64 MB now checks its write+ against the process memory limit instead of the merge-pool ceiling.+ (Refs #2427)- `_bulk` throughput with explicit `_id`s: `index` actions that carry an `_id`(and prepared `semantic_text` documents) are now published as one batch perindex instead of one `index_document` call each, which took the WAL lock,diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 94962712d..aed689dfd 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -904,15 +904,38 @@ impl ResourceGovernor {extra: u64,forced: bool,exempt: bool,+ ) -> bool {+ self.admit_merge_write_for(reservation, held, extra, forced, exempt, 0)+ }++ /// [`Self::admit_merge_write`] for a batch of `input_bytes`: after a+ /// streak of yields with nothing merged, a small background batch's+ /// write is checked against the hard process limit instead of the+ /// merge-pool ceiling, so a node resting between the two still merges+ /// (#2427). See `merge_budget::merge_write_limit_after`.+ pub fn admit_merge_write_for(+ &self,+ reservation: Option<&mut MergeReservation>,+ held: u64,+ extra: u64,+ forced: bool,+ exempt: bool,+ input_bytes: u64,) -> bool {let observed = self.observed_process_memory();- if !crate::merge_budget::decide_merge_write(- observed,+ let streak = self+ .merge_ledger+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .yield_streak;+ let line = crate::merge_budget::merge_write_limit_after(self.merge_pool_ceiling_bytes(),- extra,+ self.memory_limit_bytes,forced,- exempt,- ) {+ input_bytes,+ streak,+ );+ if !crate::merge_budget::decide_merge_write(observed, line, extra, forced, exempt) {return false;}if let Some(reservation) = reservation {@@ -4694,6 +4717,57 @@ mod tests {assert_eq!(g.merges_in_flight(), 0);}+ /// #2427: in the 1 GB gate the search node rested between the merge-pool+ /// ceiling and the watermark, so the breaker never engaged, the #2468+ /// exemption never applied, and every write step was refused: 70 yields+ /// of 1-10 MB batches and nothing merged. After the streak, a small+ /// batch's write is checked against the hard limit and runs.+ #[test]+ fn a_small_batch_write_runs_after_a_streak_below_the_watermark() {+ const MIB: u64 = 1024 * 1024;+ let mut cfg = Config::default();+ cfg.limits.max_total_memtable_mb = 0;+ cfg.limits.memory_watermark_percent = 95;+ let g = build(&cfg);+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);+ let limit = g.process_memory_limit_bytes();+ let ceiling = g.merge_pool_ceiling_bytes();+ assert!(+ ceiling < wm && wm < limit,+ "this test needs ceiling < watermark < limit"+ );+ // Resting above the ceiling, below the watermark: not engaged.+ let rss = ceiling + (wm - ceiling) / 2;+ g.refresh(0, rss, 0);+ assert!(!g.memory_breaker_engaged());+ let small = 6 * MIB;+ let large = crate::merge_budget::MERGE_YIELD_STREAK_MAX_INPUT_BYTES + 1;+ // Fits under the hard limit, not under the ceiling.+ let write = (limit - rss) / 2;+ assert!(!g.merge_write_exempt(false, small));++ // Before the streak: refused, as before.+ assert!(!g.admit_merge_write_for(None, small, write, false, false, small));+ for _ in 0..crate::merge_budget::MERGE_YIELD_STREAK_LIMIT {+ g.note_merge_yield(small);+ }+ // The old exemption still does not apply below the watermark ...+ assert!(!g.merge_write_exempt(false, small));+ // ... but the small batch's write now runs, charged to the ledger.+ let mut reservation = g.try_reserve_merge(small, true).expect("forced admission");+ let held = reservation.bytes();+ assert!(g.admit_merge_write_for(Some(&mut reservation), held, write, false, false, small));+ assert_eq!(reservation.bytes(), held + write);+ drop(reservation);+ // A large batch, or a write that would reach the hard limit, is+ // still refused.+ assert!(!g.admit_merge_write_for(None, large, write, false, false, large));+ assert!(!g.admit_merge_write_for(None, small, limit - rss, false, false, small));+ // A completed batch ends the streak.+ g.note_merge_completed(small);+ assert!(!g.admit_merge_write_for(None, small, write, false, false, small));+ }+/// #2427 x #2468: a small background batch that the yield-streak rule/// lets run to completion is not refused at its output write, even with/// RSS at the hard limit. A large batch, a batch before the streak, anddiff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex f1e3f1315..7e7d72f32 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -14618,7 +14618,14 @@ impl Index {// rule is what waived this batch's yield, the write runs// too, or refusing it would bring the stall back.let exempt = governor.merge_write_exempt(forced_for_task, input_bytes_for_task);- if governor.admit_merge_write(reservation, held, extra, forced_for_task, exempt) {+ if governor.admit_merge_write_for(+ reservation,+ held,+ extra,+ forced_for_task,+ exempt,+ input_bytes_for_task,+ ) {return true;}let cap = governor.note_merge_yield(input_bytes_for_task);diff --git a/engine/crates/xerj-engine/src/merge_budget.rs b/engine/crates/xerj-engine/src/merge_budget.rsindex 901529447..0a256edda 100644--- a/engine/crates/xerj-engine/src/merge_budget.rs+++ b/engine/crates/xerj-engine/src/merge_budget.rs@@ -400,6 +400,43 @@ pub fn effective_batch_cap_bytes(static_cap: u64, pressure_cap: u64) -> u64 {}}+/// The memory line a background merge's output-write step is checked+/// against (#2427, #2468).+///+/// Normally that is the merge-pool `ceiling` (the process limit less the+/// search-read reserve). After [`MERGE_YIELD_STREAK_LIMIT`] yields in a row+/// with nothing merged, a batch of at most+/// [`MERGE_YIELD_STREAK_MAX_INPUT_BYTES`] is checked against the hard+/// process `limit` instead.+///+/// Without this a search node whose resting RSS sits between the ceiling+/// and the watermark never merges. In the #2427 1 GB gate the search+/// node's floor was 2.3-2.5 GB against a 2,458 MB ceiling, so every write+/// step was refused (70 yields of 1-10 MB batches, retried every 5 s, none+/// completed). The #2468 streak exemption did not apply, because it only+/// covers batches the in-flight yield rule stops, and that rule needs RSS+/// at the watermark. Each retry decoded its inputs again, which kept RSS+/// near the watermark. The hard limit still bounds the write, and searches+/// are refused only at that limit.+pub fn merge_write_limit_after(+ ceiling: u64,+ limit: u64,+ forced: bool,+ input_bytes: u64,+ yield_streak: u64,+) -> u64 {+ if !forced+ && limit > ceiling+ && yield_streak >= MERGE_YIELD_STREAK_LIMIT+ && input_bytes > 0+ && input_bytes <= MERGE_YIELD_STREAK_MAX_INPUT_BYTES+ {+ limit+ } else {+ ceiling+ }+}+/// Multiplier on the merged stored JSON for the merge output-write step/// (#2427).///@@ -471,6 +508,51 @@ pub fn decide_merge_write(mod tests {use super::*;+ #[test]+ fn a_small_batch_write_uses_the_hard_limit_after_a_yield_streak() {+ const MIB: u64 = 1024 * 1024;+ let ceiling = 2458 * MIB;+ let limit = 3072 * MIB;+ let small = 6 * MIB;+ // Before the streak: the merge-pool ceiling.+ for streak in 0..MERGE_YIELD_STREAK_LIMIT {+ assert_eq!(+ merge_write_limit_after(ceiling, limit, false, small, streak),+ ceiling+ );+ }+ // At the streak: the hard limit, for a small background batch only.+ let at = MERGE_YIELD_STREAK_LIMIT;+ assert_eq!(+ merge_write_limit_after(ceiling, limit, false, small, at),+ limit+ );+ let large = MERGE_YIELD_STREAK_MAX_INPUT_BYTES + 1;+ assert_eq!(+ merge_write_limit_after(ceiling, limit, false, large, at),+ ceiling+ );+ assert_eq!(+ merge_write_limit_after(ceiling, limit, false, 0, at),+ ceiling+ );+ assert_eq!(+ merge_write_limit_after(ceiling, limit, true, small, at),+ ceiling+ );+ // Never below the ceiling.+ assert_eq!(+ merge_write_limit_after(ceiling, 0, false, small, at),+ ceiling+ );+ // The gate-f case: rss 2,517 MB plus a 408 MB write fits under the+ // hard limit but not under the ceiling.+ let (rss, extra) = (2517 * MIB, 408 * MIB);+ assert!(!decide_merge_write(rss, ceiling, extra, false, false));+ let line = merge_write_limit_after(ceiling, limit, false, small, at);+ assert!(decide_merge_write(rss, line, extra, false, false));+ }+#[test]fn a_background_merge_yields_only_when_engaged_at_the_limit() {const MIB: u64 = 1024 * 1024;