Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(governor): let a small merge write run after a yield streak below the watermark (#2722)

xerj-orgb1678d1f130a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a1bb8a10d..8ddf475bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- A node whose resting memory sits between the merge-pool ceiling and the
+ memory watermark merges again. Every background merge's output write was
+ refused there, so small batches were retried every 5 s without ever
+ completing, and each retry decoded its inputs again. After three yields in
+ a row with nothing merged, a batch of at most 64 MB now checks its write
+ against the process memory limit instead of the merge-pool ceiling.
+ (Refs #2427)
- `_bulk` throughput with explicit `_id`s: `index` actions that carry an `_id`
(and prepared `semantic_text` documents) are now published as one batch per
index instead of one `index_document` call each, which took the WAL lock,
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 94962712d..aed689dfd 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -904,15 +904,38 @@ impl ResourceGovernor {
extra: u64,
forced: bool,
exempt: bool,
+ ) -> bool {
+ self.admit_merge_write_for(reservation, held, extra, forced, exempt, 0)
+ }
+
+ /// [`Self::admit_merge_write`] for a batch of `input_bytes`: after a
+ /// streak of yields with nothing merged, a small background batch's
+ /// write is checked against the hard process limit instead of the
+ /// merge-pool ceiling, so a node resting between the two still merges
+ /// (#2427). See `merge_budget::merge_write_limit_after`.
+ pub fn admit_merge_write_for(
+ &self,
+ reservation: Option<&mut MergeReservation>,
+ held: u64,
+ extra: u64,
+ forced: bool,
+ exempt: bool,
+ input_bytes: u64,
) -> bool {
let observed = self.observed_process_memory();
- if !crate::merge_budget::decide_merge_write(
- observed,
+ let streak = self
+ .merge_ledger
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .yield_streak;
+ let line = crate::merge_budget::merge_write_limit_after(
self.merge_pool_ceiling_bytes(),
- extra,
+ self.memory_limit_bytes,
forced,
- exempt,
- ) {
+ input_bytes,
+ streak,
+ );
+ if !crate::merge_budget::decide_merge_write(observed, line, extra, forced, exempt) {
return false;
}
if let Some(reservation) = reservation {
@@ -4694,6 +4717,57 @@ mod tests {
assert_eq!(g.merges_in_flight(), 0);
}
+ /// #2427: in the 1 GB gate the search node rested between the merge-pool
+ /// ceiling and the watermark, so the breaker never engaged, the #2468
+ /// exemption never applied, and every write step was refused: 70 yields
+ /// of 1-10 MB batches and nothing merged. After the streak, a small
+ /// batch's write is checked against the hard limit and runs.
+ #[test]
+ fn a_small_batch_write_runs_after_a_streak_below_the_watermark() {
+ const MIB: u64 = 1024 * 1024;
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 0;
+ cfg.limits.memory_watermark_percent = 95;
+ let g = build(&cfg);
+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);
+ let limit = g.process_memory_limit_bytes();
+ let ceiling = g.merge_pool_ceiling_bytes();
+ assert!(
+ ceiling < wm && wm < limit,
+ "this test needs ceiling < watermark < limit"
+ );
+ // Resting above the ceiling, below the watermark: not engaged.
+ let rss = ceiling + (wm - ceiling) / 2;
+ g.refresh(0, rss, 0);
+ assert!(!g.memory_breaker_engaged());
+ let small = 6 * MIB;
+ let large = crate::merge_budget::MERGE_YIELD_STREAK_MAX_INPUT_BYTES + 1;
+ // Fits under the hard limit, not under the ceiling.
+ let write = (limit - rss) / 2;
+ assert!(!g.merge_write_exempt(false, small));
+
+ // Before the streak: refused, as before.
+ assert!(!g.admit_merge_write_for(None, small, write, false, false, small));
+ for _ in 0..crate::merge_budget::MERGE_YIELD_STREAK_LIMIT {
+ g.note_merge_yield(small);
+ }
+ // The old exemption still does not apply below the watermark ...
+ assert!(!g.merge_write_exempt(false, small));
+ // ... but the small batch's write now runs, charged to the ledger.
+ let mut reservation = g.try_reserve_merge(small, true).expect("forced admission");
+ let held = reservation.bytes();
+ assert!(g.admit_merge_write_for(Some(&mut reservation), held, write, false, false, small));
+ assert_eq!(reservation.bytes(), held + write);
+ drop(reservation);
+ // A large batch, or a write that would reach the hard limit, is
+ // still refused.
+ assert!(!g.admit_merge_write_for(None, large, write, false, false, large));
+ assert!(!g.admit_merge_write_for(None, small, limit - rss, false, false, small));
+ // A completed batch ends the streak.
+ g.note_merge_completed(small);
+ assert!(!g.admit_merge_write_for(None, small, write, false, false, small));
+ }
+
/// #2427 x #2468: a small background batch that the yield-streak rule
/// lets run to completion is not refused at its output write, even with
/// RSS at the hard limit. A large batch, a batch before the streak, and
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index f1e3f1315..7e7d72f32 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -14618,7 +14618,14 @@ impl Index {
// rule is what waived this batch's yield, the write runs
// too, or refusing it would bring the stall back.
let exempt = governor.merge_write_exempt(forced_for_task, input_bytes_for_task);
- if governor.admit_merge_write(reservation, held, extra, forced_for_task, exempt) {
+ if governor.admit_merge_write_for(
+ reservation,
+ held,
+ extra,
+ forced_for_task,
+ exempt,
+ input_bytes_for_task,
+ ) {
return true;
}
let cap = governor.note_merge_yield(input_bytes_for_task);
diff --git a/engine/crates/xerj-engine/src/merge_budget.rs b/engine/crates/xerj-engine/src/merge_budget.rs
index 901529447..0a256edda 100644
--- a/engine/crates/xerj-engine/src/merge_budget.rs
+++ b/engine/crates/xerj-engine/src/merge_budget.rs
@@ -400,6 +400,43 @@ pub fn effective_batch_cap_bytes(static_cap: u64, pressure_cap: u64) -> u64 {
}
}
+/// The memory line a background merge's output-write step is checked
+/// against (#2427, #2468).
+///
+/// Normally that is the merge-pool `ceiling` (the process limit less the
+/// search-read reserve). After [`MERGE_YIELD_STREAK_LIMIT`] yields in a row
+/// with nothing merged, a batch of at most
+/// [`MERGE_YIELD_STREAK_MAX_INPUT_BYTES`] is checked against the hard
+/// process `limit` instead.
+///
+/// Without this a search node whose resting RSS sits between the ceiling
+/// and the watermark never merges. In the #2427 1 GB gate the search
+/// node's floor was 2.3-2.5 GB against a 2,458 MB ceiling, so every write
+/// step was refused (70 yields of 1-10 MB batches, retried every 5 s, none
+/// completed). The #2468 streak exemption did not apply, because it only
+/// covers batches the in-flight yield rule stops, and that rule needs RSS
+/// at the watermark. Each retry decoded its inputs again, which kept RSS
+/// near the watermark. The hard limit still bounds the write, and searches
+/// are refused only at that limit.
+pub fn merge_write_limit_after(
+ ceiling: u64,
+ limit: u64,
+ forced: bool,
+ input_bytes: u64,
+ yield_streak: u64,
+) -> u64 {
+ if !forced
+ && limit > ceiling
+ && yield_streak >= MERGE_YIELD_STREAK_LIMIT
+ && input_bytes > 0
+ && input_bytes <= MERGE_YIELD_STREAK_MAX_INPUT_BYTES
+ {
+ limit
+ } else {
+ ceiling
+ }
+}
+
/// Multiplier on the merged stored JSON for the merge output-write step
/// (#2427).
///
@@ -471,6 +508,51 @@ pub fn decide_merge_write(
mod tests {
use super::*;
+ #[test]
+ fn a_small_batch_write_uses_the_hard_limit_after_a_yield_streak() {
+ const MIB: u64 = 1024 * 1024;
+ let ceiling = 2458 * MIB;
+ let limit = 3072 * MIB;
+ let small = 6 * MIB;
+ // Before the streak: the merge-pool ceiling.
+ for streak in 0..MERGE_YIELD_STREAK_LIMIT {
+ assert_eq!(
+ merge_write_limit_after(ceiling, limit, false, small, streak),
+ ceiling
+ );
+ }
+ // At the streak: the hard limit, for a small background batch only.
+ let at = MERGE_YIELD_STREAK_LIMIT;
+ assert_eq!(
+ merge_write_limit_after(ceiling, limit, false, small, at),
+ limit
+ );
+ let large = MERGE_YIELD_STREAK_MAX_INPUT_BYTES + 1;
+ assert_eq!(
+ merge_write_limit_after(ceiling, limit, false, large, at),
+ ceiling
+ );
+ assert_eq!(
+ merge_write_limit_after(ceiling, limit, false, 0, at),
+ ceiling
+ );
+ assert_eq!(
+ merge_write_limit_after(ceiling, limit, true, small, at),
+ ceiling
+ );
+ // Never below the ceiling.
+ assert_eq!(
+ merge_write_limit_after(ceiling, 0, false, small, at),
+ ceiling
+ );
+ // The gate-f case: rss 2,517 MB plus a 408 MB write fits under the
+ // hard limit but not under the ceiling.
+ let (rss, extra) = (2517 * MIB, 408 * MIB);
+ assert!(!decide_merge_write(rss, ceiling, extra, false, false));
+ let line = merge_write_limit_after(ceiling, limit, false, small, at);
+ assert!(decide_merge_write(rss, line, extra, false, false));
+ }
+
#[test]
fn a_background_merge_yields_only_when_engaged_at_the_limit() {
const MIB: u64 = 1024 * 1024;