Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(merge): charge the output-write step to the merge budget before it runs (Refs #2427) (#2470)

xerj-org768f2471e29c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 410603866..a128ee403 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -79,6 +79,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- A segment merge's output-write step (doc-values build, stored encode, FTS
+ side-car build) is now charged to the merge budget before it starts, and a
+ background batch yields instead of starting it when it would take RSS to
+ the process limit (#2427). That step runs after the last yield checkpoint,
+ and neither admission nor the yield checks saw it: large pg-docs merges
+ (29k-166k docs) took a 3,072 MB search node to 4.4-4.9 GB about 5 s before
+ they completed. The step is estimated at 6x the merged stored JSON or 2x
+ the FTS build input, whichever is larger. An admitted step grows the
+ batch's merge reservation, so concurrent merge admission sees it.
+ `_forcemerge` batches are charged but never refused, and neither is a
+ small batch that the yield-streak rule (#2468) lets run to completion.
+
- The parent memory breaker no longer flaps, and background merges and flush
finalizes are bounded while it is engaged (#2427). A 3,000 MB autoindex run
engaged the breaker 20 times in 16 minutes, often releasing 100-500 ms
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 5e032bf94..6c250a35c 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -769,6 +769,65 @@ impl ResourceGovernor {
.pressure_cap
}
+ /// Whether a batch's output-write step is exempt from
+ /// [`Self::admit_merge_write`]'s hard-limit refusal (#2427, #2468).
+ ///
+ /// True exactly when #2468's streak rule is what keeps this batch
+ /// running: the plain rule ([`Self::merge_must_yield`]) says yield, and
+ /// the streak-aware rule ([`Self::merge_must_yield_for`]) waives it for
+ /// a small batch after a streak of yields with nothing merged. Refusing
+ /// that batch's write would bring back the stall the rule exists to
+ /// break.
+ pub fn merge_write_exempt(&self, forced: bool, input_bytes: u64) -> bool {
+ self.merge_must_yield(forced) && !self.merge_must_yield_for(forced, input_bytes)
+ }
+
+ /// Charge a merge batch's output-write step to the merge budget before
+ /// it runs (#2427).
+ ///
+ /// The step comes after the last yield checkpoint: the doc-values
+ /// build, the stored encode and the FTS side-car build. It allocates
+ /// `extra` bytes (`merge_budget::merge_write_phase_bytes`) on top of the
+ /// `held` bytes the batch already has. Before this, neither admission
+ /// nor the yield checks saw it, and large pg-docs merges took a
+ /// 3,072 MB search node to 4.4–4.9 GB in their last seconds.
+ ///
+ /// Returns `false` when the step would take observed process memory to
+ /// the hard limit (`merge_budget::decide_merge_write`). The caller then
+ /// yields the batch, keeping its inputs. On `true`, `reservation` grows
+ /// to at least `held + extra`, so admission for other merges sees the
+ /// step until the batch drops its reservation. A refused step leaves
+ /// the ledger as it was. `exempt` covers batches that #2468's streak
+ /// rule lets run to completion.
+ pub fn admit_merge_write(
+ &self,
+ reservation: Option<&mut MergeReservation>,
+ held: u64,
+ extra: u64,
+ forced: bool,
+ exempt: bool,
+ ) -> bool {
+ let observed = self.observed_process_memory();
+ if !crate::merge_budget::decide_merge_write(
+ observed,
+ self.memory_limit_bytes,
+ extra,
+ forced,
+ exempt,
+ ) {
+ return false;
+ }
+ if let Some(reservation) = reservation {
+ let wanted = held.saturating_add(extra);
+ if wanted > reservation.bytes {
+ let mut ledger = self.merge_ledger.lock().unwrap_or_else(|e| e.into_inner());
+ ledger.reserved = ledger.reserved.saturating_add(wanted - reservation.bytes);
+ reservation.bytes = wanted;
+ }
+ }
+ true
+ }
+
/// Whether the summed-memtable budget breaker is currently engaged.
pub fn memtable_breaker_engaged(&self) -> bool {
self.memtable_tripped.load(Ordering::Relaxed)
@@ -4090,6 +4149,129 @@ mod tests {
assert!(g.merge_must_yield_for(false, 49 * MIB));
}
+ /// #2427: the merge output-write step (doc-values build, stored encode,
+ /// FTS build) runs after the last yield checkpoint. It is charged to the
+ /// merge ledger before it starts. A step that would take RSS to the hard
+ /// limit is refused and leaves the ledger as it was. An admitted step
+ /// grows the batch's reservation, which other merges' admission sees,
+ /// and dropping the reservation releases all of it.
+ #[test]
+ fn the_merge_write_step_is_charged_and_refused_at_the_hard_limit() {
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 0;
+ cfg.limits.memory_watermark_percent = 95;
+ let g = build(&cfg);
+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);
+ let limit = g.process_memory_limit_bytes();
+ assert!(wm > 0 && limit > wm, "this test needs a live RSS watermark");
+
+ // RSS at half the watermark: one merge admitted.
+ g.refresh(0, wm / 2, 0);
+ let input = wm / 16 / crate::merge_budget::MERGE_WORKING_SET_NUMERATOR;
+ let mut reservation = g
+ .try_reserve_merge(input, false)
+ .expect("first merge starts");
+ let admitted = reservation.bytes();
+ assert_eq!(g.merge_reserved_bytes(), admitted);
+
+ // A write that fits under the limit is admitted and charged.
+ let held = admitted;
+ let extra = wm / 8;
+ assert!(g.admit_merge_write(Some(&mut reservation), held, extra, false, false));
+ assert_eq!(reservation.bytes(), held + extra);
+ assert_eq!(g.merge_reserved_bytes(), held + extra);
+ // Charging again for a smaller step never shrinks the reservation.
+ assert!(g.admit_merge_write(Some(&mut reservation), 1, 1, false, false));
+ assert_eq!(g.merge_reserved_bytes(), held + extra);
+
+ // The charge is visible to admission: a second batch that fit
+ // beside the admitted estimate no longer fits beside the write.
+ let second = (wm / 2 - admitted) / crate::merge_budget::MERGE_WORKING_SET_NUMERATOR - 1;
+ assert!(wm / 2 + admitted + second * crate::merge_budget::MERGE_WORKING_SET_NUMERATOR < wm);
+ assert!(g.try_reserve_merge(second, false).is_none());
+
+ // A write that would reach the hard limit is refused, and the
+ // ledger does not change, background or not engaged.
+ let before = g.merge_reserved_bytes();
+ let too_big = limit - wm / 2;
+ assert!(!g.admit_merge_write(Some(&mut reservation), held, too_big, false, false));
+ assert_eq!(g.merge_reserved_bytes(), before);
+ assert_eq!(reservation.bytes(), held + extra);
+ // No reservation (no governor at launch): only the decision.
+ assert!(!g.admit_merge_write(None, 0, too_big, false, false));
+ assert!(g.admit_merge_write(None, 0, 1, false, false));
+ // A #2468 streak-exempt batch and `_forcemerge` still write.
+ assert!(g.admit_merge_write(None, 0, too_big, false, true));
+ assert!(g.admit_merge_write(Some(&mut reservation), held, too_big, true, false));
+ assert_eq!(reservation.bytes(), held + too_big);
+ assert_eq!(g.merge_reserved_bytes(), held + too_big);
+
+ drop(reservation);
+ assert_eq!(g.merge_reserved_bytes(), 0);
+ assert_eq!(g.merges_in_flight(), 0);
+ }
+
+ /// #2427 x #2468: a small background batch that the yield-streak rule
+ /// lets run to completion is not refused at its output write, even with
+ /// RSS at the hard limit. A large batch, a batch before the streak, and
+ /// a batch the plain rule never stopped get no exemption. A completed
+ /// batch ends it.
+ #[test]
+ fn a_streak_exempt_batch_is_not_refused_at_its_write() {
+ const MIB: u64 = 1024 * 1024;
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 0;
+ cfg.limits.memory_watermark_percent = 95;
+ let g = build(&cfg);
+ let wm = g.memory_watermark_bytes.load(Ordering::Relaxed);
+ let limit = g.process_memory_limit_bytes();
+ assert!(wm > 0 && limit > wm, "this test needs a live RSS watermark");
+ let small = 49 * MIB;
+ let large = crate::merge_budget::MERGE_YIELD_STREAK_MAX_INPUT_BYTES + 1;
+ let write = 256 * MIB;
+
+ // Below the limit the plain rule never stops a batch: no exemption,
+ // and the write decision is the hard-limit check alone.
+ g.refresh(0, wm / 2, 0);
+ assert!(!g.merge_write_exempt(false, small));
+
+ // Engaged at the limit, before the streak: the small batch yields at
+ // its checkpoints and its write would be refused.
+ g.refresh(0, limit, 0);
+ assert!(g.memory_breaker_engaged());
+ assert!(!g.merge_write_exempt(false, small));
+ let exempt = g.merge_write_exempt(false, small);
+ assert!(!g.admit_merge_write(None, small, write, false, exempt));
+
+ // After the streak, #2468 lets the small batch run, and so does its
+ // write. The large batch still yields and its write is still refused.
+ for _ in 0..crate::merge_budget::MERGE_YIELD_STREAK_LIMIT {
+ g.note_merge_yield(small);
+ }
+ assert!(!g.merge_must_yield_for(false, small));
+ assert!(g.merge_write_exempt(false, small));
+ let exempt = g.merge_write_exempt(false, small);
+ let mut reservation = g.try_reserve_merge(small, true).expect("forced admission");
+ let held = reservation.bytes();
+ assert!(g.admit_merge_write(Some(&mut reservation), held, write, false, exempt));
+ // The exempt write is still charged to the ledger.
+ assert_eq!(reservation.bytes(), held + write);
+ assert_eq!(g.merge_reserved_bytes(), held + write);
+ drop(reservation);
+ assert!(!g.merge_write_exempt(false, large));
+ let exempt = g.merge_write_exempt(false, large);
+ assert!(!g.admit_merge_write(None, large, write, false, exempt));
+
+ // `_forcemerge` is never stopped by either rule, so it needs no
+ // exemption; its write runs on `forced` alone.
+ assert!(!g.merge_write_exempt(true, small));
+ assert!(g.admit_merge_write(None, small, write, true, false));
+
+ // A completed batch resets the streak: no exemption again.
+ g.note_merge_completed(small);
+ assert!(!g.merge_write_exempt(false, small));
+ }
+
/// #2427: the 1 GB release gate released at rss=5,121MB against a
/// 2,918MB watermark (the old #1981 flat-footprint release), re-engaged
/// 100 ms later at 5,125MB, and was OOM-killed at 8.4 GB. A footprint at
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index b91158836..c5838d838 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -14381,6 +14381,44 @@ impl Index {
);
true
};
+ // #2427 — the output-write step (doc-values build, stored
+ // encode, FTS side-car build) runs after the last `must_yield`
+ // checkpoint and is the batch's peak: large pg-docs merges took a
+ // 3,072 MB search node to 4.4–4.9 GB there, about 5 s before they
+ // completed. It is charged to the batch's merge reservation before
+ // it starts, and refused (the batch yields, inputs kept, later
+ // batches narrowed) when it would take RSS to the hard limit. See
+ // `ResourceGovernor::admit_merge_write`.
+ let admit_write = move |reservation: Option<&mut crate::governor::MergeReservation>,
+ held: u64,
+ extra: u64|
+ -> bool {
+ let Some(governor) = crate::governor::global() else {
+ return true;
+ };
+ // #2468 lets a small background batch run to completion
+ // after a streak of yields with nothing merged. When that
+ // rule is what waived this batch's yield, the write runs
+ // too, or refusing it would bring the stall back.
+ let exempt = governor.merge_write_exempt(forced_for_task, input_bytes_for_task);
+ if governor.admit_merge_write(reservation, held, extra, forced_for_task, exempt) {
+ return true;
+ }
+ let cap = governor.note_merge_yield(input_bytes_for_task);
+ tracing::warn!(
+ input_mb = input_bytes_for_task / (1024 * 1024),
+ write_mb = extra / (1024 * 1024),
+ rss_mb = governor.observed_rss_bytes() / (1024 * 1024),
+ limit_mb = governor.process_memory_limit_bytes() / (1024 * 1024),
+ next_batch_cap_mb = cap / (1024 * 1024),
+ "merge batch yielded before its output write: the write would take RSS \
+ to the process limit; inputs kept, later batches narrowed"
+ );
+ false
+ };
+ // Held by the encode job until it ends, so the write step can
+ // grow it (see `admit_write`).
+ let mut reservation_for_encode = reservation;
#[cfg(test)]
let hold_merge_encode = Arc::clone(&self.test_hold_merge_encode);
#[cfg(test)]
@@ -14484,14 +14522,19 @@ impl Index {
// remapped, not re-derived). `retained_bytes` charges only
// what the reader OWNS (an mmap'd `.post` contributes 0),
// matching what this batch adds to the heap.
- let merge_reader_retained = merge_readers.as_ref().map(|readers| {
- let bytes: u64 = readers
- .iter()
- .map(|reader| reader.retained_bytes())
- .sum();
+ let merge_reader_bytes: u64 = merge_readers
+ .as_ref()
+ .map(|readers| {
+ readers
+ .iter()
+ .map(|reader| reader.retained_bytes())
+ .sum::<u64>()
+ })
+ .unwrap_or(0);
+ let merge_reader_retained = merge_readers.as_ref().map(|_| {
crate::ingest_memory::Retained::new(
crate::ingest_memory::Category::MergeDecoded,
- bytes as usize,
+ merge_reader_bytes as usize,
)
});
@@ -14979,6 +15022,29 @@ impl Index {
yielded_for_task.fetch_add(1, Ordering::Relaxed);
return None;
}
+ // #2427 — last chance to stop: everything below allocates
+ // on top of what the batch holds now and cannot yield.
+ // Charge it to the merge budget first, or yield.
+ {
+ let json_bytes = merged_json_buf.len() as u64;
+ let fts_bytes = fts_input_bytes as u64;
+ let write_fts_bytes = if merge_postings {
+ merge_reader_bytes
+ } else {
+ fts_bytes
+ };
+ let held = json_bytes
+ .saturating_add(fts_bytes)
+ .saturating_add(merge_reader_bytes);
+ let extra = crate::merge_budget::merge_write_phase_bytes(
+ json_bytes,
+ write_fts_bytes,
+ );
+ if !admit_write(reservation_for_encode.as_mut(), held, extra) {
+ yielded_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
+ }
// Doc-values columns, built here from the merged stored
// JSON one document at a time (#2427) and written once the
// merged segment id exists, below. The columns are what
@@ -15292,12 +15358,12 @@ impl Index {
version_repoint_intents,
))
}));
+ // Held until the encode (the allocation-heavy part: stored
+ // decode, FST and doc-values build) has finished.
+ drop(reservation_for_encode);
let _ = encode_tx.send(encoded);
});
tokio::spawn(async move {
- // Held until the encode (the allocation-heavy part: stored
- // decode, FST and doc-values build) has finished.
- let _reservation = reservation;
match encode_rx.await {
Ok(Ok(value)) => {
if value.is_some() && !forced_for_task {
diff --git a/engine/crates/xerj-engine/src/merge_budget.rs b/engine/crates/xerj-engine/src/merge_budget.rs
index b8e415f32..16a521238 100644
--- a/engine/crates/xerj-engine/src/merge_budget.rs
+++ b/engine/crates/xerj-engine/src/merge_budget.rs
@@ -302,6 +302,73 @@ pub fn effective_batch_cap_bytes(static_cap: u64, pressure_cap: u64) -> u64 {
}
}
+/// Multiplier on the merged stored JSON for the merge output-write step
+/// (#2427).
+///
+/// After the last yield checkpoint, a batch builds its doc-values columns
+/// from the merged JSON and then runs the stored encode
+/// (`encode_stored_v2_*_at_level`). The encode parses the whole JSON array
+/// into a `Vec<serde_json::Value>` (about 4–5× the JSON bytes in maps,
+/// strings and per-value overhead), then builds per-column buffers (about
+/// 1×) and the compressed output. All of that sits on top of the merged
+/// buffer, which is still alive. On the 3,072 MB search node, the large
+/// pg-docs merges (29k–166k docs) peaked at 4.4–4.9 GB about 5 s before
+/// they completed, which is this step. Neither admission nor the yield
+/// checks charged it.
+pub const MERGE_WRITE_JSON_NUMERATOR: u64 = 6;
+
+/// Multiplier on the FTS input for the output's FTS side-car build
+/// (#2427). The rebuild path re-analyses `fts_input` into in-memory
+/// posting lists. The replay path remaps the inputs' decoded postings,
+/// whose retained bytes bound it. In both cases about 2× covers the
+/// writer's buffers plus its encoded output.
+pub const MERGE_WRITE_FTS_NUMERATOR: u64 = 2;
+
+/// Extra bytes the merge output-write step allocates beyond what the
+/// batch already holds at its last yield checkpoint (#2427).
+///
+/// `json_bytes` is the merged stored JSON. `fts_bytes` is what the FTS
+/// side-car build works from: `fts_input`'s bytes on the rebuild path, or
+/// the replay readers' retained bytes. The stored encode and the FTS build
+/// run one after the other, and the encode's parsed copy is dropped before
+/// the FTS build starts, so the step's peak is the larger of the two, not
+/// their sum. Saturates instead of wrapping.
+pub fn merge_write_phase_bytes(json_bytes: u64, fts_bytes: u64) -> u64 {
+ json_bytes
+ .saturating_mul(MERGE_WRITE_JSON_NUMERATOR)
+ .max(fts_bytes.saturating_mul(MERGE_WRITE_FTS_NUMERATOR))
+}
+
+/// Whether a merge batch may run its output-write step (#2427).
+///
+/// The step comes after the last yield checkpoint, and once it starts it
+/// cannot stop without throwing the batch away, so the check runs before
+/// it starts. `extra` is [`merge_write_phase_bytes`], and `observed_rss`
+/// already includes what the batch holds. The step is refused when
+/// `observed_rss + extra` would reach the process `limit` (the hard limit,
+/// not the watermark). A refused batch yields like any other: no output,
+/// inputs kept, and later batches narrowed.
+///
+/// These always run:
+/// * `forced` batches (`_forcemerge`), which are never stopped;
+/// * `exempt` batches. This is for the #2468 rule: once a background
+/// batch has been allowed to run to completion after a streak of yields
+/// with nothing merged, refusing its write would bring back the stall
+/// that rule exists to break;
+/// * batches where the limit or RSS is unknown (`0`).
+pub fn decide_merge_write(
+ observed_rss: u64,
+ limit: u64,
+ extra: u64,
+ forced: bool,
+ exempt: bool,
+) -> bool {
+ if forced || exempt || limit == 0 || observed_rss == 0 {
+ return true;
+ }
+ observed_rss.saturating_add(extra) < limit
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -642,4 +709,59 @@ mod tests {
// Saturates instead of wrapping.
assert_eq!(merge_reservation_bytes(0, u64::MAX), u64::MAX);
}
+
+ #[test]
+ fn the_write_phase_charges_the_larger_of_encode_and_fts() {
+ const MIB: u64 = 1024 * 1024;
+ // Encode dominates on a JSON-heavy batch.
+ assert_eq!(
+ merge_write_phase_bytes(300 * MIB, 100 * MIB),
+ 300 * MIB * MERGE_WRITE_JSON_NUMERATOR
+ );
+ // The FTS build dominates when its input is the larger one.
+ assert_eq!(
+ merge_write_phase_bytes(10 * MIB, 100 * MIB),
+ 100 * MIB * MERGE_WRITE_FTS_NUMERATOR
+ );
+ assert_eq!(merge_write_phase_bytes(0, 0), 0);
+ assert_eq!(merge_write_phase_bytes(u64::MAX, 1), u64::MAX);
+ assert_eq!(merge_write_phase_bytes(1, u64::MAX), u64::MAX);
+ }
+
+ /// The search-node shape from #2427: a 3,072 MB limit, RSS at about
+ /// 2.9 GB, and a large pg-docs batch whose merged JSON is a few hundred
+ /// MB. Its write step would take RSS to 4.4–4.9 GB, so it must not start.
+ #[test]
+ fn a_write_that_would_cross_the_hard_limit_is_refused() {
+ const MIB: u64 = 1024 * 1024;
+ let limit = 3072 * MIB;
+ let rss = 2900 * MIB;
+ let extra = merge_write_phase_bytes(300 * MIB, 50 * MIB);
+ assert!(rss + extra > 4400 * MIB);
+ assert!(!decide_merge_write(rss, limit, extra, false, false));
+ // Exactly reaching the limit is refused, just under is allowed.
+ assert!(!decide_merge_write(rss, limit, limit - rss, false, false));
+ assert!(decide_merge_write(
+ rss,
+ limit,
+ limit - rss - 1,
+ false,
+ false
+ ));
+ // A small write with headroom runs, whatever the breaker state.
+ assert!(decide_merge_write(
+ 1000 * MIB,
+ limit,
+ 600 * MIB,
+ false,
+ false
+ ));
+ // `_forcemerge`, a #2468 streak-exempt batch, or unknown limit/RSS: runs.
+ assert!(decide_merge_write(rss, limit, extra, true, false));
+ assert!(decide_merge_write(rss, limit, extra, false, true));
+ assert!(decide_merge_write(rss, 0, extra, false, false));
+ assert!(decide_merge_write(0, limit, extra, false, false));
+ // Saturates instead of wrapping past the limit.
+ assert!(!decide_merge_write(rss, limit, u64::MAX, false, false));
+ }
}