Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

perf(merge): stop lone merge batches from overshooting the ingest cap (#2427) (#2485)

xerj-orgd435fb9c780c
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 02ead306c..972c34abe 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -379,6 +379,9 @@ pub struct ResourceGovernor {
/// How long a search over the hard limit waits for the drain (#2427).
/// [`SEARCH_HEADROOM_WAIT`] in production; tests shorten it.
search_headroom_wait: Duration,
+ /// Bound on a lone background merge's wait for headroom (#2427).
+ /// `merge_budget::MERGE_HEADROOM_MAX_WAIT` in production.
+ merge_headroom_max_wait: Duration,
// ── item 2b: global bulk pool (concurrent bulk parse cap) ─────────
/// Global bulk-concurrency permits. Caps the number of concurrent
@@ -733,6 +736,27 @@ impl ResourceGovernor {
let observed = self.observed_process_memory();
let engaged = self.memory_breaker_engaged();
let mut ledger = self.merge_ledger.lock().unwrap_or_else(|e| e.into_inner());
+ // #2427 ingest overshoot: a lone background batch does not start
+ // inside the read reserve; it waits (bounded) for the drain. See
+ // `merge_budget::lone_merge_waits_for_headroom`.
+ if !forced && ledger.in_flight == 0 && watermark > 0 && observed >= watermark {
+ let since = *ledger
+ .headroom_wait_since
+ .get_or_insert_with(std::time::Instant::now);
+ if crate::merge_budget::lone_merge_waits_for_headroom(
+ observed,
+ watermark,
+ ledger.in_flight,
+ forced,
+ since.elapsed(),
+ self.merge_headroom_max_wait,
+ ) {
+ return None;
+ }
+ } else if observed < watermark {
+ // Headroom came back: the next wait starts its own clock.
+ ledger.headroom_wait_since = None;
+ }
if !crate::merge_budget::decide_merge_admission(
watermark,
observed,
@@ -746,6 +770,7 @@ impl ResourceGovernor {
}
ledger.reserved = ledger.reserved.saturating_add(bytes);
ledger.in_flight += 1;
+ ledger.headroom_wait_since = None;
Some(MergeReservation {
ledger: Arc::clone(&self.merge_ledger),
bytes,
@@ -755,14 +780,25 @@ impl ResourceGovernor {
/// Whether an in-flight background merge batch must stop now (#2427).
/// See `merge_budget::merge_must_yield`.
pub fn merge_must_yield(&self, forced: bool) -> bool {
+ let observed = self.observed_process_memory();
crate::merge_budget::merge_must_yield(
- self.observed_process_memory(),
+ observed,
self.merge_pool_ceiling_bytes(),
- self.memory_breaker_engaged(),
+ self.merge_memory_pressure(observed),
forced,
)
}
+ /// Whether merges see memory pressure at `observed` bytes (#2427): the
+ /// breaker is engaged, or `observed` is already at the watermark and the
+ /// 100 ms sampler has not latched it yet. A merge checks live process
+ /// memory, so it need not run on for up to a sampler tick past the
+ /// watermark.
+ fn merge_memory_pressure(&self, observed: u64) -> bool {
+ let watermark = self.memory_watermark_bytes();
+ self.memory_breaker_engaged() || (watermark > 0 && observed >= watermark)
+ }
+
/// [`Self::merge_must_yield`] for a batch of `input_bytes`: after a
/// streak of yields a small batch completes, so compaction cannot stall
/// for good. See `merge_budget::merge_must_yield_after`.
@@ -772,10 +808,11 @@ impl ResourceGovernor {
.lock()
.unwrap_or_else(|e| e.into_inner())
.yield_streak;
+ let observed = self.observed_process_memory();
crate::merge_budget::merge_must_yield_after(
- self.observed_process_memory(),
+ observed,
self.merge_pool_ceiling_bytes(),
- self.memory_breaker_engaged(),
+ self.merge_memory_pressure(observed),
forced,
input_bytes,
streak,
@@ -1996,6 +2033,7 @@ fn build(config: &Config) -> ResourceGovernor {
search_inflight_peak: Arc::new(AtomicU64::new(0)),
search_pressure: Arc::new(Semaphore::new(1)),
search_headroom_wait: SEARCH_HEADROOM_WAIT,
+ merge_headroom_max_wait: crate::merge_budget::MERGE_HEADROOM_MAX_WAIT,
bulk_pool: Arc::new(Semaphore::new(max_concurrent_bulks)),
max_concurrent_bulks,
disk_flood_pct_default: limits.disk_flood_stage_percent.min(100),
@@ -2430,6 +2468,10 @@ struct MergeLedger {
/// Background yields since the last completed batch. See
/// `merge_budget::merge_must_yield_after`.
yield_streak: u64,
+ /// When a lone background merge was first held back for headroom
+ /// since the last admitted batch. See
+ /// `merge_budget::lone_merge_waits_for_headroom`.
+ headroom_wait_since: Option<std::time::Instant>,
}
/// Holds one merge batch's estimated working set against the RSS watermark
@@ -4410,6 +4452,54 @@ mod tests {
assert!(g.merge_must_yield_for(false, 49 * MIB));
}
+ /// #2427 ingest overshoot: on the 1 GB gate every over-cap sample on the
+ /// ingest node was a pg-docs batch launched alone the moment the last one
+ /// finished, with RSS already at 2.4-2.9 GB. A lone background batch now
+ /// waits while RSS is at or above the merge-pool ceiling, starts once the
+ /// drain brings it under, and never waits past the bound.
+ #[test]
+ fn a_lone_background_merge_waits_for_headroom_under_the_merge_pool() {
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 0;
+ cfg.limits.memory_watermark_percent = 95;
+ let mut g = build(&cfg);
+ let wm = g.memory_watermark_bytes();
+ let ceiling = g.merge_pool_ceiling_bytes();
+ assert!(
+ wm > 0 && ceiling > 0 && ceiling < wm,
+ "this test needs a live RSS watermark"
+ );
+
+ // Inside the read reserve, under the watermark: not engaged, but the
+ // lone background batch waits. `_forcemerge` does not.
+ let high = (ceiling + wm) / 2;
+ g.refresh(0, high, 0);
+ assert!(!g.memory_breaker_engaged());
+ assert!(g.try_reserve_merge(1, false).is_none());
+ let forced = g
+ .try_reserve_merge(1, true)
+ .expect("forcemerge never waits");
+ drop(forced);
+ assert!(g.try_reserve_merge(1, false).is_none());
+
+ // The drain brings RSS under the ceiling: it starts, and the wait
+ // clock resets.
+ g.refresh(0, ceiling / 2, 0);
+ let first = g
+ .try_reserve_merge(1, false)
+ .expect("starts after the drain");
+ drop(first);
+ assert!(g.merge_ledger.lock().unwrap().headroom_wait_since.is_none());
+
+ // The wait is bounded: past it, a lone batch starts even inside the
+ // reserve, so a node whose live set sits there still compacts.
+ g.merge_headroom_max_wait = Duration::ZERO;
+ g.refresh(0, high, 0);
+ assert!(!g.memory_breaker_engaged());
+ let late = g.try_reserve_merge(1, false).expect("bounded wait");
+ drop(late);
+ }
+
/// #2427: the merge output-write step (doc-values build, stored encode,
/// FTS build) runs after the last yield checkpoint. It is charged to the
/// merge ledger before it starts. A step that would take RSS to the hard
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 23aa05e92..e07c1149e 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -14400,7 +14400,7 @@ impl Index {
rss_mb = governor.observed_rss_bytes() / (1024 * 1024),
limit_mb = governor.process_memory_limit_bytes() / (1024 * 1024),
next_batch_cap_mb = cap / (1024 * 1024),
- "merge batch yielded: breaker engaged with RSS at the process limit; \
+ "merge batch yielded: memory pressure with RSS at the merge-pool ceiling; \
inputs kept, later batches narrowed"
);
true
@@ -14850,6 +14850,19 @@ impl Index {
ordinals.resize(raw_docs.len(), None);
doc_maps[source_index] = ordinals;
for (source_ordinal, raw) in raw_docs.iter().enumerate() {
+ // #2427 — the survivor scan copies each input's
+ // live docs, so a batch can grow hundreds of MB
+ // inside one input. Check the governor every
+ // `MERGE_YIELD_CHECKPOINT_DOCS` docs, not only
+ // between inputs.
+ if source_ordinal > 0
+ && (source_ordinal as u64)
+ .is_multiple_of(crate::merge_budget::MERGE_YIELD_CHECKPOINT_DOCS)
+ && must_yield()
+ {
+ yielded_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
let raw_str = raw.get();
let id_seq: IdSeq = match serde_json::from_str(raw_str) {
Ok(v) => v,
@@ -15008,11 +15021,18 @@ impl Index {
// `fts_input` grow, so the crossover IS the batch's
// stored-side peak.
drain_checkpoint += 1;
+ // #2427 — the yield check runs every
+ // `MERGE_YIELD_CHECKPOINT_DOCS` docs. At 8192 a pg-docs
+ // batch on the 1 GB gate grew RSS 300+ MB past the
+ // watermark between two checks.
+ if drain_checkpoint
+ .is_multiple_of(crate::merge_budget::MERGE_YIELD_CHECKPOINT_DOCS)
+ && must_yield()
+ {
+ yielded_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
if drain_checkpoint.is_multiple_of(8192) {
- if must_yield() {
- yielded_for_task.fetch_add(1, Ordering::Relaxed);
- return None;
- }
crate::ingest_memory::observe_checkpoint(
crate::ingest_memory::Category::MergeJsonBuffer,
merged_json_buf.len(),
diff --git a/engine/crates/xerj-engine/src/merge_budget.rs b/engine/crates/xerj-engine/src/merge_budget.rs
index 16a521238..1b4306c95 100644
--- a/engine/crates/xerj-engine/src/merge_budget.rs
+++ b/engine/crates/xerj-engine/src/merge_budget.rs
@@ -219,6 +219,53 @@ pub fn merge_must_yield(rss: u64, limit: u64, breaker_engaged: bool, forced: boo
!forced && breaker_engaged && limit > 0 && rss > 0 && rss >= limit
}
+/// Docs between two governor yield checks inside a merge batch's survivor
+/// scan and drain (#2427).
+///
+/// The checks used to run once per input segment and every 8,192 drained
+/// docs. On the 1 GB gate a pg-docs batch (7 inputs, about 90k docs) took
+/// the ingest node from the watermark (2,918 MB) to 3,100-3,440 MB between
+/// two checks. A check is one atomic load and one `/proc/self/statm` read.
+pub const MERGE_YIELD_CHECKPOINT_DOCS: u64 = 1024;
+
+/// Longest a lone background merge waits for RSS to come back under the
+/// merge-pool ceiling before it starts anyway (#2427). See
+/// [`lone_merge_waits_for_headroom`].
+pub const MERGE_HEADROOM_MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(30);
+
+/// Whether a background merge that would run alone must wait for headroom
+/// before it starts (#2427 ingest overshoot).
+///
+/// [`decide_merge_admission`] always starts a lone merge unless the breaker
+/// is engaged, so merging makes progress. On the 1 GB gate the ingest node
+/// launched the next pg-docs batch the moment the previous one finished,
+/// with RSS at 2,400-2,900 MB, just under the 2,918 MB watermark. Each batch
+/// grew RSS by 400-900 MB, and every one of the run's over-cap samples
+/// (3,100-3,502 MB against 3,072 MB) is one of those batches.
+///
+/// A lone background batch now waits while `observed_rss` is at or above
+/// `ceiling` (the merge-pool ceiling), so it starts after the drain, with
+/// the batch's growth room under the cap. The wait is bounded by `max_wait`
+/// (`waited` is how long lone merges have been held back), so a node whose
+/// live set sits above the ceiling still compacts. `_forcemerge`, a merge
+/// beside others (already checked against the ledger), and an unknown RSS
+/// or ceiling never wait.
+pub fn lone_merge_waits_for_headroom(
+ observed_rss: u64,
+ ceiling: u64,
+ in_flight: u64,
+ forced: bool,
+ waited: std::time::Duration,
+ max_wait: std::time::Duration,
+) -> bool {
+ !forced
+ && in_flight == 0
+ && ceiling > 0
+ && observed_rss > 0
+ && observed_rss >= ceiling
+ && waited < max_wait
+}
+
/// Consecutive background yields after which a small batch stops yielding.
/// See [`merge_must_yield_after`].
pub const MERGE_YIELD_STREAK_LIMIT: u64 = 3;
@@ -583,6 +630,84 @@ mod tests {
));
}
+ #[test]
+ fn a_lone_background_merge_waits_for_headroom_under_the_ceiling() {
+ use std::time::Duration;
+ let ceiling = 2_458 * MIB;
+ let max = MERGE_HEADROOM_MAX_WAIT;
+ // The gate's case: the next batch right after the last one, at
+ // 2,700 MB, with nothing else merging. It waits.
+ assert!(lone_merge_waits_for_headroom(
+ 2_700 * MIB,
+ ceiling,
+ 0,
+ false,
+ Duration::ZERO,
+ max
+ ));
+ assert!(lone_merge_waits_for_headroom(
+ ceiling,
+ ceiling,
+ 0,
+ false,
+ max - Duration::from_millis(1),
+ max
+ ));
+ // After the drain it starts.
+ assert!(!lone_merge_waits_for_headroom(
+ ceiling - 1,
+ ceiling,
+ 0,
+ false,
+ Duration::ZERO,
+ max
+ ));
+ // The wait is bounded, so a node whose live set sits above the
+ // ceiling still compacts.
+ assert!(!lone_merge_waits_for_headroom(
+ 2_700 * MIB,
+ ceiling,
+ 0,
+ false,
+ max,
+ max
+ ));
+ // `_forcemerge`, a merge beside others (the ledger check covers it),
+ // and unknown RSS or ceiling never wait here.
+ assert!(!lone_merge_waits_for_headroom(
+ 2_700 * MIB,
+ ceiling,
+ 0,
+ true,
+ Duration::ZERO,
+ max
+ ));
+ assert!(!lone_merge_waits_for_headroom(
+ 2_700 * MIB,
+ ceiling,
+ 1,
+ false,
+ Duration::ZERO,
+ max
+ ));
+ assert!(!lone_merge_waits_for_headroom(
+ 0,
+ ceiling,
+ 0,
+ false,
+ Duration::ZERO,
+ max
+ ));
+ assert!(!lone_merge_waits_for_headroom(
+ 2_700 * MIB,
+ 0,
+ 0,
+ false,
+ Duration::ZERO,
+ max
+ ));
+ }
+
#[test]
fn merge_admission_always_lets_one_merge_run_when_not_engaged() {
// Even an estimate that alone exceeds the watermark starts when no