AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(merge): stop lone merge batches from overshooting the ingest cap (#2427) (#2485)
xerj-orgd435fb9c780c
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 02ead306c..972c34abe 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -379,6 +379,9 @@ pub struct ResourceGovernor {/// How long a search over the hard limit waits for the drain (#2427)./// [`SEARCH_HEADROOM_WAIT`] in production; tests shorten it.search_headroom_wait: Duration,+ /// Bound on a lone background merge's wait for headroom (#2427).+ /// `merge_budget::MERGE_HEADROOM_MAX_WAIT` in production.+ merge_headroom_max_wait: Duration,// ── item 2b: global bulk pool (concurrent bulk parse cap) ─────────/// Global bulk-concurrency permits. Caps the number of concurrent@@ -733,6 +736,27 @@ impl ResourceGovernor {let observed = self.observed_process_memory();let engaged = self.memory_breaker_engaged();let mut ledger = self.merge_ledger.lock().unwrap_or_else(|e| e.into_inner());+ // #2427 ingest overshoot: a lone background batch does not start+ // inside the read reserve; it waits (bounded) for the drain. See+ // `merge_budget::lone_merge_waits_for_headroom`.+ if !forced && ledger.in_flight == 0 && watermark > 0 && observed >= watermark {+ let since = *ledger+ .headroom_wait_since+ .get_or_insert_with(std::time::Instant::now);+ if crate::merge_budget::lone_merge_waits_for_headroom(+ observed,+ watermark,+ ledger.in_flight,+ forced,+ since.elapsed(),+ self.merge_headroom_max_wait,+ ) {+ return None;+ }+ } else if observed < watermark {+ // Headroom came back: the next wait starts its own clock.+ ledger.headroom_wait_since = None;+ }if !crate::merge_budget::decide_merge_admission(watermark,observed,@@ -746,6 +770,7 @@ impl ResourceGovernor {}ledger.reserved = ledger.reserved.saturating_add(bytes);ledger.in_flight += 1;+ ledger.headroom_wait_since = None;Some(MergeReservation {ledger: Arc::clone(&self.merge_ledger),bytes,@@ -755,14 +780,25 @@ impl ResourceGovernor {/// Whether an in-flight background merge batch must stop now (#2427)./// See `merge_budget::merge_must_yield`.pub fn merge_must_yield(&self, forced: bool) -> bool {+ let observed = self.observed_process_memory();crate::merge_budget::merge_must_yield(- self.observed_process_memory(),+ observed,self.merge_pool_ceiling_bytes(),- self.memory_breaker_engaged(),+ self.merge_memory_pressure(observed),forced,)}+ /// Whether merges see memory pressure at `observed` bytes (#2427): the+ /// breaker is engaged, or `observed` is already at the watermark and the+ /// 100 ms sampler has not latched it yet. A merge checks live process+ /// memory, so it need not run on for up to a sampler tick past the+ /// watermark.+ fn merge_memory_pressure(&self, observed: u64) -> bool {+ let watermark = self.memory_watermark_bytes();+ self.memory_breaker_engaged() || (watermark > 0 && observed >= watermark)+ }+/// [`Self::merge_must_yield`] for a batch of `input_bytes`: after a/// streak of yields a small batch completes, so compaction cannot stall/// for good. See `merge_budget::merge_must_yield_after`.@@ -772,10 +808,11 @@ impl ResourceGovernor {.lock().unwrap_or_else(|e| e.into_inner()).yield_streak;+ let observed = self.observed_process_memory();crate::merge_budget::merge_must_yield_after(- self.observed_process_memory(),+ observed,self.merge_pool_ceiling_bytes(),- self.memory_breaker_engaged(),+ self.merge_memory_pressure(observed),forced,input_bytes,streak,@@ -1996,6 +2033,7 @@ fn build(config: &Config) -> ResourceGovernor {search_inflight_peak: Arc::new(AtomicU64::new(0)),search_pressure: Arc::new(Semaphore::new(1)),search_headroom_wait: SEARCH_HEADROOM_WAIT,+ merge_headroom_max_wait: crate::merge_budget::MERGE_HEADROOM_MAX_WAIT,bulk_pool: Arc::new(Semaphore::new(max_concurrent_bulks)),max_concurrent_bulks,disk_flood_pct_default: limits.disk_flood_stage_percent.min(100),@@ -2430,6 +2468,10 @@ struct MergeLedger {/// Background yields since the last completed batch. See/// `merge_budget::merge_must_yield_after`.yield_streak: u64,+ /// When a lone background merge was first held back for headroom+ /// since the last admitted batch. See+ /// `merge_budget::lone_merge_waits_for_headroom`.+ headroom_wait_since: Option<std::time::Instant>,}/// Holds one merge batch's estimated working set against the RSS watermark@@ -4410,6 +4452,54 @@ mod tests {assert!(g.merge_must_yield_for(false, 49 * MIB));}+ /// #2427 ingest overshoot: on the 1 GB gate every over-cap sample on the+ /// ingest node was a pg-docs batch launched alone the moment the last one+ /// finished, with RSS already at 2.4-2.9 GB. A lone background batch now+ /// waits while RSS is at or above the merge-pool ceiling, starts once the+ /// drain brings it under, and never waits past the bound.+ #[test]+ fn a_lone_background_merge_waits_for_headroom_under_the_merge_pool() {+ let mut cfg = Config::default();+ cfg.limits.max_total_memtable_mb = 0;+ cfg.limits.memory_watermark_percent = 95;+ let mut g = build(&cfg);+ let wm = g.memory_watermark_bytes();+ let ceiling = g.merge_pool_ceiling_bytes();+ assert!(+ wm > 0 && ceiling > 0 && ceiling < wm,+ "this test needs a live RSS watermark"+ );++ // Inside the read reserve, under the watermark: not engaged, but the+ // lone background batch waits. `_forcemerge` does not.+ let high = (ceiling + wm) / 2;+ g.refresh(0, high, 0);+ assert!(!g.memory_breaker_engaged());+ assert!(g.try_reserve_merge(1, false).is_none());+ let forced = g+ .try_reserve_merge(1, true)+ .expect("forcemerge never waits");+ drop(forced);+ assert!(g.try_reserve_merge(1, false).is_none());++ // The drain brings RSS under the ceiling: it starts, and the wait+ // clock resets.+ g.refresh(0, ceiling / 2, 0);+ let first = g+ .try_reserve_merge(1, false)+ .expect("starts after the drain");+ drop(first);+ assert!(g.merge_ledger.lock().unwrap().headroom_wait_since.is_none());++ // The wait is bounded: past it, a lone batch starts even inside the+ // reserve, so a node whose live set sits there still compacts.+ g.merge_headroom_max_wait = Duration::ZERO;+ g.refresh(0, high, 0);+ assert!(!g.memory_breaker_engaged());+ let late = g.try_reserve_merge(1, false).expect("bounded wait");+ drop(late);+ }+/// #2427: the merge output-write step (doc-values build, stored encode,/// FTS build) runs after the last yield checkpoint. It is charged to the/// merge ledger before it starts. A step that would take RSS to the harddiff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 23aa05e92..e07c1149e 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -14400,7 +14400,7 @@ impl Index {rss_mb = governor.observed_rss_bytes() / (1024 * 1024),limit_mb = governor.process_memory_limit_bytes() / (1024 * 1024),next_batch_cap_mb = cap / (1024 * 1024),- "merge batch yielded: breaker engaged with RSS at the process limit; \+ "merge batch yielded: memory pressure with RSS at the merge-pool ceiling; \inputs kept, later batches narrowed");true@@ -14850,6 +14850,19 @@ impl Index {ordinals.resize(raw_docs.len(), None);doc_maps[source_index] = ordinals;for (source_ordinal, raw) in raw_docs.iter().enumerate() {+ // #2427 — the survivor scan copies each input's+ // live docs, so a batch can grow hundreds of MB+ // inside one input. Check the governor every+ // `MERGE_YIELD_CHECKPOINT_DOCS` docs, not only+ // between inputs.+ if source_ordinal > 0+ && (source_ordinal as u64)+ .is_multiple_of(crate::merge_budget::MERGE_YIELD_CHECKPOINT_DOCS)+ && must_yield()+ {+ yielded_for_task.fetch_add(1, Ordering::Relaxed);+ return None;+ }let raw_str = raw.get();let id_seq: IdSeq = match serde_json::from_str(raw_str) {Ok(v) => v,@@ -15008,11 +15021,18 @@ impl Index {// `fts_input` grow, so the crossover IS the batch's// stored-side peak.drain_checkpoint += 1;+ // #2427 — the yield check runs every+ // `MERGE_YIELD_CHECKPOINT_DOCS` docs. At 8192 a pg-docs+ // batch on the 1 GB gate grew RSS 300+ MB past the+ // watermark between two checks.+ if drain_checkpoint+ .is_multiple_of(crate::merge_budget::MERGE_YIELD_CHECKPOINT_DOCS)+ && must_yield()+ {+ yielded_for_task.fetch_add(1, Ordering::Relaxed);+ return None;+ }if drain_checkpoint.is_multiple_of(8192) {- if must_yield() {- yielded_for_task.fetch_add(1, Ordering::Relaxed);- return None;- }crate::ingest_memory::observe_checkpoint(crate::ingest_memory::Category::MergeJsonBuffer,merged_json_buf.len(),diff --git a/engine/crates/xerj-engine/src/merge_budget.rs b/engine/crates/xerj-engine/src/merge_budget.rsindex 16a521238..1b4306c95 100644--- a/engine/crates/xerj-engine/src/merge_budget.rs+++ b/engine/crates/xerj-engine/src/merge_budget.rs@@ -219,6 +219,53 @@ pub fn merge_must_yield(rss: u64, limit: u64, breaker_engaged: bool, forced: boo!forced && breaker_engaged && limit > 0 && rss > 0 && rss >= limit}+/// Docs between two governor yield checks inside a merge batch's survivor+/// scan and drain (#2427).+///+/// The checks used to run once per input segment and every 8,192 drained+/// docs. On the 1 GB gate a pg-docs batch (7 inputs, about 90k docs) took+/// the ingest node from the watermark (2,918 MB) to 3,100-3,440 MB between+/// two checks. A check is one atomic load and one `/proc/self/statm` read.+pub const MERGE_YIELD_CHECKPOINT_DOCS: u64 = 1024;++/// Longest a lone background merge waits for RSS to come back under the+/// merge-pool ceiling before it starts anyway (#2427). See+/// [`lone_merge_waits_for_headroom`].+pub const MERGE_HEADROOM_MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(30);++/// Whether a background merge that would run alone must wait for headroom+/// before it starts (#2427 ingest overshoot).+///+/// [`decide_merge_admission`] always starts a lone merge unless the breaker+/// is engaged, so merging makes progress. On the 1 GB gate the ingest node+/// launched the next pg-docs batch the moment the previous one finished,+/// with RSS at 2,400-2,900 MB, just under the 2,918 MB watermark. Each batch+/// grew RSS by 400-900 MB, and every one of the run's over-cap samples+/// (3,100-3,502 MB against 3,072 MB) is one of those batches.+///+/// A lone background batch now waits while `observed_rss` is at or above+/// `ceiling` (the merge-pool ceiling), so it starts after the drain, with+/// the batch's growth room under the cap. The wait is bounded by `max_wait`+/// (`waited` is how long lone merges have been held back), so a node whose+/// live set sits above the ceiling still compacts. `_forcemerge`, a merge+/// beside others (already checked against the ledger), and an unknown RSS+/// or ceiling never wait.+pub fn lone_merge_waits_for_headroom(+ observed_rss: u64,+ ceiling: u64,+ in_flight: u64,+ forced: bool,+ waited: std::time::Duration,+ max_wait: std::time::Duration,+) -> bool {+ !forced+ && in_flight == 0+ && ceiling > 0+ && observed_rss > 0+ && observed_rss >= ceiling+ && waited < max_wait+}+/// Consecutive background yields after which a small batch stops yielding./// See [`merge_must_yield_after`].pub const MERGE_YIELD_STREAK_LIMIT: u64 = 3;@@ -583,6 +630,84 @@ mod tests {));}+ #[test]+ fn a_lone_background_merge_waits_for_headroom_under_the_ceiling() {+ use std::time::Duration;+ let ceiling = 2_458 * MIB;+ let max = MERGE_HEADROOM_MAX_WAIT;+ // The gate's case: the next batch right after the last one, at+ // 2,700 MB, with nothing else merging. It waits.+ assert!(lone_merge_waits_for_headroom(+ 2_700 * MIB,+ ceiling,+ 0,+ false,+ Duration::ZERO,+ max+ ));+ assert!(lone_merge_waits_for_headroom(+ ceiling,+ ceiling,+ 0,+ false,+ max - Duration::from_millis(1),+ max+ ));+ // After the drain it starts.+ assert!(!lone_merge_waits_for_headroom(+ ceiling - 1,+ ceiling,+ 0,+ false,+ Duration::ZERO,+ max+ ));+ // The wait is bounded, so a node whose live set sits above the+ // ceiling still compacts.+ assert!(!lone_merge_waits_for_headroom(+ 2_700 * MIB,+ ceiling,+ 0,+ false,+ max,+ max+ ));+ // `_forcemerge`, a merge beside others (the ledger check covers it),+ // and unknown RSS or ceiling never wait here.+ assert!(!lone_merge_waits_for_headroom(+ 2_700 * MIB,+ ceiling,+ 0,+ true,+ Duration::ZERO,+ max+ ));+ assert!(!lone_merge_waits_for_headroom(+ 2_700 * MIB,+ ceiling,+ 1,+ false,+ Duration::ZERO,+ max+ ));+ assert!(!lone_merge_waits_for_headroom(+ 0,+ ceiling,+ 0,+ false,+ Duration::ZERO,+ max+ ));+ assert!(!lone_merge_waits_for_headroom(+ 2_700 * MIB,+ 0,+ 0,+ false,+ Duration::ZERO,+ max+ ));+ }+#[test]fn merge_admission_always_lets_one_merge_run_when_not_engaged() {// Even an estimate that alone exceeds the watermark starts when no