AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): drain on memtable pressure so corpus index makes progress (#2238) (#2244)
xerj-orgc16fad5877ec
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 9e2807884..a452d3777 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -25,6 +25,9 @@ pub struct Es {/// How long one `bulk` call keeps re-offering records the server/// rejected with a per-item 429 (see [`THROTTLE_PATIENCE`]).throttle_patience: Duration,+ /// How long a bulk that has not had a single item accepted keeps+ /// re-asking a parent memory breaker (see [`ZERO_PROGRESS_PATIENCE`]).+ zero_progress_patience: Duration,/// Most actions this client puts in one `_bulk` request before the server/// has said anything about it. See [`BULK_MAX_ACTIONS`]; tests shrink it.bulk_max_actions: usize,@@ -624,6 +627,53 @@ impl BulkAdmission {/// 5xx and write blocks are NOT waited on — they are not congestion.pub const THROTTLE_PATIENCE: Duration = Duration::from_secs(600);+/// How long a bulk that has accepted nothing will keep re-asking the parent+/// memory breaker before the run stops and names the cap (#2238).+///+/// [`THROTTLE_PATIENCE`] stays the budget once any item in the bulk has been+/// accepted: a drain that is freeing memory (the #944 mailbox held the+/// breaker for 57 s before drains existed; releases after #948 are seconds)+/// must be allowed to finish. A bulk the server has not taken a single item+/// from is the other shape — the 8 GiB auto tier's breaker loop, indexed+/// count stuck at 0 for several minutes with no statement of what to change.+/// Ninety seconds is past that 57 s hold and past one drain of the auto+/// tier's 2 GiB memtable budget, and short of a multi-minute spin.+pub const ZERO_PROGRESS_PATIENCE: Duration = Duration::from_secs(90);++/// The parent memory breaker, as opposed to a disk flood-stage 429 or a+/// thread-pool rejection. Both of its reasons carry one of these phrases.+pub(crate) fn is_parent_memory_breaker(reason: &str) -> bool {+ reason.contains("circuit breaker") || reason.contains("memtable byte budget")+}++/// Name `limits.max_process_memory_mb` on a parent-breaker 429 that does not+/// already name it (#2238). Other failures are returned unchanged, and a+/// reason that already carries the knob is not annotated twice.+pub(crate) fn memory_cap_hint(reason: &str) -> String {+ const MARKER: &str = "limits.max_process_memory_mb";+ if reason.contains(MARKER) || !is_parent_memory_breaker(reason) {+ return reason.to_owned();+ }+ format!(+ "{reason} — raise {MARKER} or XERJ_MAX_PROCESS_MEMORY_MB \+ (the auto memory tier derives this breaker's limit from that cap; 0 = whole machine)"+ )+}++/// The back-pressure stop, with the process-memory cap named when the+/// rejection is the parent breaker.+pub(crate) fn backpressure_exhausted(+ items: u64,+ patience: Duration,+ reason: &str,+) -> BackpressureExhausted {+ BackpressureExhausted {+ items,+ patience,+ reason: memory_cap_hint(reason),+ }+}+/// How often the wait is announced on stderr, so a run that is waiting on/// the server does not look like a hang.const NOTICE_EVERY: Duration = Duration::from_secs(30);@@ -948,7 +998,7 @@ fn parse_round(v: &Value) -> Round {op["error"].get("reason").and_then(Value::as_str)- .map(|r| r.chars().take(300).collect())+ .map(|r| r.chars().take(SERVER_REASON_MAX).collect()).unwrap_or_else(error_text),);}@@ -987,6 +1037,10 @@ pub struct BulkOutcome {/// failure that says "come back later" rather than "something is broken"./// Callers use it to name the stop as back-pressure, not as an abort.pub throttled_out: u64,+ /// The patience budget that elapsed when `throttled_out` was set.+ /// `None` until a bulk gives up. The full [`THROTTLE_PATIENCE`] once any+ /// item was accepted; [`ZERO_PROGRESS_PATIENCE`] when none was (#2238).+ pub exhausted_after: Option<Duration>,}impl BulkOutcome {@@ -1001,6 +1055,9 @@ impl BulkOutcome {if self.first_server_error.is_none() {self.first_server_error = other.first_server_error;}+ if self.exhausted_after.is_none() {+ self.exhausted_after = other.exhausted_after;+ }}}@@ -1161,6 +1218,7 @@ impl Es {retry_max_delay,admission: Arc::new(BulkAdmission::off()),throttle_patience: THROTTLE_PATIENCE,+ zero_progress_patience: ZERO_PROGRESS_PATIENCE,bulk_max_actions: BULK_MAX_ACTIONS,request_size: Arc::new(RequestSizeLedger::default()),throttle_ledger: Arc::new(ThrottleLedger::default()),@@ -1187,6 +1245,14 @@ impl Es {self}+ /// How long a bulk the server has not accepted a single item from keeps+ /// re-asking a parent memory breaker (tests shrink it).+ #[cfg(test)]+ fn with_zero_progress_patience(mut self, patience: Duration) -> Self {+ self.zero_progress_patience = patience;+ self+ }+/// How long one `bulk` call waits on a server that is accepting nothing/// before the rejection is handed back — what the terminal line quotes.pub fn throttle_patience(&self) -> Duration {@@ -1479,7 +1545,11 @@ impl Es {/// re-asking at the capped backoff for up to `throttle_patience`,/// announcing the wait every [`NOTICE_EVERY`]; a one-shot client (gate/// off — a probe, a status read) treats a 429 like a 5xx and gives up- /// after six, because a 10-minute wait is wrong for a question.+ /// after six, because a 10-minute wait is wrong for a question. A+ /// parent memory breaker that has accepted nothing is the exception+ /// inside that budget: it stops at [`ZERO_PROGRESS_PATIENCE`] and names+ /// `limits.max_process_memory_mb` (#2238), because waiting out ten+ /// minutes cannot shrink a budget the drain is not being asked to free./// Measured reason for the patience: the parent memory circuit breaker/// answered a bulk with HTTP 429 and held the server above the/// watermark for 57 s on the 1 GB mailbox run — the six attempts@@ -1501,20 +1571,33 @@ impl Es {Attempt::Abort(e) => return Err(e),Attempt::Throttled(reason) => {self.admission.on_congestion();+ let raw = reason.as_deref().unwrap_or("HTTP 429");let err = match &reason {- Some(r) => anyhow!("{what}: HTTP 429 Too Many Requests: {r}"),+ Some(r) => {+ anyhow!("{what}: HTTP 429 Too Many Requests: {}", memory_cap_hint(r))+ }None => anyhow!("{what}: HTTP 429 Too Many Requests"),};if self.admission.enabled() {let waited = started.elapsed();- if waited >= self.throttle_patience {+ // A whole-request 429 accepted nothing. The parent+ // memory breaker stops at the short budget; every+ // other 429 (disk flood, thread-pool rejection) keeps+ // the full patience, because that condition can clear.+ let limit = if is_parent_memory_breaker(raw) {+ self.throttle_patience.min(self.zero_progress_patience)+ } else {+ self.throttle_patience+ };+ if waited >= limit {return Err(err);}self.throttle_notice(&mut last_notice,- reason.as_deref().unwrap_or("HTTP 429"),+ raw,&format!("retrying {what}"),waited,+ limit,);} else {failures += 1;@@ -1577,14 +1660,16 @@ impl Es {reason: &str,doing: &str,waited: Duration,+ giving_up_after: Duration,) {if !self.admission.announces() || matches!(last, Some(t) if t.elapsed() < NOTICE_EVERY) {return;}+ let reason = memory_cap_hint(reason);eprintln!("autoindex: server is shedding load — {reason}; {doing} (waited {}s, giving up after {}s)",waited.as_secs(),- self.throttle_patience.as_secs()+ giving_up_after.as_secs());*last = Some(Instant::now());}@@ -1794,15 +1879,21 @@ impl Es {/// 429/5xx retries the WHOLE request (`retry_loop`). A 200 whose items/// carry `status: 429` re-offers ONLY those items, after a backoff, for up/// to `throttle_patience` — the memory breaker's shape, see- /// [`THROTTLE_PATIENCE`]. Records the server took are never sent twice,- /// and a record it refused for a reason waiting cannot fix (a per-item- /// 400, 5xx, or a write block) is not re-offered.+ /// [`THROTTLE_PATIENCE`]. A parent memory breaker that has not accepted a+ /// single item of this bulk stops at [`ZERO_PROGRESS_PATIENCE`] instead+ /// (#2238). Records the server took are never sent twice, and a record it+ /// refused for a reason waiting cannot fix (a per-item 400, 5xx, or a+ /// write block) is not re-offered.fn bulk_window(&self, body: Vec<u8>) -> Result<SentWindow> {let mut body = body;// Item errors settled by earlier rounds of this call (a bad record's// 400): the record is not re-offered, its error is still reported.let mut settled_item_errors = 0u64;let mut settled_first_error: Option<String> = None;+ // Set once any round took an item. From then on the full patience+ // applies, even if later rounds are all 429: a drain in progress+ // (#944 held 57 s) must be allowed to finish.+ let mut accepted_any = false;let episode = Instant::now();let mut delay = self.retry_initial_delay;let mut last_notice: Option<Instant> = None;@@ -1819,14 +1910,30 @@ impl Es {if round.throttled.is_empty() || round.other_server_errors > 0 {return Ok(SentWindow::Done(outcome));}+ if round.items > round.throttled.len() {+ accepted_any = true;+ }// Only per-item 429s stand between this bulk and a clean// outcome. Wait while patience lasts; after that the caller gets// the server's own reason, exactly as before — but named as// back-pressure (#944), so the run's terminal line can say// `reason=server-backpressure` rather than `reason=aborted`.+ // Zero accepts against the parent memory breaker use the short+ // budget (#2238). Any other 429, and any bulk that has landed an+ // item, keeps the full patience.+ let parent = round+ .throttle_reason+ .as_deref()+ .is_some_and(is_parent_memory_breaker);+ let limit = if accepted_any || !parent {+ self.throttle_patience+ } else {+ self.throttle_patience.min(self.zero_progress_patience)+ };let waited = episode.elapsed();- if waited >= self.throttle_patience {+ if waited >= limit {outcome.throttled_out = round.throttled.len() as u64;+ outcome.exhausted_after = Some(limit);return Ok(SentWindow::Done(outcome));}// A response whose item count does not match the body cannot be@@ -1848,6 +1955,7 @@ impl Es {.unwrap_or("per-item HTTP 429"),&format!("re-offering {} rejected record(s)", round.throttled.len()),waited,+ limit,);self.throttle_ledger.reissues@@ -2523,6 +2631,7 @@ mod tests {use std::io::Read;use std::io::Write;use std::net::TcpListener;+ use std::sync::atomic::{AtomicBool, Ordering};use std::sync::{mpsc, Arc, Mutex};use std::time::{Duration, Instant};@@ -4017,6 +4126,160 @@ mod tests {assert_eq!(outcome.server_errors, 2);}+ /// Answer every connection with `body` until `stop` is set. The+ /// zero-progress tests need a server that outlasts the short budget.+ fn serve_until(+ listener: TcpListener,+ body: Vec<u8>,+ stop: Arc<AtomicBool>,+ ) -> std::thread::JoinHandle<()> {+ listener.set_nonblocking(true).unwrap();+ std::thread::spawn(move || loop {+ match listener.accept() {+ Ok((mut stream, _)) => {+ stream.set_nonblocking(false).unwrap();+ let _ = read_request(&mut stream);+ respond_json(&mut stream, &body);+ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ if stop.load(Ordering::SeqCst) {+ return;+ }+ std::thread::sleep(Duration::from_millis(2));+ }+ Err(error) => panic!("accept: {error}"),+ }+ })+ }++ /// #2238: every item 429 from the parent breaker, forever. The run must+ /// stop at the zero-progress budget and name `limits.max_process_memory_mb`,+ /// not sit out the full ten-minute patience with the indexed count at 0.+ #[test]+ fn a_parent_breaker_that_accepts_nothing_stops_at_zero_progress_patience() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let stop = Arc::new(AtomicBool::new(false));+ let server = serve_until(+ listener,+ items_response(+ &[item("a", 429, Some(BREAKER)), item("b", 429, Some(BREAKER))],+ true,+ ),+ Arc::clone(&stop),+ );+ let es = client(address, 4)+ .with_throttle_patience(Duration::from_secs(2))+ .with_zero_progress_patience(Duration::from_millis(50));+ let started = Instant::now();+ let outcome = es.bulk([pair("a"), pair("b")].concat()).unwrap();+ let elapsed = started.elapsed();+ stop.store(true, Ordering::SeqCst);+ server.join().unwrap();+ assert!(+ elapsed < Duration::from_millis(500),+ "zero accepts must not wait the 2s patience, took {elapsed:?}"+ );+ assert_eq!(+ outcome.exhausted_after,+ Some(Duration::from_millis(50)),+ "{outcome:?}"+ );+ assert!(outcome.throttled_out >= 1, "{outcome:?}");+ let backoffs = es.backoff_delays.lock().unwrap().len();+ assert!(+ (1..=10).contains(&backoffs),+ "retried inside the short budget, did not spin: {backoffs} backoffs"+ );+ let hinted = super::memory_cap_hint(outcome.first_server_error.as_deref().unwrap());+ assert!(hinted.contains("limits.max_process_memory_mb"), "{hinted}");+ }++ /// #944/#1250: once any item of the bulk has been accepted, a later run of+ /// all-429 answers keeps the full patience. Fail-fast applies only to a+ /// bulk the server has not taken a single item from.+ #[test]+ fn a_bulk_that_landed_an_item_keeps_the_full_patience() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ listener.set_nonblocking(true).unwrap();+ let stop = Arc::new(AtomicBool::new(false));+ let server_stop = Arc::clone(&stop);+ let server = std::thread::spawn(move || {+ let mut first = true;+ loop {+ match listener.accept() {+ Ok((mut stream, _)) => {+ stream.set_nonblocking(false).unwrap();+ let _ = read_request(&mut stream);+ if first {+ first = false;+ respond_json(+ &mut stream,+ &items_response(+ &[item("a", 201, None), item("b", 429, Some(BREAKER))],+ true,+ ),+ );+ } else {+ items_answered(&mut stream, &[429]);+ }+ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ if server_stop.load(Ordering::SeqCst) {+ return;+ }+ std::thread::sleep(Duration::from_millis(2));+ }+ Err(error) => panic!("accept: {error}"),+ }+ }+ });+ let es = client(address, 4)+ .with_throttle_patience(Duration::from_millis(300))+ .with_zero_progress_patience(Duration::from_millis(20));+ let started = Instant::now();+ let outcome = es.bulk([pair("a"), pair("b")].concat()).unwrap();+ let elapsed = started.elapsed();+ stop.store(true, Ordering::SeqCst);+ server.join().unwrap();+ assert!(+ elapsed >= Duration::from_millis(250),+ "an accepted item must keep the 300ms patience, stopped after {elapsed:?}"+ );+ assert!(+ elapsed < Duration::from_secs(2),+ "the full patience is 300ms, took {elapsed:?}"+ );+ assert_eq!(+ outcome.exhausted_after,+ Some(Duration::from_millis(300)),+ "{outcome:?}"+ );+ assert!(outcome.throttled_out >= 1, "{outcome:?}");+ }++ #[test]+ fn backpressure_exhausted_names_the_process_memory_cap_once() {+ let error = super::backpressure_exhausted(4, Duration::from_secs(90), BREAKER);+ let rendered = error.to_string();+ assert!(rendered.contains("after 90s"), "{rendered}");+ assert!(rendered.contains("circuit breaker"), "{rendered}");+ assert!(+ rendered.contains("limits.max_process_memory_mb"),+ "{rendered}"+ );+ assert_eq!(+ super::memory_cap_hint(&error.reason),+ error.reason,+ "a reason that already names the cap is not annotated twice"+ );+ assert_eq!(+ super::memory_cap_hint("mapper_parsing_exception"),+ "mapper_parsing_exception"+ );+ }+#[test]fn bulk_item_selection_keeps_lines_byte_identical_and_knows_delete_has_no_source() {let del = b"{\"delete\":{\"_index\":\"i\",\"_id\":\"x\"}}\n".to_vec();@@ -4144,6 +4407,57 @@ mod tests {server.join().unwrap();}+ /// A plain HTTP 429 (no bulk `items`) from the parent breaker is the same+ /// zero-progress loop: stop at the short budget and name the cap. A probe+ /// (admission off) still stops after six attempts — covered above.+ #[test]+ fn a_plain_parent_breaker_429_names_the_cap_and_stops_short() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let stop = Arc::new(AtomicBool::new(false));+ let server_stop = Arc::clone(&stop);+ listener.set_nonblocking(true).unwrap();+ let server = std::thread::spawn(move || loop {+ match listener.accept() {+ Ok((mut stream, _)) => {+ stream.set_nonblocking(false).unwrap();+ let _ = read_request(&mut stream);+ let body = br#"{"error":{"type":"circuit_breaking_exception","reason":"[parent] real memory circuit breaker tripped: rss=9MB >= watermark=8MB"},"status":429}"#;+ respond_status(&mut stream, "429 Too Many Requests", body);+ }+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ if server_stop.load(Ordering::SeqCst) {+ return;+ }+ std::thread::sleep(Duration::from_millis(2));+ }+ Err(error) => panic!("accept: {error}"),+ }+ });+ let es = client(address, 4)+ .with_throttle_patience(Duration::from_secs(2))+ .with_zero_progress_patience(Duration::from_millis(40));+ let started = Instant::now();+ let error = es.bulk(BULK.to_vec()).unwrap_err();+ let elapsed = started.elapsed();+ stop.store(true, Ordering::SeqCst);+ server.join().unwrap();+ let rendered = format!("{error:#}");+ assert!(+ rendered.contains("limits.max_process_memory_mb"),+ "{rendered}"+ );+ assert!(+ elapsed < Duration::from_millis(500),+ "a parent breaker that accepts nothing must not wait 2s, took {elapsed:?}"+ );+ let backoffs = es.backoff_delays.lock().unwrap().len();+ assert!(+ (1..=8).contains(&backoffs),+ "retried inside the short budget: {backoffs} backoffs"+ );+ }+// ── #955: a request is bounded in actions and bytes, and a request the// server still calls too large is halved instead of ending the run ──diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex b76eec0ae..7fc9118cf 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -2539,6 +2539,16 @@ fn replacement_failpoint(_boundary: u8, _state_dir: &Path) -> Result<()> {Ok(())}+/// The first server error, with `limits.max_process_memory_mb` named when+/// the rejection is the parent memory breaker (#2238).+fn server_error_text(outcome: &crate::esclient::BulkOutcome) -> String {+ outcome+ .first_server_error+ .as_deref()+ .map(crate::esclient::memory_cap_hint)+ .unwrap_or_else(|| "unknown server error".into())+}+/// Send one bulk body and fold its per-item rejections into/// `rejected_records`.///@@ -2570,10 +2580,7 @@ fn record_bulk_outcome(complete; fix the server condition (or wait for it to clear) and rerun \autoindex — the run resumes from the journal",outcome.server_errors,- outcome- .first_server_error- .as_deref()- .unwrap_or("unknown server error")+ server_error_text(&outcome)));return true;}@@ -2883,10 +2890,7 @@ wherecomplete; fix the server condition (or wait for it to clear) and rerun \autoindex — the run resumes from the journal",outcome.server_errors,- outcome- .first_server_error- .as_deref()- .unwrap_or("unknown server error")+ server_error_text(&outcome));for t in &tokens {st.failed.insert(*t);@@ -14041,6 +14045,7 @@ mod bulk_pipe_tests {first_error: None,first_server_error: None,throttled_out: 0,+ exhausted_after: None,}}diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex ee04751a5..567f8feb1 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -2679,28 +2679,35 @@ fn check_bulk_outcome(outcome: crate::esclient::BulkOutcome,) -> Result<()> {// `Es::bulk` has already re-sent per-item 429s, and only those, for the- // whole of its patience (`THROTTLE_PATIENCE`, #944/#949).- // What is left is either a server condition that did not clear — fatal,- // and resumable, because a sealed operation is journaled applied only- // after this returns — or a record the server refused outright.+ // whole of its patience (`THROTTLE_PATIENCE`, #944/#949, shortened to+ // `ZERO_PROGRESS_PATIENCE` when a parent memory breaker accepts nothing,+ // #2238). What is left is either a server condition that did not clear —+ // fatal, and resumable, because a sealed operation is journaled applied+ // only after this returns — or a record the server refused outright.if outcome.throttled_out > 0 {- return Err(anyhow::Error::new(crate::esclient::BackpressureExhausted {- items: outcome.server_errors,- patience: es.throttle_patience(),- reason: outcome- .first_server_error- .unwrap_or_else(|| "unknown server error".into()),- }));+ let patience = outcome+ .exhausted_after+ .unwrap_or_else(|| es.throttle_patience());+ let reason = outcome+ .first_server_error+ .unwrap_or_else(|| "unknown server error".into());+ return Err(anyhow::Error::new(crate::esclient::backpressure_exhausted(+ outcome.server_errors,+ patience,+ &reason,+ )));}if outcome.server_errors > 0 {+ let reason = outcome+ .first_server_error+ .as_deref()+ .map(crate::esclient::memory_cap_hint)+ .unwrap_or_else(|| "unknown server error".into());anyhow::bail!(- "the server failed {} of a prepared bulk's items: {}. Nothing from this bulk was \+ "the server failed {} of a prepared bulk's items: {reason}. Nothing from this bulk was \journaled applied; fix the reported server condition and rerun the same command — \the run resumes from its last committed operation",outcome.server_errors,- outcome- .first_server_error- .unwrap_or_else(|| "unknown server error".into()));}anyhow::ensure!(diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 2da91959d..bc7e4be4d 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -21,13 +21,16 @@ use crate::{EngineError, Result};// ── Memory-pressure drain plumbing (issue #948) ─────────────────────────────//-// The parent RSS breaker used to only REJECT writes (429). Under a workload+// The parent breaker used to only REJECT writes (429). Under a workload// whose caches hold gigabytes of re-hydratable state and whose memtable// holds raw write payloads, rejection frees nothing: the breaker stays-// engaged, the client retries forever, and the run never finishes — exactly+// engaged, the client retries, and the run never finishes — exactly// the #948 mailbox ingest (16 GiB cap, 26.5 GiB demand, 118-147 s engaged-// stretches, 3-4 s releases). The sampler now also requests a DRAIN: flush-// every index, release rebuildable caches, purge allocator pages.+// stretches, 3-4 s releases) and the #2238 corpus-index loop (8 GiB auto+// tier, dozens of indices under their own flush line but over the 2 GiB+// summed-memtable budget, indexed count stuck at 0). The sampler requests+// a DRAIN on either latch: flush every index, release rebuildable caches,+// purge allocator pages./// Minimum spacing between automatic drains while the breaker stays engaged./// The engage edge always fires immediately; repeats are rate-limited so a@@ -35,6 +38,35 @@ use crate::{EngineError, Result};/// reads) without the drain worker competing with ingest for every tick.const MEMORY_DRAIN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);+/// Whether this sampler tick should ask the drain worker to run (#948, #2238).+///+/// `needs_drain` is [`crate::governor::ResourceGovernor::ingest_pressure_needs_drain`]:+/// the RSS latch or the summed-memtable latch. A memtable-only trip used to+/// return false here, so a multi-index ingest that had filled the auto tier's+/// 2 GiB memtable budget kept receiving HTTP 429 with nothing flushed and the+/// indexed count stayed at 0. The engage edge fires immediately; while the+/// latch stays up, repeats wait `min_interval`. Releasing the latch clears+/// the timer so the next engage drains at once.+fn pressure_drain_due(+ needs_drain: bool,+ last_drain_at: &mut Option<std::time::Instant>,+ now: std::time::Instant,+ min_interval: std::time::Duration,+) -> bool {+ if !needs_drain {+ *last_drain_at = None;+ return false;+ }+ let due = match *last_drain_at {+ None => true,+ Some(at) => now.saturating_duration_since(at) >= min_interval,+ };+ if due {+ *last_drain_at = Some(now);+ }+ due+}+/// Bounded request channel from the sampler thread to the drain worker./// Capacity 1 + try_send ⇒ requests coalesce: one arrives mid-drain is/// dropped, and the next sampler tick re-requests if still engaged.@@ -4222,26 +4254,23 @@ impl Engine {let (disk_pct, disk_avail) = crate::governor::disk_used_pct_and_avail(&data_dir);governor.refresh_memory_disk(rss, disk_pct, disk_avail);- // Issue #948 — a breaker that only rejects never lets a- // memory-pinned workload finish: caches hold gigabytes- // of re-hydratable state and the memtable holds raw- // write payloads, and 429s free none of it. Request a- // drain on the engage edge and at most once per window- // while engaged. try_send never blocks, so Thread A's- // no-lock/no-block contract above still holds.- if governor.memory_breaker_engaged() {- let fire = match last_drain_at {- None => true,- Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,- };- if fire {- last_drain_at = Some(std::time::Instant::now());- if let Some(tx) = MEMORY_DRAIN_TX.get() {- let _ = tx.try_send(());- }+ // Issue #948 / #2238 — a breaker that only rejects never lets+ // a memory-pinned workload finish: caches hold gigabytes of+ // re-hydratable state and the memtable holds raw write+ // payloads, and 429s free none of it. Request a drain on the+ // engage edge of EITHER parent latch (RSS or summed memtable)+ // and at most once per window while it stays engaged.+ // try_send never blocks, so Thread A's no-lock/no-block+ // contract above still holds.+ if pressure_drain_due(+ governor.ingest_pressure_needs_drain(),+ &mut last_drain_at,+ std::time::Instant::now(),+ MEMORY_DRAIN_MIN_INTERVAL,+ ) {+ if let Some(tx) = MEMORY_DRAIN_TX.get() {+ let _ = tx.try_send(());}- } else {- last_drain_at = None;}// Runtime disk flood-stage override, matching ES's@@ -5560,3 +5589,41 @@ mod governor_thread_spawn_tests {assert!(!started);}}++#[cfg(test)]+mod pressure_drain_tests {+ use super::pressure_drain_due;+ use std::time::{Duration, Instant};++ /// #2238: the engage edge drains immediately, a still-engaged latch does+ /// not drain again inside the window, and releasing the latch arms the+ /// next engage. `needs_drain` is the governor's memtable-or-RSS predicate.+ #[test]+ fn pressure_drain_fires_on_the_engage_edge_and_respects_the_window() {+ let mut last = None;+ let t0 = Instant::now();+ let window = Duration::from_secs(2);+ assert!(pressure_drain_due(true, &mut last, t0, window));+ assert!(+ !pressure_drain_due(true, &mut last, t0 + Duration::from_millis(500), window),+ "one engaged stretch drains once per window"+ );+ assert!(pressure_drain_due(+ true,+ &mut last,+ t0 + Duration::from_secs(2),+ window+ ));+ assert!(!pressure_drain_due(+ false,+ &mut last,+ t0 + Duration::from_secs(3),+ window+ ));+ assert!(last.is_none(), "a released latch clears the timer");+ assert!(+ pressure_drain_due(true, &mut last, t0 + Duration::from_secs(3), window),+ "the next engage drains immediately"+ );+ }+}diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 26b73fb9e..07436359d 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -150,7 +150,8 @@ impl ResourceGovernor {return Err(XerjError::circuit_breaking(format!("[parent] memtable byte budget exceeded: used={}MB, limit={}MB across all \indices; writes rejected to prevent an out-of-memory kill (raise \- limits.max_total_memtable_mb or slow ingest)",+ limits.max_process_memory_mb or XERJ_MAX_PROCESS_MEMORY_MB — the auto \+ tier derives this budget from that cap — or set limits.max_total_memtable_mb)",used / (1024 * 1024),self.memtable_budget_bytes / (1024 * 1024),)));@@ -169,7 +170,8 @@ impl ResourceGovernor {return Err(XerjError::circuit_breaking(format!("[parent] real memory circuit breaker tripped: rss={}MB {relation} watermark={}MB \({}% of limit={}MB); writes rejected to prevent an out-of-memory kill \- (raise it live with PUT _cluster/settings {{\"persistent\":\+ (raise limits.max_process_memory_mb or XERJ_MAX_PROCESS_MEMORY_MB, or live \+ with PUT _cluster/settings {{\"persistent\":\{{\"indices.breaker.total.limit\":\"<N>gb\"}}}})",rss / (1024 * 1024),watermark / (1024 * 1024),@@ -182,14 +184,34 @@ impl ResourceGovernor {/// Whether the parent RSS memory breaker is currently engaged (issue/// #948). Admission (429) is only half of the contract: the resource- /// sampler uses this same flag to request a memory-pressure DRAIN of- /// rebuildable state (flush + cache release), because a breaker that- /// only rejects never lets a workload whose caches hold gigabytes of- /// re-hydratable data make progress.+ /// sampler uses this flag, together with [`Self::memtable_breaker_engaged`],+ /// to request a memory-pressure DRAIN of rebuildable state (flush + cache+ /// release). A breaker that only rejects never lets a workload whose+ /// caches or memtables hold the bytes make progress.pub fn memory_breaker_engaged(&self) -> bool {self.memory_tripped.load(Ordering::Relaxed)}+ /// Whether the summed-memtable budget breaker is currently engaged.+ pub fn memtable_breaker_engaged(&self) -> bool {+ self.memtable_tripped.load(Ordering::Relaxed)+ }++ /// Whether the sampler must request a memory-pressure drain (#948, #2238).+ ///+ /// Both parent latches answer writes with HTTP 429, and a 429 frees+ /// nothing by itself. The RSS latch has requested a drain since #948.+ /// The summed-memtable latch did not. On the 8 GiB auto tier that budget+ /// is 2 GiB (`auto_memtable_budget`: 25% floored at 2 GiB) while each+ /// index flushes at `storage.flush_size_mb` (default 512 MiB), so a+ /// corpus spread across dozens of indices crosses the process budget+ /// with every index still under its own flush line. Writes then stay+ /// 429 and the indexed count stays 0 unless this latch requests the+ /// same drain.+ pub fn ingest_pressure_needs_drain(&self) -> bool {+ self.memory_breaker_engaged() || self.memtable_breaker_engaged()+ }+/// Whether it is safe to spawn a best-effort helper thread right now (#1122).////// `std::thread::Builder::spawn` panics inside the new thread when the@@ -1913,12 +1935,58 @@ mod tests {// `circuit_breaking_exception` type; the Display is the bare reason).assert_eq!(err.http_status(), 429);assert!(matches!(err, XerjError::CircuitBreaking { .. }));- assert!(format!("{err}").contains("memtable byte budget exceeded"));+ let rendered = format!("{err}");+ assert!(+ rendered.contains("memtable byte budget exceeded"),+ "{rendered}"+ );+ // #2238: the auto tier derives this budget from the process cap. The+ // remedy has to name that knob, and it has to survive the client's+ // 300-character per-item reason prefix.+ let marker = rendered.find("limits.max_process_memory_mb");+ assert!(+ marker.is_some_and(|at| at < 300),+ "the process-memory cap must be named inside the first 300 chars: {rendered}"+ );// Recovery once usage drops back under the budget.g.refresh(10 * 1024 * 1024, 0, 0);assert!(g.check_ingest_admission().is_ok());}+ /// #2238: a multi-index ingest on the 8 GiB auto tier trips the 2 GiB+ /// memtable budget while RSS is still under the watermark. The sampler+ /// used to drain only on the RSS latch, so every later write stayed 429+ /// and the indexed count stayed 0. Memtable pressure must request the+ /// same drain.+ #[test]+ fn memtable_pressure_requests_a_drain_while_rss_is_under_the_watermark() {+ let mut cfg = Config::default();+ cfg.limits.max_total_memtable_mb = 100;+ cfg.limits.memory_watermark_percent = 95;+ let g = build(&cfg);+ let watermark = g.memory_watermark_bytes.load(Ordering::Relaxed);+ assert!(watermark > 0, "this test needs a live RSS watermark");+ g.refresh(200 * 1024 * 1024, watermark / 2, 0);+ assert!(+ g.memtable_breaker_engaged(),+ "summed memtables are over budget"+ );+ assert!(+ !g.memory_breaker_engaged(),+ "RSS at half the watermark must not engage the RSS latch"+ );+ assert!(+ g.ingest_pressure_needs_drain(),+ "memtable pressure alone must request a drain (#2238)"+ );+ assert!(g.check_ingest_admission().is_err());++ g.refresh(10 * 1024 * 1024, watermark / 2, 0);+ assert!(!g.memtable_breaker_engaged());+ assert!(!g.ingest_pressure_needs_drain());+ assert!(g.check_ingest_admission().is_ok());+ }+/// #948: the sampler's drain wiring keys on this accessor, so it must/// track the RSS watermark latch exactly — engaging when the breaker/// engages, releasing when it releases. A stale engaged reading drains@@ -2256,6 +2324,10 @@ mod tests {err.contains("draining below watermark"),"an in-band rejection must not claim rss >= watermark: {err}");+ assert!(+ err.contains("limits.max_process_memory_mb"),+ "an RSS rejection must name the process-memory cap: {err}"+ );g.refresh(0, wm * 97 / 100, 0);assert!(g.check_ingest_admission().is_err(),diff --git a/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rs b/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rsnew file mode 100644index 000000000..124ee14c0--- /dev/null+++ b/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rs@@ -0,0 +1,178 @@+//! Issue #2238: on the 8 GiB auto memory tier the summed-memtable budget is+//! 2 GiB while each index flushes at `storage.flush_size_mb` (512 MiB). A+//! corpus spread across dozens of indices crosses the process budget with+//! every index still under its own flush line. The parent breaker then+//! answers every write HTTP 429, and until this change that latch did not+//! request a drain — so nothing flushed, the client retried, and the indexed+//! count stayed at 0.+//!+//! This test is its own binary so `governor::init` sees this config (the+//! first `Engine::new` in a process wins). One test, so a second engine+//! cannot install a different governor first.++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::governor;+use xerj_engine::Engine;+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ // 1 MiB process budget. The auto tier's 2 GiB is the same shape at a+ // size a test can fill. RSS watermark off, so only the memtable latch+ // can reject.+ config.limits.max_total_memtable_mb = 1;+ config.limits.memory_watermark_percent = 0;+ // Keep every index under its own flush line. Idle flush disabled, and+ // the periodic flusher pushed far past this test.+ config.storage.flush_size_mb = 64;+ config.storage.flush_idle_secs = 0;+ config.storage.flush_interval_secs = 3600;+ Engine::new(config).expect("engine::new")+}++fn text_schema() -> Schema {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("body", FieldType::Text));+ schema+}++async fn count_all(engine: &Engine, name: &str) -> u64 {+ let idx = engine.get_index(name).expect("get index");+ let req = parse_request(&json!({ "query": { "match_all": {} }, "size": 0, "from": 0 }))+ .expect("parse_request");+ idx.search(&req).await.expect("search").total.value+}++/// Fill four indices past the 1 MiB summed budget without any one of them+/// reaching its flush threshold, watch the next writes get rejected with the+/// indexed bytes unchanged, then drain — the request the sampler now makes+/// on this latch — and accept one more document.+#[tokio::test]+async fn summed_memtable_pressure_rejects_until_a_drain_frees_the_budget() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let names = ["a", "b", "c", "d"];+ for name in names {+ engine+ .create_index(name, text_schema())+ .expect("create index");+ }+ // One long token. The memtable charges ~3× the source bytes, so a few+ // dozen of these cross 1 MiB without a per-index flush.+ let body = format!("n{}", "a".repeat(4096));+ let mut accepted = 0usize;+ for n in 0..400 {+ let name = names[n % names.len()];+ let idx = engine.get_index(name).unwrap();+ idx.index_document(Some(format!("{name}-{n}")), json!({ "body": body }))+ .await+ .unwrap_or_else(|error| panic!("write {n} before the budget: {error}"));+ accepted += 1;+ if engine.total_memtable_bytes() >= 1024 * 1024 {+ break;+ }+ }+ let summed = engine.total_memtable_bytes();+ assert!(+ summed >= 1024 * 1024,+ "the four indices must cross the 1 MiB budget (accepted {accepted}, summed {summed} bytes)"+ );+ for name in names {+ let idx = engine.get_index(name).unwrap();+ let held = idx.memtable_bytes();+ let line = idx.flush_threshold();+ assert!(+ held < line,+ "index {name} holds {held} bytes, flush line is {line}: this must be the multi-index shape, not a per-index flush"+ );+ assert!(held > 0, "index {name} took none of the writes");+ }++ let governor = governor::global().expect("Engine::new installs the governor");+ governor.refresh_memtable(summed);+ assert!(+ !governor.memory_breaker_engaged(),+ "RSS watermark is off; only the memtable latch may engage"+ );+ assert!(governor.memtable_breaker_engaged());+ assert!(+ governor.ingest_pressure_needs_drain(),+ "memtable pressure must request the drain the sampler was not asking for"+ );+ let admission = governor.check_ingest_admission().unwrap_err().to_string();+ assert!(+ admission.contains("limits.max_process_memory_mb"),+ "{admission}"+ );++ let stuck = engine.total_memtable_bytes();+ for attempt in 0..2 {+ let err = engine+ .get_index("a")+ .unwrap()+ .index_document(Some(format!("rejected-{attempt}")), json!({ "body": "no" }))+ .await+ .expect_err("a write over the summed budget must be rejected");+ let rendered = err.to_string();+ assert!(+ rendered.contains("limits.max_process_memory_mb"),+ "attempt {attempt}: {rendered}"+ );+ assert_eq!(+ engine.total_memtable_bytes(),+ stuck,+ "attempt {attempt}: a rejected write must not move the indexed bytes"+ );+ }+ let visible_while_stuck: u64 = {+ let mut total = 0;+ for name in names {+ total += count_all(&engine, name).await;+ }+ total+ };+ assert_eq!(+ visible_while_stuck, accepted as u64,+ "the two rejected writes must not increase the searchable count (the no-progress loop)"+ );++ let drained = engine.drain_rebuildable_memory().await;+ assert_eq!(drained, names.len(), "every loaded index is drained");+ let after = engine.total_memtable_bytes();+ assert!(+ after < 1024 * 1024,+ "the drain must flush the memtables that were holding the budget (still {after} bytes)"+ );+ governor.refresh_memtable(after);+ assert!(!governor.memtable_breaker_engaged());+ assert!(!governor.ingest_pressure_needs_drain());+ assert!(governor.check_ingest_admission().is_ok());++ engine+ .get_index("a")+ .unwrap()+ .index_document(Some("after-drain".into()), json!({ "body": "progress" }))+ .await+ .expect("a write after the drain must be accepted");+ engine+ .get_index("a")+ .unwrap()+ .refresh()+ .await+ .expect("refresh");+ let mut visible = 0u64;+ for name in names {+ visible += count_all(&engine, name).await;+ }+ assert_eq!(+ visible,+ (accepted + 1) as u64,+ "the docs the breaker was sitting on, plus the one write the drain made possible"+ );+}