Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): drain on memtable pressure so corpus index makes progress (#2238) (#2244)

xerj-orgc16fad5877ec
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 9e2807884..a452d3777 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -25,6 +25,9 @@ pub struct Es {
/// How long one `bulk` call keeps re-offering records the server
/// rejected with a per-item 429 (see [`THROTTLE_PATIENCE`]).
throttle_patience: Duration,
+ /// How long a bulk that has not had a single item accepted keeps
+ /// re-asking a parent memory breaker (see [`ZERO_PROGRESS_PATIENCE`]).
+ zero_progress_patience: Duration,
/// Most actions this client puts in one `_bulk` request before the server
/// has said anything about it. See [`BULK_MAX_ACTIONS`]; tests shrink it.
bulk_max_actions: usize,
@@ -624,6 +627,53 @@ impl BulkAdmission {
/// 5xx and write blocks are NOT waited on — they are not congestion.
pub const THROTTLE_PATIENCE: Duration = Duration::from_secs(600);
+/// How long a bulk that has accepted nothing will keep re-asking the parent
+/// memory breaker before the run stops and names the cap (#2238).
+///
+/// [`THROTTLE_PATIENCE`] stays the budget once any item in the bulk has been
+/// accepted: a drain that is freeing memory (the #944 mailbox held the
+/// breaker for 57 s before drains existed; releases after #948 are seconds)
+/// must be allowed to finish. A bulk the server has not taken a single item
+/// from is the other shape — the 8 GiB auto tier's breaker loop, indexed
+/// count stuck at 0 for several minutes with no statement of what to change.
+/// Ninety seconds is past that 57 s hold and past one drain of the auto
+/// tier's 2 GiB memtable budget, and short of a multi-minute spin.
+pub const ZERO_PROGRESS_PATIENCE: Duration = Duration::from_secs(90);
+
+/// The parent memory breaker, as opposed to a disk flood-stage 429 or a
+/// thread-pool rejection. Both of its reasons carry one of these phrases.
+pub(crate) fn is_parent_memory_breaker(reason: &str) -> bool {
+ reason.contains("circuit breaker") || reason.contains("memtable byte budget")
+}
+
+/// Name `limits.max_process_memory_mb` on a parent-breaker 429 that does not
+/// already name it (#2238). Other failures are returned unchanged, and a
+/// reason that already carries the knob is not annotated twice.
+pub(crate) fn memory_cap_hint(reason: &str) -> String {
+ const MARKER: &str = "limits.max_process_memory_mb";
+ if reason.contains(MARKER) || !is_parent_memory_breaker(reason) {
+ return reason.to_owned();
+ }
+ format!(
+ "{reason} — raise {MARKER} or XERJ_MAX_PROCESS_MEMORY_MB \
+ (the auto memory tier derives this breaker's limit from that cap; 0 = whole machine)"
+ )
+}
+
+/// The back-pressure stop, with the process-memory cap named when the
+/// rejection is the parent breaker.
+pub(crate) fn backpressure_exhausted(
+ items: u64,
+ patience: Duration,
+ reason: &str,
+) -> BackpressureExhausted {
+ BackpressureExhausted {
+ items,
+ patience,
+ reason: memory_cap_hint(reason),
+ }
+}
+
/// How often the wait is announced on stderr, so a run that is waiting on
/// the server does not look like a hang.
const NOTICE_EVERY: Duration = Duration::from_secs(30);
@@ -948,7 +998,7 @@ fn parse_round(v: &Value) -> Round {
op["error"]
.get("reason")
.and_then(Value::as_str)
- .map(|r| r.chars().take(300).collect())
+ .map(|r| r.chars().take(SERVER_REASON_MAX).collect())
.unwrap_or_else(error_text),
);
}
@@ -987,6 +1037,10 @@ pub struct BulkOutcome {
/// failure that says "come back later" rather than "something is broken".
/// Callers use it to name the stop as back-pressure, not as an abort.
pub throttled_out: u64,
+ /// The patience budget that elapsed when `throttled_out` was set.
+ /// `None` until a bulk gives up. The full [`THROTTLE_PATIENCE`] once any
+ /// item was accepted; [`ZERO_PROGRESS_PATIENCE`] when none was (#2238).
+ pub exhausted_after: Option<Duration>,
}
impl BulkOutcome {
@@ -1001,6 +1055,9 @@ impl BulkOutcome {
if self.first_server_error.is_none() {
self.first_server_error = other.first_server_error;
}
+ if self.exhausted_after.is_none() {
+ self.exhausted_after = other.exhausted_after;
+ }
}
}
@@ -1161,6 +1218,7 @@ impl Es {
retry_max_delay,
admission: Arc::new(BulkAdmission::off()),
throttle_patience: THROTTLE_PATIENCE,
+ zero_progress_patience: ZERO_PROGRESS_PATIENCE,
bulk_max_actions: BULK_MAX_ACTIONS,
request_size: Arc::new(RequestSizeLedger::default()),
throttle_ledger: Arc::new(ThrottleLedger::default()),
@@ -1187,6 +1245,14 @@ impl Es {
self
}
+ /// How long a bulk the server has not accepted a single item from keeps
+ /// re-asking a parent memory breaker (tests shrink it).
+ #[cfg(test)]
+ fn with_zero_progress_patience(mut self, patience: Duration) -> Self {
+ self.zero_progress_patience = patience;
+ self
+ }
+
/// How long one `bulk` call waits on a server that is accepting nothing
/// before the rejection is handed back — what the terminal line quotes.
pub fn throttle_patience(&self) -> Duration {
@@ -1479,7 +1545,11 @@ impl Es {
/// re-asking at the capped backoff for up to `throttle_patience`,
/// announcing the wait every [`NOTICE_EVERY`]; a one-shot client (gate
/// off — a probe, a status read) treats a 429 like a 5xx and gives up
- /// after six, because a 10-minute wait is wrong for a question.
+ /// after six, because a 10-minute wait is wrong for a question. A
+ /// parent memory breaker that has accepted nothing is the exception
+ /// inside that budget: it stops at [`ZERO_PROGRESS_PATIENCE`] and names
+ /// `limits.max_process_memory_mb` (#2238), because waiting out ten
+ /// minutes cannot shrink a budget the drain is not being asked to free.
/// Measured reason for the patience: the parent memory circuit breaker
/// answered a bulk with HTTP 429 and held the server above the
/// watermark for 57 s on the 1 GB mailbox run — the six attempts
@@ -1501,20 +1571,33 @@ impl Es {
Attempt::Abort(e) => return Err(e),
Attempt::Throttled(reason) => {
self.admission.on_congestion();
+ let raw = reason.as_deref().unwrap_or("HTTP 429");
let err = match &reason {
- Some(r) => anyhow!("{what}: HTTP 429 Too Many Requests: {r}"),
+ Some(r) => {
+ anyhow!("{what}: HTTP 429 Too Many Requests: {}", memory_cap_hint(r))
+ }
None => anyhow!("{what}: HTTP 429 Too Many Requests"),
};
if self.admission.enabled() {
let waited = started.elapsed();
- if waited >= self.throttle_patience {
+ // A whole-request 429 accepted nothing. The parent
+ // memory breaker stops at the short budget; every
+ // other 429 (disk flood, thread-pool rejection) keeps
+ // the full patience, because that condition can clear.
+ let limit = if is_parent_memory_breaker(raw) {
+ self.throttle_patience.min(self.zero_progress_patience)
+ } else {
+ self.throttle_patience
+ };
+ if waited >= limit {
return Err(err);
}
self.throttle_notice(
&mut last_notice,
- reason.as_deref().unwrap_or("HTTP 429"),
+ raw,
&format!("retrying {what}"),
waited,
+ limit,
);
} else {
failures += 1;
@@ -1577,14 +1660,16 @@ impl Es {
reason: &str,
doing: &str,
waited: Duration,
+ giving_up_after: Duration,
) {
if !self.admission.announces() || matches!(last, Some(t) if t.elapsed() < NOTICE_EVERY) {
return;
}
+ let reason = memory_cap_hint(reason);
eprintln!(
"autoindex: server is shedding load — {reason}; {doing} (waited {}s, giving up after {}s)",
waited.as_secs(),
- self.throttle_patience.as_secs()
+ giving_up_after.as_secs()
);
*last = Some(Instant::now());
}
@@ -1794,15 +1879,21 @@ impl Es {
/// 429/5xx retries the WHOLE request (`retry_loop`). A 200 whose items
/// carry `status: 429` re-offers ONLY those items, after a backoff, for up
/// to `throttle_patience` — the memory breaker's shape, see
- /// [`THROTTLE_PATIENCE`]. Records the server took are never sent twice,
- /// and a record it refused for a reason waiting cannot fix (a per-item
- /// 400, 5xx, or a write block) is not re-offered.
+ /// [`THROTTLE_PATIENCE`]. A parent memory breaker that has not accepted a
+ /// single item of this bulk stops at [`ZERO_PROGRESS_PATIENCE`] instead
+ /// (#2238). Records the server took are never sent twice, and a record it
+ /// refused for a reason waiting cannot fix (a per-item 400, 5xx, or a
+ /// write block) is not re-offered.
fn bulk_window(&self, body: Vec<u8>) -> Result<SentWindow> {
let mut body = body;
// Item errors settled by earlier rounds of this call (a bad record's
// 400): the record is not re-offered, its error is still reported.
let mut settled_item_errors = 0u64;
let mut settled_first_error: Option<String> = None;
+ // Set once any round took an item. From then on the full patience
+ // applies, even if later rounds are all 429: a drain in progress
+ // (#944 held 57 s) must be allowed to finish.
+ let mut accepted_any = false;
let episode = Instant::now();
let mut delay = self.retry_initial_delay;
let mut last_notice: Option<Instant> = None;
@@ -1819,14 +1910,30 @@ impl Es {
if round.throttled.is_empty() || round.other_server_errors > 0 {
return Ok(SentWindow::Done(outcome));
}
+ if round.items > round.throttled.len() {
+ accepted_any = true;
+ }
// Only per-item 429s stand between this bulk and a clean
// outcome. Wait while patience lasts; after that the caller gets
// the server's own reason, exactly as before — but named as
// back-pressure (#944), so the run's terminal line can say
// `reason=server-backpressure` rather than `reason=aborted`.
+ // Zero accepts against the parent memory breaker use the short
+ // budget (#2238). Any other 429, and any bulk that has landed an
+ // item, keeps the full patience.
+ let parent = round
+ .throttle_reason
+ .as_deref()
+ .is_some_and(is_parent_memory_breaker);
+ let limit = if accepted_any || !parent {
+ self.throttle_patience
+ } else {
+ self.throttle_patience.min(self.zero_progress_patience)
+ };
let waited = episode.elapsed();
- if waited >= self.throttle_patience {
+ if waited >= limit {
outcome.throttled_out = round.throttled.len() as u64;
+ outcome.exhausted_after = Some(limit);
return Ok(SentWindow::Done(outcome));
}
// A response whose item count does not match the body cannot be
@@ -1848,6 +1955,7 @@ impl Es {
.unwrap_or("per-item HTTP 429"),
&format!("re-offering {} rejected record(s)", round.throttled.len()),
waited,
+ limit,
);
self.throttle_ledger
.reissues
@@ -2523,6 +2631,7 @@ mod tests {
use std::io::Read;
use std::io::Write;
use std::net::TcpListener;
+ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc, Mutex};
use std::time::{Duration, Instant};
@@ -4017,6 +4126,160 @@ mod tests {
assert_eq!(outcome.server_errors, 2);
}
+ /// Answer every connection with `body` until `stop` is set. The
+ /// zero-progress tests need a server that outlasts the short budget.
+ fn serve_until(
+ listener: TcpListener,
+ body: Vec<u8>,
+ stop: Arc<AtomicBool>,
+ ) -> std::thread::JoinHandle<()> {
+ listener.set_nonblocking(true).unwrap();
+ std::thread::spawn(move || loop {
+ match listener.accept() {
+ Ok((mut stream, _)) => {
+ stream.set_nonblocking(false).unwrap();
+ let _ = read_request(&mut stream);
+ respond_json(&mut stream, &body);
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ if stop.load(Ordering::SeqCst) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(2));
+ }
+ Err(error) => panic!("accept: {error}"),
+ }
+ })
+ }
+
+ /// #2238: every item 429 from the parent breaker, forever. The run must
+ /// stop at the zero-progress budget and name `limits.max_process_memory_mb`,
+ /// not sit out the full ten-minute patience with the indexed count at 0.
+ #[test]
+ fn a_parent_breaker_that_accepts_nothing_stops_at_zero_progress_patience() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let stop = Arc::new(AtomicBool::new(false));
+ let server = serve_until(
+ listener,
+ items_response(
+ &[item("a", 429, Some(BREAKER)), item("b", 429, Some(BREAKER))],
+ true,
+ ),
+ Arc::clone(&stop),
+ );
+ let es = client(address, 4)
+ .with_throttle_patience(Duration::from_secs(2))
+ .with_zero_progress_patience(Duration::from_millis(50));
+ let started = Instant::now();
+ let outcome = es.bulk([pair("a"), pair("b")].concat()).unwrap();
+ let elapsed = started.elapsed();
+ stop.store(true, Ordering::SeqCst);
+ server.join().unwrap();
+ assert!(
+ elapsed < Duration::from_millis(500),
+ "zero accepts must not wait the 2s patience, took {elapsed:?}"
+ );
+ assert_eq!(
+ outcome.exhausted_after,
+ Some(Duration::from_millis(50)),
+ "{outcome:?}"
+ );
+ assert!(outcome.throttled_out >= 1, "{outcome:?}");
+ let backoffs = es.backoff_delays.lock().unwrap().len();
+ assert!(
+ (1..=10).contains(&backoffs),
+ "retried inside the short budget, did not spin: {backoffs} backoffs"
+ );
+ let hinted = super::memory_cap_hint(outcome.first_server_error.as_deref().unwrap());
+ assert!(hinted.contains("limits.max_process_memory_mb"), "{hinted}");
+ }
+
+ /// #944/#1250: once any item of the bulk has been accepted, a later run of
+ /// all-429 answers keeps the full patience. Fail-fast applies only to a
+ /// bulk the server has not taken a single item from.
+ #[test]
+ fn a_bulk_that_landed_an_item_keeps_the_full_patience() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ listener.set_nonblocking(true).unwrap();
+ let stop = Arc::new(AtomicBool::new(false));
+ let server_stop = Arc::clone(&stop);
+ let server = std::thread::spawn(move || {
+ let mut first = true;
+ loop {
+ match listener.accept() {
+ Ok((mut stream, _)) => {
+ stream.set_nonblocking(false).unwrap();
+ let _ = read_request(&mut stream);
+ if first {
+ first = false;
+ respond_json(
+ &mut stream,
+ &items_response(
+ &[item("a", 201, None), item("b", 429, Some(BREAKER))],
+ true,
+ ),
+ );
+ } else {
+ items_answered(&mut stream, &[429]);
+ }
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ if server_stop.load(Ordering::SeqCst) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(2));
+ }
+ Err(error) => panic!("accept: {error}"),
+ }
+ }
+ });
+ let es = client(address, 4)
+ .with_throttle_patience(Duration::from_millis(300))
+ .with_zero_progress_patience(Duration::from_millis(20));
+ let started = Instant::now();
+ let outcome = es.bulk([pair("a"), pair("b")].concat()).unwrap();
+ let elapsed = started.elapsed();
+ stop.store(true, Ordering::SeqCst);
+ server.join().unwrap();
+ assert!(
+ elapsed >= Duration::from_millis(250),
+ "an accepted item must keep the 300ms patience, stopped after {elapsed:?}"
+ );
+ assert!(
+ elapsed < Duration::from_secs(2),
+ "the full patience is 300ms, took {elapsed:?}"
+ );
+ assert_eq!(
+ outcome.exhausted_after,
+ Some(Duration::from_millis(300)),
+ "{outcome:?}"
+ );
+ assert!(outcome.throttled_out >= 1, "{outcome:?}");
+ }
+
+ #[test]
+ fn backpressure_exhausted_names_the_process_memory_cap_once() {
+ let error = super::backpressure_exhausted(4, Duration::from_secs(90), BREAKER);
+ let rendered = error.to_string();
+ assert!(rendered.contains("after 90s"), "{rendered}");
+ assert!(rendered.contains("circuit breaker"), "{rendered}");
+ assert!(
+ rendered.contains("limits.max_process_memory_mb"),
+ "{rendered}"
+ );
+ assert_eq!(
+ super::memory_cap_hint(&error.reason),
+ error.reason,
+ "a reason that already names the cap is not annotated twice"
+ );
+ assert_eq!(
+ super::memory_cap_hint("mapper_parsing_exception"),
+ "mapper_parsing_exception"
+ );
+ }
+
#[test]
fn bulk_item_selection_keeps_lines_byte_identical_and_knows_delete_has_no_source() {
let del = b"{\"delete\":{\"_index\":\"i\",\"_id\":\"x\"}}\n".to_vec();
@@ -4144,6 +4407,57 @@ mod tests {
server.join().unwrap();
}
+ /// A plain HTTP 429 (no bulk `items`) from the parent breaker is the same
+ /// zero-progress loop: stop at the short budget and name the cap. A probe
+ /// (admission off) still stops after six attempts — covered above.
+ #[test]
+ fn a_plain_parent_breaker_429_names_the_cap_and_stops_short() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let stop = Arc::new(AtomicBool::new(false));
+ let server_stop = Arc::clone(&stop);
+ listener.set_nonblocking(true).unwrap();
+ let server = std::thread::spawn(move || loop {
+ match listener.accept() {
+ Ok((mut stream, _)) => {
+ stream.set_nonblocking(false).unwrap();
+ let _ = read_request(&mut stream);
+ let body = br#"{"error":{"type":"circuit_breaking_exception","reason":"[parent] real memory circuit breaker tripped: rss=9MB >= watermark=8MB"},"status":429}"#;
+ respond_status(&mut stream, "429 Too Many Requests", body);
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
+ if server_stop.load(Ordering::SeqCst) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(2));
+ }
+ Err(error) => panic!("accept: {error}"),
+ }
+ });
+ let es = client(address, 4)
+ .with_throttle_patience(Duration::from_secs(2))
+ .with_zero_progress_patience(Duration::from_millis(40));
+ let started = Instant::now();
+ let error = es.bulk(BULK.to_vec()).unwrap_err();
+ let elapsed = started.elapsed();
+ stop.store(true, Ordering::SeqCst);
+ server.join().unwrap();
+ let rendered = format!("{error:#}");
+ assert!(
+ rendered.contains("limits.max_process_memory_mb"),
+ "{rendered}"
+ );
+ assert!(
+ elapsed < Duration::from_millis(500),
+ "a parent breaker that accepts nothing must not wait 2s, took {elapsed:?}"
+ );
+ let backoffs = es.backoff_delays.lock().unwrap().len();
+ assert!(
+ (1..=8).contains(&backoffs),
+ "retried inside the short budget: {backoffs} backoffs"
+ );
+ }
+
// ── #955: a request is bounded in actions and bytes, and a request the
// server still calls too large is halved instead of ending the run ──
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index b76eec0ae..7fc9118cf 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -2539,6 +2539,16 @@ fn replacement_failpoint(_boundary: u8, _state_dir: &Path) -> Result<()> {
Ok(())
}
+/// The first server error, with `limits.max_process_memory_mb` named when
+/// the rejection is the parent memory breaker (#2238).
+fn server_error_text(outcome: &crate::esclient::BulkOutcome) -> String {
+ outcome
+ .first_server_error
+ .as_deref()
+ .map(crate::esclient::memory_cap_hint)
+ .unwrap_or_else(|| "unknown server error".into())
+}
+
/// Send one bulk body and fold its per-item rejections into
/// `rejected_records`.
///
@@ -2570,10 +2580,7 @@ fn record_bulk_outcome(
complete; fix the server condition (or wait for it to clear) and rerun \
autoindex — the run resumes from the journal",
outcome.server_errors,
- outcome
- .first_server_error
- .as_deref()
- .unwrap_or("unknown server error")
+ server_error_text(&outcome)
));
return true;
}
@@ -2883,10 +2890,7 @@ where
complete; fix the server condition (or wait for it to clear) and rerun \
autoindex — the run resumes from the journal",
outcome.server_errors,
- outcome
- .first_server_error
- .as_deref()
- .unwrap_or("unknown server error")
+ server_error_text(&outcome)
);
for t in &tokens {
st.failed.insert(*t);
@@ -14041,6 +14045,7 @@ mod bulk_pipe_tests {
first_error: None,
first_server_error: None,
throttled_out: 0,
+ exhausted_after: None,
}
}
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index ee04751a5..567f8feb1 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -2679,28 +2679,35 @@ fn check_bulk_outcome(
outcome: crate::esclient::BulkOutcome,
) -> Result<()> {
// `Es::bulk` has already re-sent per-item 429s, and only those, for the
- // whole of its patience (`THROTTLE_PATIENCE`, #944/#949).
- // What is left is either a server condition that did not clear — fatal,
- // and resumable, because a sealed operation is journaled applied only
- // after this returns — or a record the server refused outright.
+ // whole of its patience (`THROTTLE_PATIENCE`, #944/#949, shortened to
+ // `ZERO_PROGRESS_PATIENCE` when a parent memory breaker accepts nothing,
+ // #2238). What is left is either a server condition that did not clear —
+ // fatal, and resumable, because a sealed operation is journaled applied
+ // only after this returns — or a record the server refused outright.
if outcome.throttled_out > 0 {
- return Err(anyhow::Error::new(crate::esclient::BackpressureExhausted {
- items: outcome.server_errors,
- patience: es.throttle_patience(),
- reason: outcome
- .first_server_error
- .unwrap_or_else(|| "unknown server error".into()),
- }));
+ let patience = outcome
+ .exhausted_after
+ .unwrap_or_else(|| es.throttle_patience());
+ let reason = outcome
+ .first_server_error
+ .unwrap_or_else(|| "unknown server error".into());
+ return Err(anyhow::Error::new(crate::esclient::backpressure_exhausted(
+ outcome.server_errors,
+ patience,
+ &reason,
+ )));
}
if outcome.server_errors > 0 {
+ let reason = outcome
+ .first_server_error
+ .as_deref()
+ .map(crate::esclient::memory_cap_hint)
+ .unwrap_or_else(|| "unknown server error".into());
anyhow::bail!(
- "the server failed {} of a prepared bulk's items: {}. Nothing from this bulk was \
+ "the server failed {} of a prepared bulk's items: {reason}. Nothing from this bulk was \
journaled applied; fix the reported server condition and rerun the same command — \
the run resumes from its last committed operation",
outcome.server_errors,
- outcome
- .first_server_error
- .unwrap_or_else(|| "unknown server error".into())
);
}
anyhow::ensure!(
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 2da91959d..bc7e4be4d 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -21,13 +21,16 @@ use crate::{EngineError, Result};
// ── Memory-pressure drain plumbing (issue #948) ─────────────────────────────
//
-// The parent RSS breaker used to only REJECT writes (429). Under a workload
+// The parent breaker used to only REJECT writes (429). Under a workload
// whose caches hold gigabytes of re-hydratable state and whose memtable
// holds raw write payloads, rejection frees nothing: the breaker stays
-// engaged, the client retries forever, and the run never finishes — exactly
+// engaged, the client retries, and the run never finishes — exactly
// the #948 mailbox ingest (16 GiB cap, 26.5 GiB demand, 118-147 s engaged
-// stretches, 3-4 s releases). The sampler now also requests a DRAIN: flush
-// every index, release rebuildable caches, purge allocator pages.
+// stretches, 3-4 s releases) and the #2238 corpus-index loop (8 GiB auto
+// tier, dozens of indices under their own flush line but over the 2 GiB
+// summed-memtable budget, indexed count stuck at 0). The sampler requests
+// a DRAIN on either latch: flush every index, release rebuildable caches,
+// purge allocator pages.
/// Minimum spacing between automatic drains while the breaker stays engaged.
/// The engage edge always fires immediately; repeats are rate-limited so a
@@ -35,6 +38,35 @@ use crate::{EngineError, Result};
/// reads) without the drain worker competing with ingest for every tick.
const MEMORY_DRAIN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);
+/// Whether this sampler tick should ask the drain worker to run (#948, #2238).
+///
+/// `needs_drain` is [`crate::governor::ResourceGovernor::ingest_pressure_needs_drain`]:
+/// the RSS latch or the summed-memtable latch. A memtable-only trip used to
+/// return false here, so a multi-index ingest that had filled the auto tier's
+/// 2 GiB memtable budget kept receiving HTTP 429 with nothing flushed and the
+/// indexed count stayed at 0. The engage edge fires immediately; while the
+/// latch stays up, repeats wait `min_interval`. Releasing the latch clears
+/// the timer so the next engage drains at once.
+fn pressure_drain_due(
+ needs_drain: bool,
+ last_drain_at: &mut Option<std::time::Instant>,
+ now: std::time::Instant,
+ min_interval: std::time::Duration,
+) -> bool {
+ if !needs_drain {
+ *last_drain_at = None;
+ return false;
+ }
+ let due = match *last_drain_at {
+ None => true,
+ Some(at) => now.saturating_duration_since(at) >= min_interval,
+ };
+ if due {
+ *last_drain_at = Some(now);
+ }
+ due
+}
+
/// Bounded request channel from the sampler thread to the drain worker.
/// Capacity 1 + try_send ⇒ requests coalesce: one arrives mid-drain is
/// dropped, and the next sampler tick re-requests if still engaged.
@@ -4222,26 +4254,23 @@ impl Engine {
let (disk_pct, disk_avail) = crate::governor::disk_used_pct_and_avail(&data_dir);
governor.refresh_memory_disk(rss, disk_pct, disk_avail);
- // Issue #948 — a breaker that only rejects never lets a
- // memory-pinned workload finish: caches hold gigabytes
- // of re-hydratable state and the memtable holds raw
- // write payloads, and 429s free none of it. Request a
- // drain on the engage edge and at most once per window
- // while engaged. try_send never blocks, so Thread A's
- // no-lock/no-block contract above still holds.
- if governor.memory_breaker_engaged() {
- let fire = match last_drain_at {
- None => true,
- Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,
- };
- if fire {
- last_drain_at = Some(std::time::Instant::now());
- if let Some(tx) = MEMORY_DRAIN_TX.get() {
- let _ = tx.try_send(());
- }
+ // Issue #948 / #2238 — a breaker that only rejects never lets
+ // a memory-pinned workload finish: caches hold gigabytes of
+ // re-hydratable state and the memtable holds raw write
+ // payloads, and 429s free none of it. Request a drain on the
+ // engage edge of EITHER parent latch (RSS or summed memtable)
+ // and at most once per window while it stays engaged.
+ // try_send never blocks, so Thread A's no-lock/no-block
+ // contract above still holds.
+ if pressure_drain_due(
+ governor.ingest_pressure_needs_drain(),
+ &mut last_drain_at,
+ std::time::Instant::now(),
+ MEMORY_DRAIN_MIN_INTERVAL,
+ ) {
+ if let Some(tx) = MEMORY_DRAIN_TX.get() {
+ let _ = tx.try_send(());
}
- } else {
- last_drain_at = None;
}
// Runtime disk flood-stage override, matching ES's
@@ -5560,3 +5589,41 @@ mod governor_thread_spawn_tests {
assert!(!started);
}
}
+
+#[cfg(test)]
+mod pressure_drain_tests {
+ use super::pressure_drain_due;
+ use std::time::{Duration, Instant};
+
+ /// #2238: the engage edge drains immediately, a still-engaged latch does
+ /// not drain again inside the window, and releasing the latch arms the
+ /// next engage. `needs_drain` is the governor's memtable-or-RSS predicate.
+ #[test]
+ fn pressure_drain_fires_on_the_engage_edge_and_respects_the_window() {
+ let mut last = None;
+ let t0 = Instant::now();
+ let window = Duration::from_secs(2);
+ assert!(pressure_drain_due(true, &mut last, t0, window));
+ assert!(
+ !pressure_drain_due(true, &mut last, t0 + Duration::from_millis(500), window),
+ "one engaged stretch drains once per window"
+ );
+ assert!(pressure_drain_due(
+ true,
+ &mut last,
+ t0 + Duration::from_secs(2),
+ window
+ ));
+ assert!(!pressure_drain_due(
+ false,
+ &mut last,
+ t0 + Duration::from_secs(3),
+ window
+ ));
+ assert!(last.is_none(), "a released latch clears the timer");
+ assert!(
+ pressure_drain_due(true, &mut last, t0 + Duration::from_secs(3), window),
+ "the next engage drains immediately"
+ );
+ }
+}
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 26b73fb9e..07436359d 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -150,7 +150,8 @@ impl ResourceGovernor {
return Err(XerjError::circuit_breaking(format!(
"[parent] memtable byte budget exceeded: used={}MB, limit={}MB across all \
indices; writes rejected to prevent an out-of-memory kill (raise \
- limits.max_total_memtable_mb or slow ingest)",
+ limits.max_process_memory_mb or XERJ_MAX_PROCESS_MEMORY_MB — the auto \
+ tier derives this budget from that cap — or set limits.max_total_memtable_mb)",
used / (1024 * 1024),
self.memtable_budget_bytes / (1024 * 1024),
)));
@@ -169,7 +170,8 @@ impl ResourceGovernor {
return Err(XerjError::circuit_breaking(format!(
"[parent] real memory circuit breaker tripped: rss={}MB {relation} watermark={}MB \
({}% of limit={}MB); writes rejected to prevent an out-of-memory kill \
- (raise it live with PUT _cluster/settings {{\"persistent\":\
+ (raise limits.max_process_memory_mb or XERJ_MAX_PROCESS_MEMORY_MB, or live \
+ with PUT _cluster/settings {{\"persistent\":\
{{\"indices.breaker.total.limit\":\"<N>gb\"}}}})",
rss / (1024 * 1024),
watermark / (1024 * 1024),
@@ -182,14 +184,34 @@ impl ResourceGovernor {
/// Whether the parent RSS memory breaker is currently engaged (issue
/// #948). Admission (429) is only half of the contract: the resource
- /// sampler uses this same flag to request a memory-pressure DRAIN of
- /// rebuildable state (flush + cache release), because a breaker that
- /// only rejects never lets a workload whose caches hold gigabytes of
- /// re-hydratable data make progress.
+ /// sampler uses this flag, together with [`Self::memtable_breaker_engaged`],
+ /// to request a memory-pressure DRAIN of rebuildable state (flush + cache
+ /// release). A breaker that only rejects never lets a workload whose
+ /// caches or memtables hold the bytes make progress.
pub fn memory_breaker_engaged(&self) -> bool {
self.memory_tripped.load(Ordering::Relaxed)
}
+ /// Whether the summed-memtable budget breaker is currently engaged.
+ pub fn memtable_breaker_engaged(&self) -> bool {
+ self.memtable_tripped.load(Ordering::Relaxed)
+ }
+
+ /// Whether the sampler must request a memory-pressure drain (#948, #2238).
+ ///
+ /// Both parent latches answer writes with HTTP 429, and a 429 frees
+ /// nothing by itself. The RSS latch has requested a drain since #948.
+ /// The summed-memtable latch did not. On the 8 GiB auto tier that budget
+ /// is 2 GiB (`auto_memtable_budget`: 25% floored at 2 GiB) while each
+ /// index flushes at `storage.flush_size_mb` (default 512 MiB), so a
+ /// corpus spread across dozens of indices crosses the process budget
+ /// with every index still under its own flush line. Writes then stay
+ /// 429 and the indexed count stays 0 unless this latch requests the
+ /// same drain.
+ pub fn ingest_pressure_needs_drain(&self) -> bool {
+ self.memory_breaker_engaged() || self.memtable_breaker_engaged()
+ }
+
/// Whether it is safe to spawn a best-effort helper thread right now (#1122).
///
/// `std::thread::Builder::spawn` panics inside the new thread when the
@@ -1913,12 +1935,58 @@ mod tests {
// `circuit_breaking_exception` type; the Display is the bare reason).
assert_eq!(err.http_status(), 429);
assert!(matches!(err, XerjError::CircuitBreaking { .. }));
- assert!(format!("{err}").contains("memtable byte budget exceeded"));
+ let rendered = format!("{err}");
+ assert!(
+ rendered.contains("memtable byte budget exceeded"),
+ "{rendered}"
+ );
+ // #2238: the auto tier derives this budget from the process cap. The
+ // remedy has to name that knob, and it has to survive the client's
+ // 300-character per-item reason prefix.
+ let marker = rendered.find("limits.max_process_memory_mb");
+ assert!(
+ marker.is_some_and(|at| at < 300),
+ "the process-memory cap must be named inside the first 300 chars: {rendered}"
+ );
// Recovery once usage drops back under the budget.
g.refresh(10 * 1024 * 1024, 0, 0);
assert!(g.check_ingest_admission().is_ok());
}
+ /// #2238: a multi-index ingest on the 8 GiB auto tier trips the 2 GiB
+ /// memtable budget while RSS is still under the watermark. The sampler
+ /// used to drain only on the RSS latch, so every later write stayed 429
+ /// and the indexed count stayed 0. Memtable pressure must request the
+ /// same drain.
+ #[test]
+ fn memtable_pressure_requests_a_drain_while_rss_is_under_the_watermark() {
+ let mut cfg = Config::default();
+ cfg.limits.max_total_memtable_mb = 100;
+ cfg.limits.memory_watermark_percent = 95;
+ let g = build(&cfg);
+ let watermark = g.memory_watermark_bytes.load(Ordering::Relaxed);
+ assert!(watermark > 0, "this test needs a live RSS watermark");
+ g.refresh(200 * 1024 * 1024, watermark / 2, 0);
+ assert!(
+ g.memtable_breaker_engaged(),
+ "summed memtables are over budget"
+ );
+ assert!(
+ !g.memory_breaker_engaged(),
+ "RSS at half the watermark must not engage the RSS latch"
+ );
+ assert!(
+ g.ingest_pressure_needs_drain(),
+ "memtable pressure alone must request a drain (#2238)"
+ );
+ assert!(g.check_ingest_admission().is_err());
+
+ g.refresh(10 * 1024 * 1024, watermark / 2, 0);
+ assert!(!g.memtable_breaker_engaged());
+ assert!(!g.ingest_pressure_needs_drain());
+ assert!(g.check_ingest_admission().is_ok());
+ }
+
/// #948: the sampler's drain wiring keys on this accessor, so it must
/// track the RSS watermark latch exactly — engaging when the breaker
/// engages, releasing when it releases. A stale engaged reading drains
@@ -2256,6 +2324,10 @@ mod tests {
err.contains("draining below watermark"),
"an in-band rejection must not claim rss >= watermark: {err}"
);
+ assert!(
+ err.contains("limits.max_process_memory_mb"),
+ "an RSS rejection must name the process-memory cap: {err}"
+ );
g.refresh(0, wm * 97 / 100, 0);
assert!(
g.check_ingest_admission().is_err(),
diff --git a/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rs b/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rs
new file mode 100644
index 000000000..124ee14c0
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/memtable_budget_drain_unblocks_ingest.rs
@@ -0,0 +1,178 @@
+//! Issue #2238: on the 8 GiB auto memory tier the summed-memtable budget is
+//! 2 GiB while each index flushes at `storage.flush_size_mb` (512 MiB). A
+//! corpus spread across dozens of indices crosses the process budget with
+//! every index still under its own flush line. The parent breaker then
+//! answers every write HTTP 429, and until this change that latch did not
+//! request a drain — so nothing flushed, the client retried, and the indexed
+//! count stayed at 0.
+//!
+//! This test is its own binary so `governor::init` sees this config (the
+//! first `Engine::new` in a process wins). One test, so a second engine
+//! cannot install a different governor first.
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::governor;
+use xerj_engine::Engine;
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ // 1 MiB process budget. The auto tier's 2 GiB is the same shape at a
+ // size a test can fill. RSS watermark off, so only the memtable latch
+ // can reject.
+ config.limits.max_total_memtable_mb = 1;
+ config.limits.memory_watermark_percent = 0;
+ // Keep every index under its own flush line. Idle flush disabled, and
+ // the periodic flusher pushed far past this test.
+ config.storage.flush_size_mb = 64;
+ config.storage.flush_idle_secs = 0;
+ config.storage.flush_interval_secs = 3600;
+ Engine::new(config).expect("engine::new")
+}
+
+fn text_schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("body", FieldType::Text));
+ schema
+}
+
+async fn count_all(engine: &Engine, name: &str) -> u64 {
+ let idx = engine.get_index(name).expect("get index");
+ let req = parse_request(&json!({ "query": { "match_all": {} }, "size": 0, "from": 0 }))
+ .expect("parse_request");
+ idx.search(&req).await.expect("search").total.value
+}
+
+/// Fill four indices past the 1 MiB summed budget without any one of them
+/// reaching its flush threshold, watch the next writes get rejected with the
+/// indexed bytes unchanged, then drain — the request the sampler now makes
+/// on this latch — and accept one more document.
+#[tokio::test]
+async fn summed_memtable_pressure_rejects_until_a_drain_frees_the_budget() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let names = ["a", "b", "c", "d"];
+ for name in names {
+ engine
+ .create_index(name, text_schema())
+ .expect("create index");
+ }
+ // One long token. The memtable charges ~3× the source bytes, so a few
+ // dozen of these cross 1 MiB without a per-index flush.
+ let body = format!("n{}", "a".repeat(4096));
+ let mut accepted = 0usize;
+ for n in 0..400 {
+ let name = names[n % names.len()];
+ let idx = engine.get_index(name).unwrap();
+ idx.index_document(Some(format!("{name}-{n}")), json!({ "body": body }))
+ .await
+ .unwrap_or_else(|error| panic!("write {n} before the budget: {error}"));
+ accepted += 1;
+ if engine.total_memtable_bytes() >= 1024 * 1024 {
+ break;
+ }
+ }
+ let summed = engine.total_memtable_bytes();
+ assert!(
+ summed >= 1024 * 1024,
+ "the four indices must cross the 1 MiB budget (accepted {accepted}, summed {summed} bytes)"
+ );
+ for name in names {
+ let idx = engine.get_index(name).unwrap();
+ let held = idx.memtable_bytes();
+ let line = idx.flush_threshold();
+ assert!(
+ held < line,
+ "index {name} holds {held} bytes, flush line is {line}: this must be the multi-index shape, not a per-index flush"
+ );
+ assert!(held > 0, "index {name} took none of the writes");
+ }
+
+ let governor = governor::global().expect("Engine::new installs the governor");
+ governor.refresh_memtable(summed);
+ assert!(
+ !governor.memory_breaker_engaged(),
+ "RSS watermark is off; only the memtable latch may engage"
+ );
+ assert!(governor.memtable_breaker_engaged());
+ assert!(
+ governor.ingest_pressure_needs_drain(),
+ "memtable pressure must request the drain the sampler was not asking for"
+ );
+ let admission = governor.check_ingest_admission().unwrap_err().to_string();
+ assert!(
+ admission.contains("limits.max_process_memory_mb"),
+ "{admission}"
+ );
+
+ let stuck = engine.total_memtable_bytes();
+ for attempt in 0..2 {
+ let err = engine
+ .get_index("a")
+ .unwrap()
+ .index_document(Some(format!("rejected-{attempt}")), json!({ "body": "no" }))
+ .await
+ .expect_err("a write over the summed budget must be rejected");
+ let rendered = err.to_string();
+ assert!(
+ rendered.contains("limits.max_process_memory_mb"),
+ "attempt {attempt}: {rendered}"
+ );
+ assert_eq!(
+ engine.total_memtable_bytes(),
+ stuck,
+ "attempt {attempt}: a rejected write must not move the indexed bytes"
+ );
+ }
+ let visible_while_stuck: u64 = {
+ let mut total = 0;
+ for name in names {
+ total += count_all(&engine, name).await;
+ }
+ total
+ };
+ assert_eq!(
+ visible_while_stuck, accepted as u64,
+ "the two rejected writes must not increase the searchable count (the no-progress loop)"
+ );
+
+ let drained = engine.drain_rebuildable_memory().await;
+ assert_eq!(drained, names.len(), "every loaded index is drained");
+ let after = engine.total_memtable_bytes();
+ assert!(
+ after < 1024 * 1024,
+ "the drain must flush the memtables that were holding the budget (still {after} bytes)"
+ );
+ governor.refresh_memtable(after);
+ assert!(!governor.memtable_breaker_engaged());
+ assert!(!governor.ingest_pressure_needs_drain());
+ assert!(governor.check_ingest_admission().is_ok());
+
+ engine
+ .get_index("a")
+ .unwrap()
+ .index_document(Some("after-drain".into()), json!({ "body": "progress" }))
+ .await
+ .expect("a write after the drain must be accepted");
+ engine
+ .get_index("a")
+ .unwrap()
+ .refresh()
+ .await
+ .expect("refresh");
+ let mut visible = 0u64;
+ for name in names {
+ visible += count_all(&engine, name).await;
+ }
+ assert_eq!(
+ visible,
+ (accepted + 1) as u64,
+ "the docs the breaker was sitting on, plus the one write the drain made possible"
+ );
+}