Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): governor threads and drain-free spawn survive a failed spawn (#1122) (#1460)

xerj-orgde1b1331f277
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index e340d24f1..7da8b05f1 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -32,6 +32,9 @@
//! same-machine CLI/MCP commands (`xerj mcp`, `xerj autoindex`, `xerj init`)
//! - [`xccode`] — the reference-coding semantics shared by `xerj code`,
//! `xerj corpus`, and the `xerj_code_search` MCP tool (issue #977)
+//! - [`thread_spawn`] — spawning helper threads without aborting the node
+//! on a failed spawn (#1122): the closure comes back, plus a process-wide
+//! memory spawn gate
//! - [`hub`] — Corpus Hub v2 shared types: the catalogue, pack manifest,
//! definition record, and the [`hub::Answer`] every surface renders (#1374)
@@ -49,6 +52,7 @@ pub mod metrics;
pub mod net;
pub mod resource;
pub mod schema;
+pub mod thread_spawn;
pub mod types;
pub mod xccode;
diff --git a/engine/crates/xerj-common/src/thread_spawn.rs b/engine/crates/xerj-common/src/thread_spawn.rs
new file mode 100644
index 000000000..e4435e893
--- /dev/null
+++ b/engine/crates/xerj-common/src/thread_spawn.rs
@@ -0,0 +1,339 @@
+//! Spawning helper threads without taking the node down (#1122).
+//!
+//! The shipped release profile is `panic = "abort"`, so every way a thread
+//! spawn can panic is a whole-node outage:
+//!
+//! * `std::thread::spawn` is `Builder::spawn(..).expect(..)` — an `EAGAIN`
+//! from `pthread_create` (thread or memory limit) aborts the process;
+//! * `Builder::spawn` returns that `Err`, but std then **panics inside the
+//! new thread** when it cannot mmap the thread's alternate signal stack
+//! under memory exhaustion — a failure the caller can never observe.
+//!
+//! The first is handled by returning the error, and — because
+//! `Builder::spawn` consumes and drops the closure on failure — by handing
+//! the closure *back* in [`SpawnFailure::job`], so the caller can run it
+//! inline, retry, or drop it deliberately. The second can only be avoided by
+//! not spawning while memory is exhausted: a process-wide spawn gate (the
+//! engine installs its governor's RSS-watermark check through
+//! [`install_spawn_gate`]) refuses the spawn up front, with the same
+//! hand-back.
+//!
+//! The actual spawn goes through the [`Spawn`] trait so the failure paths are
+//! unit-testable without exhausting a real machine.
+
+use std::io;
+use std::sync::{Arc, Mutex, OnceLock};
+use std::thread::JoinHandle;
+use std::time::Duration;
+
+/// A process-wide "is it safe to spawn a thread right now?" predicate.
+pub type SpawnGate = fn() -> bool;
+
+static GATE: OnceLock<SpawnGate> = OnceLock::new();
+
+/// Install the process-wide spawn gate. The first installation wins (the
+/// engine's governor is process-global too); returns whether this call
+/// installed it.
+pub fn install_spawn_gate(gate: SpawnGate) -> bool {
+ GATE.set(gate).is_ok()
+}
+
+/// Whether the installed gate currently allows spawning. With no gate
+/// installed (unit tests, library embedding, the CLI) spawning is
+/// unrestricted.
+pub fn spawn_allowed() -> bool {
+ GATE.get().map(|gate| gate()).unwrap_or(true)
+}
+
+/// The boxed body handed to a [`Spawn`] implementation.
+pub type BoxedJob = Box<dyn FnOnce() + Send + 'static>;
+
+/// How a named thread is actually started. [`StdSpawn`] in production; tests
+/// inject a spawner that fails.
+pub trait Spawn {
+ /// Start `job` on a new thread called `name`.
+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>>;
+}
+
+/// `std::thread::Builder` with a thread name.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct StdSpawn;
+
+impl Spawn for StdSpawn {
+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>> {
+ std::thread::Builder::new()
+ .name(name.to_string())
+ .spawn(job)
+ }
+}
+
+/// A spawn that did not happen. `job` is the closure that would have run on
+/// the thread, returned to the caller so it can degrade (run it inline),
+/// retry, or drop it on purpose. It is `None` only if the spawner reported a
+/// failure *after* running the job, which [`StdSpawn`] never does.
+pub struct SpawnFailure<F> {
+ /// Why the thread was not started.
+ pub error: io::Error,
+ /// The un-run job, handed back.
+ pub job: Option<F>,
+}
+
+impl<F> std::fmt::Debug for SpawnFailure<F> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("SpawnFailure")
+ .field("error", &self.error)
+ .field("job_returned", &self.job.is_some())
+ .finish()
+ }
+}
+
+impl<F> std::fmt::Display for SpawnFailure<F> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ self.error.fmt(f)
+ }
+}
+
+/// The error a gate refusal carries.
+fn gate_refused(name: &str) -> io::Error {
+ io::Error::new(
+ io::ErrorKind::OutOfMemory,
+ format!(
+ "not spawning thread {name}: memory is at the governor watermark, \
+ and a spawn under memory exhaustion aborts the process (#1122)"
+ ),
+ )
+}
+
+/// Spawn a named thread, honouring the process-wide spawn gate. Never
+/// panics on spawn failure; see [`SpawnFailure`].
+pub fn try_spawn<F>(name: &str, f: F) -> Result<JoinHandle<()>, SpawnFailure<F>>
+where
+ F: FnOnce() + Send + 'static,
+{
+ try_spawn_with(&StdSpawn, spawn_allowed(), name, f)
+}
+
+/// [`try_spawn`] with an explicit spawner and gate verdict (for tests and
+/// callers that already sampled the gate).
+pub fn try_spawn_with<S, F>(
+ spawner: &S,
+ allowed: bool,
+ name: &str,
+ f: F,
+) -> Result<JoinHandle<()>, SpawnFailure<F>>
+where
+ S: Spawn + ?Sized,
+ F: FnOnce() + Send + 'static,
+{
+ if !allowed {
+ return Err(SpawnFailure {
+ error: gate_refused(name),
+ job: Some(f),
+ });
+ }
+ // The thread takes the job out of the slot when it runs. If the spawn
+ // fails, the thread never ran, and the job is still in the slot even
+ // though the spawner dropped the boxed wrapper.
+ let slot = Arc::new(Mutex::new(Some(f)));
+ let thread_slot = Arc::clone(&slot);
+ let wrapper: BoxedJob = Box::new(move || {
+ let job = thread_slot
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .take();
+ if let Some(job) = job {
+ job();
+ }
+ });
+ match spawner.spawn(name, wrapper) {
+ Ok(handle) => Ok(handle),
+ Err(error) => {
+ let job = slot
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .take();
+ Err(SpawnFailure { error, job })
+ }
+ }
+}
+
+/// [`try_spawn`] retried up to `attempts` times (at least once), sleeping
+/// `backoff`, then twice that, … between attempts. For threads that must
+/// exist (samplers, workers) where a transient `EAGAIN` or a brief trip of
+/// the memory watermark should not cost the thread for the life of the
+/// process. The gate is re-sampled on every attempt.
+pub fn spawn_with_retry<F>(
+ name: &str,
+ attempts: u32,
+ backoff: Duration,
+ f: F,
+) -> Result<JoinHandle<()>, SpawnFailure<F>>
+where
+ F: FnOnce() + Send + 'static,
+{
+ spawn_with_retry_using(&StdSpawn, spawn_allowed, name, attempts, backoff, f)
+}
+
+/// [`spawn_with_retry`] with an explicit spawner and gate.
+pub fn spawn_with_retry_using<S, G, F>(
+ spawner: &S,
+ gate: G,
+ name: &str,
+ attempts: u32,
+ backoff: Duration,
+ f: F,
+) -> Result<JoinHandle<()>, SpawnFailure<F>>
+where
+ S: Spawn + ?Sized,
+ G: Fn() -> bool,
+ F: FnOnce() + Send + 'static,
+{
+ let attempts = attempts.max(1);
+ let mut job = f;
+ let mut delay = backoff;
+ let mut attempt = 1;
+ loop {
+ match try_spawn_with(spawner, gate(), name, job) {
+ Ok(handle) => return Ok(handle),
+ Err(failure) => {
+ let Some(returned) = failure.job else {
+ return Err(failure);
+ };
+ if attempt >= attempts {
+ return Err(SpawnFailure {
+ error: failure.error,
+ job: Some(returned),
+ });
+ }
+ job = returned;
+ std::thread::sleep(delay);
+ delay = delay.saturating_mul(2);
+ attempt += 1;
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
+
+ /// Fails the first `fail_first` spawns with EAGAIN, then spawns for real.
+ struct FlakySpawn {
+ fail_first: u32,
+ calls: AtomicU32,
+ }
+
+ impl FlakySpawn {
+ fn new(fail_first: u32) -> Self {
+ Self {
+ fail_first,
+ calls: AtomicU32::new(0),
+ }
+ }
+ }
+
+ impl Spawn for FlakySpawn {
+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>> {
+ let call = self.calls.fetch_add(1, Ordering::SeqCst);
+ if call < self.fail_first {
+ drop(job); // exactly what Builder::spawn does on failure
+ return Err(io::Error::from_raw_os_error(libc::EAGAIN));
+ }
+ StdSpawn.spawn(name, job)
+ }
+ }
+
+ #[test]
+ fn a_failed_spawn_returns_the_job_instead_of_panicking() {
+ let ran = Arc::new(AtomicUsize::new(0));
+ let job_ran = Arc::clone(&ran);
+ let failure = try_spawn_with(&FlakySpawn::new(1), true, "t1122", move || {
+ job_ran.fetch_add(1, Ordering::SeqCst);
+ })
+ .expect_err("the spawner fails");
+ assert_eq!(failure.error.raw_os_error(), Some(libc::EAGAIN));
+ assert_eq!(ran.load(Ordering::SeqCst), 0, "nothing ran on a thread");
+ // The caller can degrade by running the job inline.
+ (failure.job.expect("job handed back"))();
+ assert_eq!(ran.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn a_closed_gate_refuses_without_calling_the_spawner() {
+ let spawner = FlakySpawn::new(0);
+ let failure = try_spawn_with(&spawner, false, "t1122", || {}).expect_err("gate closed");
+ assert_eq!(failure.error.kind(), io::ErrorKind::OutOfMemory);
+ assert!(failure.job.is_some());
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn a_successful_spawn_runs_the_job_once_on_the_named_thread() {
+ let (tx, rx) = std::sync::mpsc::channel();
+ let handle = try_spawn_with(&StdSpawn, true, "xerj-t1122", move || {
+ let name = std::thread::current().name().map(str::to_string);
+ tx.send(name).unwrap();
+ })
+ .expect("spawn");
+ handle.join().unwrap();
+ assert_eq!(rx.recv().unwrap().as_deref(), Some("xerj-t1122"));
+ assert!(rx.try_recv().is_err(), "ran exactly once");
+ }
+
+ #[test]
+ fn retry_rides_out_transient_failures() {
+ let spawner = FlakySpawn::new(2);
+ let ran = Arc::new(AtomicUsize::new(0));
+ let job_ran = Arc::clone(&ran);
+ let handle = spawn_with_retry_using(
+ &spawner,
+ || true,
+ "t1122",
+ 3,
+ Duration::from_millis(1),
+ move || {
+ job_ran.fetch_add(1, Ordering::SeqCst);
+ },
+ )
+ .expect("third attempt succeeds");
+ handle.join().unwrap();
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 3);
+ assert_eq!(ran.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn retry_gives_up_after_the_attempt_budget_and_hands_the_job_back() {
+ let spawner = FlakySpawn::new(u32::MAX);
+ let failure = spawn_with_retry_using(
+ &spawner,
+ || true,
+ "t1122",
+ 3,
+ Duration::from_millis(1),
+ || {},
+ )
+ .expect_err("never succeeds");
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 3);
+ assert!(failure.job.is_some());
+ }
+
+ #[test]
+ fn retry_resamples_the_gate_each_attempt() {
+ let gate_calls = AtomicU32::new(0);
+ let spawner = FlakySpawn::new(0);
+ let handle = spawn_with_retry_using(
+ &spawner,
+ || gate_calls.fetch_add(1, Ordering::SeqCst) >= 1,
+ "t1122",
+ 2,
+ Duration::from_millis(1),
+ || {},
+ )
+ .expect("gate opens on the second attempt");
+ handle.join().unwrap();
+ assert_eq!(gate_calls.load(Ordering::SeqCst), 2);
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 1);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 79985d2e7..047b9f13e 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -41,6 +41,64 @@ const MEMORY_DRAIN_MIN_INTERVAL: std::time::Duration = std::time::Duration::from
static MEMORY_DRAIN_TX: std::sync::OnceLock<std::sync::mpsc::SyncSender<()>> =
std::sync::OnceLock::new();
+/// Spawn attempts (and the first back-off) for the governor's sampler and
+/// drain threads. They are started once, at boot; losing one disables the
+/// parent memory breaker for the life of the process, so a transient
+/// `EAGAIN` is worth riding out.
+const GOVERNOR_THREAD_SPAWN_ATTEMPTS: u32 = 3;
+const GOVERNOR_THREAD_SPAWN_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50);
+
+/// Start one of the governor's background threads (#1122).
+///
+/// These used to be `let _ = Builder::spawn(..)`: a failed spawn was dropped
+/// on the floor, and the node silently ran with no memory breaker. Now the
+/// spawn is retried with back-off through `xerj_common::thread_spawn` (which
+/// never panics), and a final failure is logged at ERROR naming the thread
+/// and what the node loses. Returns whether the thread is running.
+///
+/// These threads deliberately do NOT consult the governor's spawn gate
+/// (`thread_spawn_safe`). The gate is a soft RSS watermark, not an OS
+/// allocation failure, and these are the threads that sample RSS, engage the
+/// breaker and drain memory. A node that boots above its watermark (a large
+/// WAL replay under a low `XERJ_MAX_PROCESS_MEMORY_MB`) would otherwise have
+/// every attempt refused and run with no breaker for the life of the process,
+/// which is exactly when the breaker is needed. A real spawn failure still
+/// comes back as an `Err` and is logged, not aborted.
+fn spawn_governor_thread<F>(name: &str, f: F) -> bool
+where
+ F: FnOnce() + Send + 'static,
+{
+ spawn_governor_thread_using(&xerj_common::thread_spawn::StdSpawn, || true, name, f)
+}
+
+fn spawn_governor_thread_using<S, G, F>(spawner: &S, gate: G, name: &str, f: F) -> bool
+where
+ S: xerj_common::thread_spawn::Spawn,
+ G: Fn() -> bool,
+ F: FnOnce() + Send + 'static,
+{
+ match xerj_common::thread_spawn::spawn_with_retry_using(
+ spawner,
+ gate,
+ name,
+ GOVERNOR_THREAD_SPAWN_ATTEMPTS,
+ GOVERNOR_THREAD_SPAWN_BACKOFF,
+ f,
+ ) {
+ Ok(_detached) => true,
+ Err(failure) => {
+ tracing::error!(
+ thread = name,
+ attempts = GOVERNOR_THREAD_SPAWN_ATTEMPTS,
+ error = %failure.error,
+ "could not start a resource-governor thread; the parent memory/disk \
+ breaker is degraded until restart"
+ );
+ false
+ }
+ }
+}
+
/// Allocator page purge hook, installed by the binary that owns the global
/// allocator (xerj-server installs a jemalloc epoch+purge closure). Freeing
/// heap is not enough: the RSS breaker keys on resident pages, and jemalloc
@@ -3952,28 +4010,30 @@ impl Engine {
// instead of dying with this thread's runtime.
let drain_handle = tokio::runtime::Handle::try_current().ok();
let (drain_tx, drain_rx) = std::sync::mpsc::sync_channel::<()>(1);
- let _ = MEMORY_DRAIN_TX.set(drain_tx);
let weak_c = Arc::downgrade(self);
- let _ = std::thread::Builder::new()
- .name("xerj-mem-drain".to_string())
- .spawn(move || {
- let Some(handle) = drain_handle else {
- // Spawned outside a runtime (unit tests): the drain
- // method stays directly callable; only the sampler's
- // automatic wiring is inert.
- return;
+ let drain_worker = spawn_governor_thread("xerj-mem-drain", move || {
+ let Some(handle) = drain_handle else {
+ // Spawned outside a runtime (unit tests): the drain
+ // method stays directly callable; only the sampler's
+ // automatic wiring is inert.
+ return;
+ };
+ while let Ok(()) = drain_rx.recv() {
+ let Some(engine) = weak_c.upgrade() else {
+ return; // engine dropped
};
- while let Ok(()) = drain_rx.recv() {
- let Some(engine) = weak_c.upgrade() else {
- return; // engine dropped
- };
- let drained = handle.block_on(engine.drain_rebuildable_memory());
- tracing::info!(
+ let drained = handle.block_on(engine.drain_rebuildable_memory());
+ tracing::info!(
indexes = drained,
"memory breaker engaged — drained rebuildable state (flush + cache release + purge)"
);
- }
- });
+ }
+ });
+ // Publish the request channel only once a worker is draining it: with
+ // no worker, the sampler's requests have nowhere to go.
+ if drain_worker {
+ let _ = MEMORY_DRAIN_TX.set(drain_tx);
+ }
// ── Thread A: memory + disk ──────────────────────────────────────
// Touches NO engine lock — just `/proc/self/statm` + `statvfs`. Kept
@@ -3991,123 +4051,118 @@ impl Engine {
// Snapshot the uncapped machine memory once: a percentage override is
// resolved against it, and it is the hard clamp for an absolute one.
let machine_memory_bytes = crate::governor::effective_memory_limit_bytes();
- let _ = std::thread::Builder::new()
- .name("xerj-mem-sampler".to_string())
- .spawn(move || {
- tracing::info!(
- period_ms = crate::governor::SAMPLE_INTERVAL_MS,
- "memory/disk sampler thread started (parent circuit breaker)"
- );
- // #948 drain bookkeeping: None while the breaker is
- // disengaged (so the next engage edge fires immediately),
- // Some(last-drain) while engaged (rate-limited repeats).
- let mut last_drain_at: Option<std::time::Instant> = None;
- loop {
- std::thread::sleep(period);
- if weak_a.strong_count() == 0 {
- return; // engine dropped
- }
- let governor = match crate::governor::global() {
- Some(g) => g,
- None => return,
+ spawn_governor_thread("xerj-mem-sampler", move || {
+ tracing::info!(
+ period_ms = crate::governor::SAMPLE_INTERVAL_MS,
+ "memory/disk sampler thread started (parent circuit breaker)"
+ );
+ // #948 drain bookkeeping: None while the breaker is
+ // disengaged (so the next engage edge fires immediately),
+ // Some(last-drain) while engaged (rate-limited repeats).
+ let mut last_drain_at: Option<std::time::Instant> = None;
+ loop {
+ std::thread::sleep(period);
+ if weak_a.strong_count() == 0 {
+ return; // engine dropped
+ }
+ let governor = match crate::governor::global() {
+ Some(g) => g,
+ None => return,
+ };
+ let rss = crate::governor::current_rss_bytes();
+ let (disk_pct, disk_avail) = crate::governor::disk_used_pct_and_avail(&data_dir);
+ governor.refresh_memory_disk(rss, disk_pct, disk_avail);
+
+ // Issue #948 — a breaker that only rejects never lets a
+ // memory-pinned workload finish: caches hold gigabytes
+ // of re-hydratable state and the memtable holds raw
+ // write payloads, and 429s free none of it. Request a
+ // drain on the engage edge and at most once per window
+ // while engaged. try_send never blocks, so Thread A's
+ // no-lock/no-block contract above still holds.
+ if governor.memory_breaker_engaged() {
+ let fire = match last_drain_at {
+ None => true,
+ Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,
};
- let rss = crate::governor::current_rss_bytes();
- let (disk_pct, disk_avail) =
- crate::governor::disk_used_pct_and_avail(&data_dir);
- governor.refresh_memory_disk(rss, disk_pct, disk_avail);
-
- // Issue #948 — a breaker that only rejects never lets a
- // memory-pinned workload finish: caches hold gigabytes
- // of re-hydratable state and the memtable holds raw
- // write payloads, and 429s free none of it. Request a
- // drain on the engage edge and at most once per window
- // while engaged. try_send never blocks, so Thread A's
- // no-lock/no-block contract above still holds.
- if governor.memory_breaker_engaged() {
- let fire = match last_drain_at {
- None => true,
- Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,
- };
- if fire {
- last_drain_at = Some(std::time::Instant::now());
- if let Some(tx) = MEMORY_DRAIN_TX.get() {
- let _ = tx.try_send(());
- }
+ if fire {
+ last_drain_at = Some(std::time::Instant::now());
+ if let Some(tx) = MEMORY_DRAIN_TX.get() {
+ let _ = tx.try_send(());
}
- } else {
- last_drain_at = None;
}
+ } else {
+ last_drain_at = None;
+ }
- // Runtime disk flood-stage override, matching ES's
- // `PUT _cluster/settings` unblock without a restart. A
- // non-blocking read: on contention we just keep last
- // tick's threshold and retry next sample, never stall
- // the memory/disk breaker on this lock.
- if let Ok(settings) = cluster_settings.try_read() {
- let flood_override = settings
- .get("persistent")
- .and_then(|p| p.get(FLOOD_STAGE_SETTING))
- .or_else(|| {
- settings
- .get("transient")
- .and_then(|t| t.get(FLOOD_STAGE_SETTING))
- })
- .and_then(crate::governor::parse_flood_stage_pct);
- governor.set_disk_flood_pct_override(flood_override);
-
- // `threshold_enabled: false` turns the flood stage
- // off and releases an engaged block, as in ES.
- let threshold_enabled = settings
- .get("persistent")
- .and_then(|p| p.get(DISK_THRESHOLD_SETTING))
- .or_else(|| {
- settings
- .get("transient")
- .and_then(|t| t.get(DISK_THRESHOLD_SETTING))
- })
- .and_then(crate::governor::parse_setting_bool);
- governor.set_disk_threshold_enabled(threshold_enabled);
-
- // Runtime parent-memory-breaker watermark override
- // (#1250): raise the watermark on a big box live via
- // `PUT _cluster/settings {indices.breaker.total.limit}`,
- // no restart — the live equivalent of the boot-only
- // XERJ_MAX_PROCESS_MEMORY_MB cap.
- let breaker_override = settings
- .get("persistent")
- .and_then(|p| p.get(BREAKER_LIMIT_SETTING))
- .or_else(|| {
- settings
- .get("transient")
- .and_then(|t| t.get(BREAKER_LIMIT_SETTING))
- })
- .and_then(|v| {
- crate::governor::parse_breaker_total_limit(v, machine_memory_bytes)
- });
- governor.set_memory_watermark_override(breaker_override);
- }
+ // Runtime disk flood-stage override, matching ES's
+ // `PUT _cluster/settings` unblock without a restart. A
+ // non-blocking read: on contention we just keep last
+ // tick's threshold and retry next sample, never stall
+ // the memory/disk breaker on this lock.
+ if let Ok(settings) = cluster_settings.try_read() {
+ let flood_override = settings
+ .get("persistent")
+ .and_then(|p| p.get(FLOOD_STAGE_SETTING))
+ .or_else(|| {
+ settings
+ .get("transient")
+ .and_then(|t| t.get(FLOOD_STAGE_SETTING))
+ })
+ .and_then(crate::governor::parse_flood_stage_pct);
+ governor.set_disk_flood_pct_override(flood_override);
+
+ // `threshold_enabled: false` turns the flood stage
+ // off and releases an engaged block, as in ES.
+ let threshold_enabled = settings
+ .get("persistent")
+ .and_then(|p| p.get(DISK_THRESHOLD_SETTING))
+ .or_else(|| {
+ settings
+ .get("transient")
+ .and_then(|t| t.get(DISK_THRESHOLD_SETTING))
+ })
+ .and_then(crate::governor::parse_setting_bool);
+ governor.set_disk_threshold_enabled(threshold_enabled);
+
+ // Runtime parent-memory-breaker watermark override
+ // (#1250): raise the watermark on a big box live via
+ // `PUT _cluster/settings {indices.breaker.total.limit}`,
+ // no restart — the live equivalent of the boot-only
+ // XERJ_MAX_PROCESS_MEMORY_MB cap.
+ let breaker_override = settings
+ .get("persistent")
+ .and_then(|p| p.get(BREAKER_LIMIT_SETTING))
+ .or_else(|| {
+ settings
+ .get("transient")
+ .and_then(|t| t.get(BREAKER_LIMIT_SETTING))
+ })
+ .and_then(|v| {
+ crate::governor::parse_breaker_total_limit(v, machine_memory_bytes)
+ });
+ governor.set_memory_watermark_override(breaker_override);
}
- });
+ }
+ });
// ── Thread B: summed memtable budget ─────────────────────────────
// Each per-index `size_bytes()` is one relaxed atomic load of the
// aggregate the shards maintain incrementally — no shard locks, so
// this never blocks under a turbo batch (#872).
let weak_b = Arc::downgrade(self);
- let _ = std::thread::Builder::new()
- .name("xerj-memtable-sampler".to_string())
- .spawn(move || loop {
- std::thread::sleep(period);
- let engine = match weak_b.upgrade() {
- Some(e) => e,
- None => return, // engine dropped
- };
- let governor = match crate::governor::global() {
- Some(g) => g,
- None => return,
- };
- governor.refresh_memtable(engine.total_memtable_bytes());
- });
+ spawn_governor_thread("xerj-memtable-sampler", move || loop {
+ std::thread::sleep(period);
+ let engine = match weak_b.upgrade() {
+ Some(e) => e,
+ None => return, // engine dropped
+ };
+ let governor = match crate::governor::global() {
+ Some(g) => g,
+ None => return,
+ };
+ governor.refresh_memtable(engine.total_memtable_bytes());
+ });
}
// ── Transform pipeline methods ────────────────────────────────────────────
@@ -5106,3 +5161,53 @@ mod api_key_record_migration_tests {
assert!(!engine.has_open_index("alpha"));
}
}
+
+#[cfg(test)]
+mod governor_thread_spawn_tests {
+ use super::spawn_governor_thread_using;
+ use xerj_common::thread_spawn::{BoxedJob, Spawn};
+
+ /// A spawner that always fails the way `pthread_create` does at a
+ /// thread or memory limit.
+ struct AlwaysEagain;
+
+ impl Spawn for AlwaysEagain {
+ fn spawn(
+ &self,
+ _name: &str,
+ _job: BoxedJob,
+ ) -> std::io::Result<std::thread::JoinHandle<()>> {
+ Err(std::io::Error::from_raw_os_error(libc::EAGAIN))
+ }
+ }
+
+ #[test]
+ fn a_governor_thread_that_cannot_spawn_reports_false_instead_of_aborting() {
+ let ran = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
+ let job_ran = std::sync::Arc::clone(&ran);
+ let started = spawn_governor_thread_using(
+ &AlwaysEagain,
+ || true,
+ "xerj-test",
+ move || {
+ job_ran.store(true, std::sync::atomic::Ordering::SeqCst);
+ },
+ );
+ assert!(!started);
+ assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));
+ }
+
+ #[test]
+ fn a_closed_spawn_gate_is_respected() {
+ // A working spawner, but the gate is closed. Production passes an
+ // always-open gate for these threads (see `spawn_governor_thread`);
+ // this pins the plumbing for a caller that does gate.
+ let started = spawn_governor_thread_using(
+ &xerj_common::thread_spawn::StdSpawn,
+ || false,
+ "xerj-test",
+ || {},
+ );
+ assert!(!started);
+ }
+}
diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rs
index 3625b6f7b..d0b485905 100644
--- a/engine/crates/xerj-engine/src/governor.rs
+++ b/engine/crates/xerj-engine/src/governor.rs
@@ -814,7 +814,12 @@ fn resolve_segment_hydration_budget(
/// call wins (subsequent calls return the already-installed governor, so a
/// second `Engine` in-process — e.g. a test — does not re-key the budgets).
pub fn init(config: &Config) -> Arc<ResourceGovernor> {
- Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))))
+ let governor = Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))));
+ // #1122: every crate that spawns through `xerj_common::thread_spawn`
+ // (storage, server, …) consults this same watermark check, so a spawn
+ // under memory exhaustion is refused instead of aborting the node.
+ xerj_common::thread_spawn::install_spawn_gate(thread_spawn_safe);
+ governor
}
/// The installed governor, if [`init`] has run. Engine-only unit tests that
diff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rs
index 16bd1f5f0..268a2d253 100644
--- a/engine/crates/xerj-engine/src/memtable.rs
+++ b/engine/crates/xerj-engine/src/memtable.rs
@@ -2809,12 +2809,13 @@ impl FrozenClaim {
// the shipped profile is panic=abort, so under memory pressure the
// spawn is not attempted at all — the Arcs drop inline right here,
// on the same fallback path the comment always intended.
- if crate::governor::thread_spawn_safe() {
- let _ = std::thread::Builder::new()
- .name("xerj-drain-free".to_string())
- .spawn(move || drop(dead));
- } else {
- drop(dead);
+ // `try_spawn` consults the governor's spawn gate (installed by
+ // `governor::init`) and hands the job back on a refusal or a failed
+ // spawn; dropping the returned job frees the Arcs inline.
+ if let Err(failure) =
+ xerj_common::thread_spawn::try_spawn("xerj-drain-free", move || drop(dead))
+ {
+ drop(failure.job);
}
guard
}