AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): governor threads and drain-free spawn survive a failed spawn (#1122) (#1460)
xerj-orgde1b1331f277
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rsindex e340d24f1..7da8b05f1 100644--- a/engine/crates/xerj-common/src/lib.rs+++ b/engine/crates/xerj-common/src/lib.rs@@ -32,6 +32,9 @@//! same-machine CLI/MCP commands (`xerj mcp`, `xerj autoindex`, `xerj init`)//! - [`xccode`] — the reference-coding semantics shared by `xerj code`,//! `xerj corpus`, and the `xerj_code_search` MCP tool (issue #977)+//! - [`thread_spawn`] — spawning helper threads without aborting the node+//! on a failed spawn (#1122): the closure comes back, plus a process-wide+//! memory spawn gate//! - [`hub`] — Corpus Hub v2 shared types: the catalogue, pack manifest,//! definition record, and the [`hub::Answer`] every surface renders (#1374)@@ -49,6 +52,7 @@ pub mod metrics;pub mod net;pub mod resource;pub mod schema;+pub mod thread_spawn;pub mod types;pub mod xccode;diff --git a/engine/crates/xerj-common/src/thread_spawn.rs b/engine/crates/xerj-common/src/thread_spawn.rsnew file mode 100644index 000000000..e4435e893--- /dev/null+++ b/engine/crates/xerj-common/src/thread_spawn.rs@@ -0,0 +1,339 @@+//! Spawning helper threads without taking the node down (#1122).+//!+//! The shipped release profile is `panic = "abort"`, so every way a thread+//! spawn can panic is a whole-node outage:+//!+//! * `std::thread::spawn` is `Builder::spawn(..).expect(..)` — an `EAGAIN`+//! from `pthread_create` (thread or memory limit) aborts the process;+//! * `Builder::spawn` returns that `Err`, but std then **panics inside the+//! new thread** when it cannot mmap the thread's alternate signal stack+//! under memory exhaustion — a failure the caller can never observe.+//!+//! The first is handled by returning the error, and — because+//! `Builder::spawn` consumes and drops the closure on failure — by handing+//! the closure *back* in [`SpawnFailure::job`], so the caller can run it+//! inline, retry, or drop it deliberately. The second can only be avoided by+//! not spawning while memory is exhausted: a process-wide spawn gate (the+//! engine installs its governor's RSS-watermark check through+//! [`install_spawn_gate`]) refuses the spawn up front, with the same+//! hand-back.+//!+//! The actual spawn goes through the [`Spawn`] trait so the failure paths are+//! unit-testable without exhausting a real machine.++use std::io;+use std::sync::{Arc, Mutex, OnceLock};+use std::thread::JoinHandle;+use std::time::Duration;++/// A process-wide "is it safe to spawn a thread right now?" predicate.+pub type SpawnGate = fn() -> bool;++static GATE: OnceLock<SpawnGate> = OnceLock::new();++/// Install the process-wide spawn gate. The first installation wins (the+/// engine's governor is process-global too); returns whether this call+/// installed it.+pub fn install_spawn_gate(gate: SpawnGate) -> bool {+ GATE.set(gate).is_ok()+}++/// Whether the installed gate currently allows spawning. With no gate+/// installed (unit tests, library embedding, the CLI) spawning is+/// unrestricted.+pub fn spawn_allowed() -> bool {+ GATE.get().map(|gate| gate()).unwrap_or(true)+}++/// The boxed body handed to a [`Spawn`] implementation.+pub type BoxedJob = Box<dyn FnOnce() + Send + 'static>;++/// How a named thread is actually started. [`StdSpawn`] in production; tests+/// inject a spawner that fails.+pub trait Spawn {+ /// Start `job` on a new thread called `name`.+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>>;+}++/// `std::thread::Builder` with a thread name.+#[derive(Debug, Clone, Copy, Default)]+pub struct StdSpawn;++impl Spawn for StdSpawn {+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>> {+ std::thread::Builder::new()+ .name(name.to_string())+ .spawn(job)+ }+}++/// A spawn that did not happen. `job` is the closure that would have run on+/// the thread, returned to the caller so it can degrade (run it inline),+/// retry, or drop it on purpose. It is `None` only if the spawner reported a+/// failure *after* running the job, which [`StdSpawn`] never does.+pub struct SpawnFailure<F> {+ /// Why the thread was not started.+ pub error: io::Error,+ /// The un-run job, handed back.+ pub job: Option<F>,+}++impl<F> std::fmt::Debug for SpawnFailure<F> {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("SpawnFailure")+ .field("error", &self.error)+ .field("job_returned", &self.job.is_some())+ .finish()+ }+}++impl<F> std::fmt::Display for SpawnFailure<F> {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ self.error.fmt(f)+ }+}++/// The error a gate refusal carries.+fn gate_refused(name: &str) -> io::Error {+ io::Error::new(+ io::ErrorKind::OutOfMemory,+ format!(+ "not spawning thread {name}: memory is at the governor watermark, \+ and a spawn under memory exhaustion aborts the process (#1122)"+ ),+ )+}++/// Spawn a named thread, honouring the process-wide spawn gate. Never+/// panics on spawn failure; see [`SpawnFailure`].+pub fn try_spawn<F>(name: &str, f: F) -> Result<JoinHandle<()>, SpawnFailure<F>>+where+ F: FnOnce() + Send + 'static,+{+ try_spawn_with(&StdSpawn, spawn_allowed(), name, f)+}++/// [`try_spawn`] with an explicit spawner and gate verdict (for tests and+/// callers that already sampled the gate).+pub fn try_spawn_with<S, F>(+ spawner: &S,+ allowed: bool,+ name: &str,+ f: F,+) -> Result<JoinHandle<()>, SpawnFailure<F>>+where+ S: Spawn + ?Sized,+ F: FnOnce() + Send + 'static,+{+ if !allowed {+ return Err(SpawnFailure {+ error: gate_refused(name),+ job: Some(f),+ });+ }+ // The thread takes the job out of the slot when it runs. If the spawn+ // fails, the thread never ran, and the job is still in the slot even+ // though the spawner dropped the boxed wrapper.+ let slot = Arc::new(Mutex::new(Some(f)));+ let thread_slot = Arc::clone(&slot);+ let wrapper: BoxedJob = Box::new(move || {+ let job = thread_slot+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .take();+ if let Some(job) = job {+ job();+ }+ });+ match spawner.spawn(name, wrapper) {+ Ok(handle) => Ok(handle),+ Err(error) => {+ let job = slot+ .lock()+ .unwrap_or_else(|poisoned| poisoned.into_inner())+ .take();+ Err(SpawnFailure { error, job })+ }+ }+}++/// [`try_spawn`] retried up to `attempts` times (at least once), sleeping+/// `backoff`, then twice that, … between attempts. For threads that must+/// exist (samplers, workers) where a transient `EAGAIN` or a brief trip of+/// the memory watermark should not cost the thread for the life of the+/// process. The gate is re-sampled on every attempt.+pub fn spawn_with_retry<F>(+ name: &str,+ attempts: u32,+ backoff: Duration,+ f: F,+) -> Result<JoinHandle<()>, SpawnFailure<F>>+where+ F: FnOnce() + Send + 'static,+{+ spawn_with_retry_using(&StdSpawn, spawn_allowed, name, attempts, backoff, f)+}++/// [`spawn_with_retry`] with an explicit spawner and gate.+pub fn spawn_with_retry_using<S, G, F>(+ spawner: &S,+ gate: G,+ name: &str,+ attempts: u32,+ backoff: Duration,+ f: F,+) -> Result<JoinHandle<()>, SpawnFailure<F>>+where+ S: Spawn + ?Sized,+ G: Fn() -> bool,+ F: FnOnce() + Send + 'static,+{+ let attempts = attempts.max(1);+ let mut job = f;+ let mut delay = backoff;+ let mut attempt = 1;+ loop {+ match try_spawn_with(spawner, gate(), name, job) {+ Ok(handle) => return Ok(handle),+ Err(failure) => {+ let Some(returned) = failure.job else {+ return Err(failure);+ };+ if attempt >= attempts {+ return Err(SpawnFailure {+ error: failure.error,+ job: Some(returned),+ });+ }+ job = returned;+ std::thread::sleep(delay);+ delay = delay.saturating_mul(2);+ attempt += 1;+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};++ /// Fails the first `fail_first` spawns with EAGAIN, then spawns for real.+ struct FlakySpawn {+ fail_first: u32,+ calls: AtomicU32,+ }++ impl FlakySpawn {+ fn new(fail_first: u32) -> Self {+ Self {+ fail_first,+ calls: AtomicU32::new(0),+ }+ }+ }++ impl Spawn for FlakySpawn {+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<JoinHandle<()>> {+ let call = self.calls.fetch_add(1, Ordering::SeqCst);+ if call < self.fail_first {+ drop(job); // exactly what Builder::spawn does on failure+ return Err(io::Error::from_raw_os_error(libc::EAGAIN));+ }+ StdSpawn.spawn(name, job)+ }+ }++ #[test]+ fn a_failed_spawn_returns_the_job_instead_of_panicking() {+ let ran = Arc::new(AtomicUsize::new(0));+ let job_ran = Arc::clone(&ran);+ let failure = try_spawn_with(&FlakySpawn::new(1), true, "t1122", move || {+ job_ran.fetch_add(1, Ordering::SeqCst);+ })+ .expect_err("the spawner fails");+ assert_eq!(failure.error.raw_os_error(), Some(libc::EAGAIN));+ assert_eq!(ran.load(Ordering::SeqCst), 0, "nothing ran on a thread");+ // The caller can degrade by running the job inline.+ (failure.job.expect("job handed back"))();+ assert_eq!(ran.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn a_closed_gate_refuses_without_calling_the_spawner() {+ let spawner = FlakySpawn::new(0);+ let failure = try_spawn_with(&spawner, false, "t1122", || {}).expect_err("gate closed");+ assert_eq!(failure.error.kind(), io::ErrorKind::OutOfMemory);+ assert!(failure.job.is_some());+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 0);+ }++ #[test]+ fn a_successful_spawn_runs_the_job_once_on_the_named_thread() {+ let (tx, rx) = std::sync::mpsc::channel();+ let handle = try_spawn_with(&StdSpawn, true, "xerj-t1122", move || {+ let name = std::thread::current().name().map(str::to_string);+ tx.send(name).unwrap();+ })+ .expect("spawn");+ handle.join().unwrap();+ assert_eq!(rx.recv().unwrap().as_deref(), Some("xerj-t1122"));+ assert!(rx.try_recv().is_err(), "ran exactly once");+ }++ #[test]+ fn retry_rides_out_transient_failures() {+ let spawner = FlakySpawn::new(2);+ let ran = Arc::new(AtomicUsize::new(0));+ let job_ran = Arc::clone(&ran);+ let handle = spawn_with_retry_using(+ &spawner,+ || true,+ "t1122",+ 3,+ Duration::from_millis(1),+ move || {+ job_ran.fetch_add(1, Ordering::SeqCst);+ },+ )+ .expect("third attempt succeeds");+ handle.join().unwrap();+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 3);+ assert_eq!(ran.load(Ordering::SeqCst), 1);+ }++ #[test]+ fn retry_gives_up_after_the_attempt_budget_and_hands_the_job_back() {+ let spawner = FlakySpawn::new(u32::MAX);+ let failure = spawn_with_retry_using(+ &spawner,+ || true,+ "t1122",+ 3,+ Duration::from_millis(1),+ || {},+ )+ .expect_err("never succeeds");+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 3);+ assert!(failure.job.is_some());+ }++ #[test]+ fn retry_resamples_the_gate_each_attempt() {+ let gate_calls = AtomicU32::new(0);+ let spawner = FlakySpawn::new(0);+ let handle = spawn_with_retry_using(+ &spawner,+ || gate_calls.fetch_add(1, Ordering::SeqCst) >= 1,+ "t1122",+ 2,+ Duration::from_millis(1),+ || {},+ )+ .expect("gate opens on the second attempt");+ handle.join().unwrap();+ assert_eq!(gate_calls.load(Ordering::SeqCst), 2);+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 1);+ }+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 79985d2e7..047b9f13e 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -41,6 +41,64 @@ const MEMORY_DRAIN_MIN_INTERVAL: std::time::Duration = std::time::Duration::fromstatic MEMORY_DRAIN_TX: std::sync::OnceLock<std::sync::mpsc::SyncSender<()>> =std::sync::OnceLock::new();+/// Spawn attempts (and the first back-off) for the governor's sampler and+/// drain threads. They are started once, at boot; losing one disables the+/// parent memory breaker for the life of the process, so a transient+/// `EAGAIN` is worth riding out.+const GOVERNOR_THREAD_SPAWN_ATTEMPTS: u32 = 3;+const GOVERNOR_THREAD_SPAWN_BACKOFF: std::time::Duration = std::time::Duration::from_millis(50);++/// Start one of the governor's background threads (#1122).+///+/// These used to be `let _ = Builder::spawn(..)`: a failed spawn was dropped+/// on the floor, and the node silently ran with no memory breaker. Now the+/// spawn is retried with back-off through `xerj_common::thread_spawn` (which+/// never panics), and a final failure is logged at ERROR naming the thread+/// and what the node loses. Returns whether the thread is running.+///+/// These threads deliberately do NOT consult the governor's spawn gate+/// (`thread_spawn_safe`). The gate is a soft RSS watermark, not an OS+/// allocation failure, and these are the threads that sample RSS, engage the+/// breaker and drain memory. A node that boots above its watermark (a large+/// WAL replay under a low `XERJ_MAX_PROCESS_MEMORY_MB`) would otherwise have+/// every attempt refused and run with no breaker for the life of the process,+/// which is exactly when the breaker is needed. A real spawn failure still+/// comes back as an `Err` and is logged, not aborted.+fn spawn_governor_thread<F>(name: &str, f: F) -> bool+where+ F: FnOnce() + Send + 'static,+{+ spawn_governor_thread_using(&xerj_common::thread_spawn::StdSpawn, || true, name, f)+}++fn spawn_governor_thread_using<S, G, F>(spawner: &S, gate: G, name: &str, f: F) -> bool+where+ S: xerj_common::thread_spawn::Spawn,+ G: Fn() -> bool,+ F: FnOnce() + Send + 'static,+{+ match xerj_common::thread_spawn::spawn_with_retry_using(+ spawner,+ gate,+ name,+ GOVERNOR_THREAD_SPAWN_ATTEMPTS,+ GOVERNOR_THREAD_SPAWN_BACKOFF,+ f,+ ) {+ Ok(_detached) => true,+ Err(failure) => {+ tracing::error!(+ thread = name,+ attempts = GOVERNOR_THREAD_SPAWN_ATTEMPTS,+ error = %failure.error,+ "could not start a resource-governor thread; the parent memory/disk \+ breaker is degraded until restart"+ );+ false+ }+ }+}+/// Allocator page purge hook, installed by the binary that owns the global/// allocator (xerj-server installs a jemalloc epoch+purge closure). Freeing/// heap is not enough: the RSS breaker keys on resident pages, and jemalloc@@ -3952,28 +4010,30 @@ impl Engine {// instead of dying with this thread's runtime.let drain_handle = tokio::runtime::Handle::try_current().ok();let (drain_tx, drain_rx) = std::sync::mpsc::sync_channel::<()>(1);- let _ = MEMORY_DRAIN_TX.set(drain_tx);let weak_c = Arc::downgrade(self);- let _ = std::thread::Builder::new()- .name("xerj-mem-drain".to_string())- .spawn(move || {- let Some(handle) = drain_handle else {- // Spawned outside a runtime (unit tests): the drain- // method stays directly callable; only the sampler's- // automatic wiring is inert.- return;+ let drain_worker = spawn_governor_thread("xerj-mem-drain", move || {+ let Some(handle) = drain_handle else {+ // Spawned outside a runtime (unit tests): the drain+ // method stays directly callable; only the sampler's+ // automatic wiring is inert.+ return;+ };+ while let Ok(()) = drain_rx.recv() {+ let Some(engine) = weak_c.upgrade() else {+ return; // engine dropped};- while let Ok(()) = drain_rx.recv() {- let Some(engine) = weak_c.upgrade() else {- return; // engine dropped- };- let drained = handle.block_on(engine.drain_rebuildable_memory());- tracing::info!(+ let drained = handle.block_on(engine.drain_rebuildable_memory());+ tracing::info!(indexes = drained,"memory breaker engaged — drained rebuildable state (flush + cache release + purge)");- }- });+ }+ });+ // Publish the request channel only once a worker is draining it: with+ // no worker, the sampler's requests have nowhere to go.+ if drain_worker {+ let _ = MEMORY_DRAIN_TX.set(drain_tx);+ }// ── Thread A: memory + disk ──────────────────────────────────────// Touches NO engine lock — just `/proc/self/statm` + `statvfs`. Kept@@ -3991,123 +4051,118 @@ impl Engine {// Snapshot the uncapped machine memory once: a percentage override is// resolved against it, and it is the hard clamp for an absolute one.let machine_memory_bytes = crate::governor::effective_memory_limit_bytes();- let _ = std::thread::Builder::new()- .name("xerj-mem-sampler".to_string())- .spawn(move || {- tracing::info!(- period_ms = crate::governor::SAMPLE_INTERVAL_MS,- "memory/disk sampler thread started (parent circuit breaker)"- );- // #948 drain bookkeeping: None while the breaker is- // disengaged (so the next engage edge fires immediately),- // Some(last-drain) while engaged (rate-limited repeats).- let mut last_drain_at: Option<std::time::Instant> = None;- loop {- std::thread::sleep(period);- if weak_a.strong_count() == 0 {- return; // engine dropped- }- let governor = match crate::governor::global() {- Some(g) => g,- None => return,+ spawn_governor_thread("xerj-mem-sampler", move || {+ tracing::info!(+ period_ms = crate::governor::SAMPLE_INTERVAL_MS,+ "memory/disk sampler thread started (parent circuit breaker)"+ );+ // #948 drain bookkeeping: None while the breaker is+ // disengaged (so the next engage edge fires immediately),+ // Some(last-drain) while engaged (rate-limited repeats).+ let mut last_drain_at: Option<std::time::Instant> = None;+ loop {+ std::thread::sleep(period);+ if weak_a.strong_count() == 0 {+ return; // engine dropped+ }+ let governor = match crate::governor::global() {+ Some(g) => g,+ None => return,+ };+ let rss = crate::governor::current_rss_bytes();+ let (disk_pct, disk_avail) = crate::governor::disk_used_pct_and_avail(&data_dir);+ governor.refresh_memory_disk(rss, disk_pct, disk_avail);++ // Issue #948 — a breaker that only rejects never lets a+ // memory-pinned workload finish: caches hold gigabytes+ // of re-hydratable state and the memtable holds raw+ // write payloads, and 429s free none of it. Request a+ // drain on the engage edge and at most once per window+ // while engaged. try_send never blocks, so Thread A's+ // no-lock/no-block contract above still holds.+ if governor.memory_breaker_engaged() {+ let fire = match last_drain_at {+ None => true,+ Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,};- let rss = crate::governor::current_rss_bytes();- let (disk_pct, disk_avail) =- crate::governor::disk_used_pct_and_avail(&data_dir);- governor.refresh_memory_disk(rss, disk_pct, disk_avail);-- // Issue #948 — a breaker that only rejects never lets a- // memory-pinned workload finish: caches hold gigabytes- // of re-hydratable state and the memtable holds raw- // write payloads, and 429s free none of it. Request a- // drain on the engage edge and at most once per window- // while engaged. try_send never blocks, so Thread A's- // no-lock/no-block contract above still holds.- if governor.memory_breaker_engaged() {- let fire = match last_drain_at {- None => true,- Some(at) => at.elapsed() >= MEMORY_DRAIN_MIN_INTERVAL,- };- if fire {- last_drain_at = Some(std::time::Instant::now());- if let Some(tx) = MEMORY_DRAIN_TX.get() {- let _ = tx.try_send(());- }+ if fire {+ last_drain_at = Some(std::time::Instant::now());+ if let Some(tx) = MEMORY_DRAIN_TX.get() {+ let _ = tx.try_send(());}- } else {- last_drain_at = None;}+ } else {+ last_drain_at = None;+ }- // Runtime disk flood-stage override, matching ES's- // `PUT _cluster/settings` unblock without a restart. A- // non-blocking read: on contention we just keep last- // tick's threshold and retry next sample, never stall- // the memory/disk breaker on this lock.- if let Ok(settings) = cluster_settings.try_read() {- let flood_override = settings- .get("persistent")- .and_then(|p| p.get(FLOOD_STAGE_SETTING))- .or_else(|| {- settings- .get("transient")- .and_then(|t| t.get(FLOOD_STAGE_SETTING))- })- .and_then(crate::governor::parse_flood_stage_pct);- governor.set_disk_flood_pct_override(flood_override);-- // `threshold_enabled: false` turns the flood stage- // off and releases an engaged block, as in ES.- let threshold_enabled = settings- .get("persistent")- .and_then(|p| p.get(DISK_THRESHOLD_SETTING))- .or_else(|| {- settings- .get("transient")- .and_then(|t| t.get(DISK_THRESHOLD_SETTING))- })- .and_then(crate::governor::parse_setting_bool);- governor.set_disk_threshold_enabled(threshold_enabled);-- // Runtime parent-memory-breaker watermark override- // (#1250): raise the watermark on a big box live via- // `PUT _cluster/settings {indices.breaker.total.limit}`,- // no restart — the live equivalent of the boot-only- // XERJ_MAX_PROCESS_MEMORY_MB cap.- let breaker_override = settings- .get("persistent")- .and_then(|p| p.get(BREAKER_LIMIT_SETTING))- .or_else(|| {- settings- .get("transient")- .and_then(|t| t.get(BREAKER_LIMIT_SETTING))- })- .and_then(|v| {- crate::governor::parse_breaker_total_limit(v, machine_memory_bytes)- });- governor.set_memory_watermark_override(breaker_override);- }+ // Runtime disk flood-stage override, matching ES's+ // `PUT _cluster/settings` unblock without a restart. A+ // non-blocking read: on contention we just keep last+ // tick's threshold and retry next sample, never stall+ // the memory/disk breaker on this lock.+ if let Ok(settings) = cluster_settings.try_read() {+ let flood_override = settings+ .get("persistent")+ .and_then(|p| p.get(FLOOD_STAGE_SETTING))+ .or_else(|| {+ settings+ .get("transient")+ .and_then(|t| t.get(FLOOD_STAGE_SETTING))+ })+ .and_then(crate::governor::parse_flood_stage_pct);+ governor.set_disk_flood_pct_override(flood_override);++ // `threshold_enabled: false` turns the flood stage+ // off and releases an engaged block, as in ES.+ let threshold_enabled = settings+ .get("persistent")+ .and_then(|p| p.get(DISK_THRESHOLD_SETTING))+ .or_else(|| {+ settings+ .get("transient")+ .and_then(|t| t.get(DISK_THRESHOLD_SETTING))+ })+ .and_then(crate::governor::parse_setting_bool);+ governor.set_disk_threshold_enabled(threshold_enabled);++ // Runtime parent-memory-breaker watermark override+ // (#1250): raise the watermark on a big box live via+ // `PUT _cluster/settings {indices.breaker.total.limit}`,+ // no restart — the live equivalent of the boot-only+ // XERJ_MAX_PROCESS_MEMORY_MB cap.+ let breaker_override = settings+ .get("persistent")+ .and_then(|p| p.get(BREAKER_LIMIT_SETTING))+ .or_else(|| {+ settings+ .get("transient")+ .and_then(|t| t.get(BREAKER_LIMIT_SETTING))+ })+ .and_then(|v| {+ crate::governor::parse_breaker_total_limit(v, machine_memory_bytes)+ });+ governor.set_memory_watermark_override(breaker_override);}- });+ }+ });// ── Thread B: summed memtable budget ─────────────────────────────// Each per-index `size_bytes()` is one relaxed atomic load of the// aggregate the shards maintain incrementally — no shard locks, so// this never blocks under a turbo batch (#872).let weak_b = Arc::downgrade(self);- let _ = std::thread::Builder::new()- .name("xerj-memtable-sampler".to_string())- .spawn(move || loop {- std::thread::sleep(period);- let engine = match weak_b.upgrade() {- Some(e) => e,- None => return, // engine dropped- };- let governor = match crate::governor::global() {- Some(g) => g,- None => return,- };- governor.refresh_memtable(engine.total_memtable_bytes());- });+ spawn_governor_thread("xerj-memtable-sampler", move || loop {+ std::thread::sleep(period);+ let engine = match weak_b.upgrade() {+ Some(e) => e,+ None => return, // engine dropped+ };+ let governor = match crate::governor::global() {+ Some(g) => g,+ None => return,+ };+ governor.refresh_memtable(engine.total_memtable_bytes());+ });}// ── Transform pipeline methods ────────────────────────────────────────────@@ -5106,3 +5161,53 @@ mod api_key_record_migration_tests {assert!(!engine.has_open_index("alpha"));}}++#[cfg(test)]+mod governor_thread_spawn_tests {+ use super::spawn_governor_thread_using;+ use xerj_common::thread_spawn::{BoxedJob, Spawn};++ /// A spawner that always fails the way `pthread_create` does at a+ /// thread or memory limit.+ struct AlwaysEagain;++ impl Spawn for AlwaysEagain {+ fn spawn(+ &self,+ _name: &str,+ _job: BoxedJob,+ ) -> std::io::Result<std::thread::JoinHandle<()>> {+ Err(std::io::Error::from_raw_os_error(libc::EAGAIN))+ }+ }++ #[test]+ fn a_governor_thread_that_cannot_spawn_reports_false_instead_of_aborting() {+ let ran = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));+ let job_ran = std::sync::Arc::clone(&ran);+ let started = spawn_governor_thread_using(+ &AlwaysEagain,+ || true,+ "xerj-test",+ move || {+ job_ran.store(true, std::sync::atomic::Ordering::SeqCst);+ },+ );+ assert!(!started);+ assert!(!ran.load(std::sync::atomic::Ordering::SeqCst));+ }++ #[test]+ fn a_closed_spawn_gate_is_respected() {+ // A working spawner, but the gate is closed. Production passes an+ // always-open gate for these threads (see `spawn_governor_thread`);+ // this pins the plumbing for a caller that does gate.+ let started = spawn_governor_thread_using(+ &xerj_common::thread_spawn::StdSpawn,+ || false,+ "xerj-test",+ || {},+ );+ assert!(!started);+ }+}diff --git a/engine/crates/xerj-engine/src/governor.rs b/engine/crates/xerj-engine/src/governor.rsindex 3625b6f7b..d0b485905 100644--- a/engine/crates/xerj-engine/src/governor.rs+++ b/engine/crates/xerj-engine/src/governor.rs@@ -814,7 +814,12 @@ fn resolve_segment_hydration_budget(/// call wins (subsequent calls return the already-installed governor, so a/// second `Engine` in-process — e.g. a test — does not re-key the budgets).pub fn init(config: &Config) -> Arc<ResourceGovernor> {- Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))))+ let governor = Arc::clone(GOVERNOR.get_or_init(|| Arc::new(build(config))));+ // #1122: every crate that spawns through `xerj_common::thread_spawn`+ // (storage, server, …) consults this same watermark check, so a spawn+ // under memory exhaustion is refused instead of aborting the node.+ xerj_common::thread_spawn::install_spawn_gate(thread_spawn_safe);+ governor}/// The installed governor, if [`init`] has run. Engine-only unit tests thatdiff --git a/engine/crates/xerj-engine/src/memtable.rs b/engine/crates/xerj-engine/src/memtable.rsindex 16bd1f5f0..268a2d253 100644--- a/engine/crates/xerj-engine/src/memtable.rs+++ b/engine/crates/xerj-engine/src/memtable.rs@@ -2809,12 +2809,13 @@ impl FrozenClaim {// the shipped profile is panic=abort, so under memory pressure the// spawn is not attempted at all — the Arcs drop inline right here,// on the same fallback path the comment always intended.- if crate::governor::thread_spawn_safe() {- let _ = std::thread::Builder::new()- .name("xerj-drain-free".to_string())- .spawn(move || drop(dead));- } else {- drop(dead);+ // `try_spawn` consults the governor's spawn gate (installed by+ // `governor::init`) and hands the job back on a refusal or a failed+ // spawn; dropping the returned job frees the Arcs inline.+ if let Err(failure) =+ xerj_common::thread_spawn::try_spawn("xerj-drain-free", move || drop(dead))+ {+ drop(failure.job);}guard}