AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): the phase-A scan pool degrades to the calling thread when it cannot spawn (#1122)
xerj-org703cae535eaf
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex 7c0efffa4..879e2fa92 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -307,7 +307,7 @@ mod tests {);assert_eq!(OBSERVED_DIGEST_POOL_THREADS.load(std::sync::atomic::Ordering::Relaxed),- crate::pool::scan_pool().current_num_threads(),+ crate::pool::width(),"the digest phase must see exactly the scan pool the run configured");assert_eq!(diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex babded8b1..6c43a1048 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -3212,7 +3212,7 @@ fn the_resource_plan_is_announced_on_the_progress_surface_with_the_width_it_got(// Build the process-global scan pool first, so the width below is one this// run cannot possibly be granted.- let installed = crate::pool::scan_pool().current_num_threads();+ let installed = crate::pool::width();let endpoint = MockEndpoint::start(usize::MAX);let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);config.quiet = false;diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 3ba46c2d4..bbb167eb3 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -5806,7 +5806,7 @@ fn run_index_report_inner(// — `xerj-server`'s brain endpoint does exactly that — the installed width// can differ from what this run's plan asked for. Progress must state the// number the policy got, not the one it requested (#240 + #241).- let scan_threads = pool::scan_pool().current_num_threads();+ let scan_threads = pool::width();extract::pdf::configure_workers(cfg.pdf_workers);extract::pdf::configure_timeout(cfg.pdf_timeout_secs);// A mailbox is one file on one Phase-B worker; inside it, messages arediff --git a/engine/crates/xerj-autoindex/src/pool.rs b/engine/crates/xerj-autoindex/src/pool.rsindex 51a236c3e..12d03638c 100644--- a/engine/crates/xerj-autoindex/src/pool.rs+++ b/engine/crates/xerj-autoindex/src/pool.rs@@ -10,10 +10,27 @@//!//! This module owns a pool of the width the run's [`crate::resources::Plan`]//! asked for, and phase A runs inside it.+//!+//! A pool that cannot be started is not fatal (#1122). `build()` fails when+//! `pthread_create` returns `EAGAIN` (thread or memory limit), and release+//! builds are `panic = "abort"`, so the old `.expect(..)` took the whole run+//! down. Now phase A runs on the calling thread instead, slower but complete,+//! and the next phase tries to start the pool again.++use std::cell::RefCell;+use std::sync::atomic::{AtomicUsize, Ordering};+use std::sync::{Mutex, OnceLock};-use std::sync::OnceLock;+use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder};-static SCAN_POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();+static SCAN_POOL: OnceLock<ThreadPool> = OnceLock::new();+/// Serialises pool construction, so two first callers do not both start a+/// pool's worth of threads.+static BUILD_LOCK: Mutex<()> = Mutex::new(());+/// The width [`configure`] asked for (0 = never configured). Kept so a pool+/// that failed to start is retried at the configured width, not at the+/// machine's.+static REQUESTED: AtomicUsize = AtomicUsize::new(0);/// Fix the phase-A pool width for this process. Called once per run, before/// phase A starts.@@ -24,35 +41,124 @@ static SCAN_POOL: OnceLock<rayon::ThreadPool> = OnceLock::new();/// changes.pub fn configure(threads: usize) {let threads = threads.max(1);- let installed = scan_pool_with(threads).current_num_threads();- if installed != threads {- eprintln!(- "autoindex: phase-A pool is already running with {installed} threads; \- ignoring the request for {threads}"- );+ let _ = REQUESTED.compare_exchange(0, threads, Ordering::Relaxed, Ordering::Relaxed);+ // `None`: the pool could not be started and that was already reported.+ if let Some(pool) = scan_pool_with(threads) {+ let installed = pool.current_num_threads();+ if installed != threads {+ eprintln!(+ "autoindex: phase-A pool is already running with {installed} threads; \+ ignoring the request for {threads}"+ );+ }}}-/// The phase-A pool, built on first use at the machine's full width if-/// [`configure`] was never called (library use, unit tests).-pub fn scan_pool() -> &'static rayon::ThreadPool {- scan_pool_with(xerj_common::resource::cores())+/// The phase-A pool, built on first use at the configured width, or the+/// machine's full width if [`configure`] was never called (library use, unit+/// tests). `None` when the pool could not be started; [`install`] then runs+/// the work on the calling thread.+pub fn scan_pool() -> Option<&'static ThreadPool> {+ let threads = match REQUESTED.load(Ordering::Relaxed) {+ 0 => xerj_common::resource::cores(),+ n => n,+ };+ scan_pool_with(threads)}-fn scan_pool_with(threads: usize) -> &'static rayon::ThreadPool {- SCAN_POOL.get_or_init(|| {- rayon::ThreadPoolBuilder::new()+/// How many threads phase A is running on right now: the scan pool's width,+/// or 1 when it could not be started.+pub fn width() -> usize {+ scan_pool().map_or(1, ThreadPool::current_num_threads)+}++fn scan_pool_with(threads: usize) -> Option<&'static ThreadPool> {+ pool_in(&SCAN_POOL, || {+ ThreadPoolBuilder::new().num_threads(threads.max(1)).thread_name(|i| format!("xerj-scan-{i}")).build()- .expect("failed to build the autoindex scan pool")})}+/// The pool in `slot`, built with `build` if it is not there yet. A failed+/// build leaves `slot` empty, so the next call tries again.+fn pool_in(+ slot: &OnceLock<ThreadPool>,+ build: impl FnOnce() -> Result<ThreadPool, ThreadPoolBuildError>,+) -> Option<&ThreadPool> {+ if let Some(pool) = slot.get() {+ return Some(pool);+ }+ let _guard = BUILD_LOCK.lock().unwrap_or_else(|e| e.into_inner());+ if let Some(pool) = slot.get() {+ return Some(pool);+ }+ match build() {+ Ok(pool) => Some(slot.get_or_init(|| pool)),+ Err(e) => {+ eprintln!(+ "autoindex: could not start the phase-A scan pool ({e}); \+ running this phase on the calling thread (#1122)"+ );+ None+ }+ }+}+/// Run `f` inside the phase-A pool. Any rayon work started by `f` — including-/// nested `par_iter`s — is confined to that pool's threads.+/// nested `par_iter`s — is confined to that pool's threads. If the pool could+/// not be started, `f` runs on the calling thread instead (see+/// [`sequential`]).pub fn install<R: Send>(f: impl FnOnce() -> R + Send) -> R {- scan_pool().install(f)+ install_in(scan_pool(), f)+}++fn install_in<R: Send>(pool: Option<&ThreadPool>, f: impl FnOnce() -> R + Send) -> R {+ match pool {+ Some(pool) => pool.install(f),+ None => sequential(f),+ }+}++thread_local! {+ /// A one-thread pool made of the calling thread itself. Kept for the+ /// thread's lifetime: the thread stays registered as its worker either way.+ static INLINE_POOL: RefCell<Option<ThreadPool>> = const { RefCell::new(None) };+}++/// Run `f`, and every `par_iter` inside it, on the calling thread, without+/// starting a thread.+///+/// Calling `f()` directly would not do: a `par_iter` on a thread that is not+/// a rayon worker goes to rayon's global pool, which is built lazily and+/// panics if it cannot start its threads, the same failure we are avoiding.+/// A `use_current_thread()` pool of width 1 spawns nothing and makes this+/// thread its only worker, so the parallel iterators run inline. rayon falls+/// back to the same construction when threads are unsupported+/// (`rayon-core` `registry.rs`, `default_global_registry`).+fn sequential<R: Send>(f: impl FnOnce() -> R + Send) -> R {+ // Already a rayon worker (of a pool built here earlier, or of any other+ // pool): `par_iter` stays on that worker's own pool.+ if rayon::current_thread_index().is_some() {+ return f();+ }+ let built = ThreadPoolBuilder::new()+ .num_threads(1)+ .use_current_thread()+ .build();+ match built {+ Ok(pool) => {+ let out = pool.install(f);+ INLINE_POOL.with(|slot| *slot.borrow_mut() = Some(pool));+ out+ }+ Err(e) => {+ // Not expected: the builder starts no thread. Best effort.+ eprintln!("autoindex: could not register the calling thread with rayon ({e})");+ f()+ }+ }}#[cfg(test)]@@ -72,7 +178,7 @@ mod tests {.map(|_| rayon::current_num_threads()).collect()});- let pool_width = scan_pool().current_num_threads();+ let pool_width = width();assert!(widths.iter().all(|&w| w == pool_width));let name_is_ours = install(|| {(0..64).into_par_iter().all(|_| {@@ -83,4 +189,86 @@ mod tests {});assert!(name_is_ours, "phase A must not run on the global pool");}++ /// A pool whose every thread spawn fails, as `pthread_create` does with+ /// `EAGAIN` under memory exhaustion. Injected through rayon's own spawn+ /// hook, so `build()` takes its real failure path.+ fn build_with_failing_spawn() -> Result<ThreadPool, ThreadPoolBuildError> {+ ThreadPoolBuilder::new()+ .num_threads(4)+ .spawn_handler(|_| {+ Err(std::io::Error::new(+ std::io::ErrorKind::WouldBlock,+ "injected EAGAIN",+ ))+ })+ .build()+ }++ /// #1122: a scan pool that cannot start used to `.expect(..)`, which is an+ /// abort under `panic = "abort"`. Now the build reports `None`, the slot+ /// stays empty so a later call can retry, and the phase still produces+ /// its full, ordered result, computed on the calling thread alone.+ #[test]+ fn a_pool_that_cannot_spawn_degrades_to_the_calling_thread() {+ // A fresh thread: the fallback registers its caller as a rayon worker+ // for the rest of that thread's life.+ std::thread::spawn(|| {+ let slot = OnceLock::new();+ assert!(pool_in(&slot, build_with_failing_spawn).is_none());+ assert!(slot.get().is_none(), "a failed build must not be cached");++ let caller = std::thread::current().id();+ let out: Vec<(u64, bool, usize)> =+ install_in(pool_in(&slot, build_with_failing_spawn), || {+ (0..1000u64)+ .into_par_iter()+ .map(|i| {+ (+ i * i,+ std::thread::current().id() == caller,+ rayon::current_num_threads(),+ )+ })+ .collect()+ });+ assert_eq!(out.len(), 1000);+ assert!(out+ .iter()+ .enumerate()+ .all(|(i, &(sq, _, _))| sq == (i * i) as u64));+ // Every item ran here: neither a scan-pool thread nor rayon's+ // global pool (which could not have started either) was used.+ assert!(out.iter().all(|&(_, here, _)| here));+ assert!(out.iter().all(|&(_, _, width)| width == 1));++ // A second phase on the same thread reuses the inline pool.+ let again: u64 = install_in(None, || (0..100u64).into_par_iter().sum());+ assert_eq!(again, 4950);++ // Once spawning works again, the next call builds the real pool.+ let pool = pool_in(&slot, || ThreadPoolBuilder::new().num_threads(2).build());+ assert_eq!(pool.map(ThreadPool::current_num_threads), Some(2));+ assert!(slot.get().is_some());+ })+ .join()+ .unwrap();+ }++ /// The fallback inside an existing rayon worker does not register a new+ /// pool (that would fail: a thread can belong to only one); it runs on the+ /// worker's own pool.+ #[test]+ fn the_fallback_inside_a_rayon_worker_stays_on_that_pool() {+ let outer = ThreadPoolBuilder::new().num_threads(2).build().unwrap();+ let widths: Vec<usize> = outer.install(|| {+ install_in(None, || {+ (0..64)+ .into_par_iter()+ .map(|_| rayon::current_num_threads())+ .collect()+ })+ });+ assert!(widths.iter().all(|&w| w == 2));+ }}