AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): extractor reader threads fail the file, not the run, when a spawn fails (#1122)
xerj-org9c0fadd096d8
diff --git a/engine/crates/xerj-autoindex/src/extract/mbox.rs b/engine/crates/xerj-autoindex/src/extract/mbox.rsindex 1c65a3c33..905fadfb3 100644--- a/engine/crates/xerj-autoindex/src/extract/mbox.rs+++ b/engine/crates/xerj-autoindex/src/extract/mbox.rs@@ -748,7 +748,7 @@ fn extract_parallel<R: BufRead + Send>(std::thread::scope(|scope| {let splitter_tx = done_tx.clone();let (stop_ref, budget_ref) = (&stop, &budget);- scope.spawn(move || {+ let splitter = move || {let mut split = Splitter::new(reader);let mut seq = 0u64;loop {@@ -774,33 +774,74 @@ fn extract_parallel<R: BufRead + Send>(}let _ = splitter_tx.send(Done::End { total: seq });// `job_tx` drops here: the parsers drain what is queued and exit.- });+ };+ // #1122: `Scope::spawn` panics when a thread cannot be created, and+ // release builds are panic=abort. Spawn through the fallible builder:+ // fewer parsers than asked for only costs speed; none at all — or no+ // splitter — fails this mailbox through the ordinary error path below+ // instead of aborting the run.+ let mut parsers = 0usize;+ let mut spawn_error: Option<std::io::Error> = None;for _ in 0..workers {let job_rx = Arc::clone(&job_rx);let done_tx = done_tx.clone();- scope.spawn(move || loop {- // Hold the lock only while waiting for a job, never while- // parsing one: the other parsers must be able to take theirs.- let job = match job_rx.lock() {- Ok(rx) => rx.recv(),- Err(_) => return,- };- let Ok(Job { seq, msg }) = job else { return };- let _slot = gate().acquire();- let bytes = msg.bytes.len();- let (records, stats) = extract_one(&msg);- drop(msg);- if done_tx- .send(Done::Message {- seq,- bytes,- records,- stats,- })- .is_err()- {- return;+ let spawned = std::thread::Builder::new()+ .name("xerj-mbox-parse".into())+ .spawn_scoped(scope, move || loop {+ // Hold the lock only while waiting for a job, never while+ // parsing one: the other parsers must be able to take theirs.+ let job = match job_rx.lock() {+ Ok(rx) => rx.recv(),+ Err(_) => return,+ };+ let Ok(Job { seq, msg }) = job else { return };+ let _slot = gate().acquire();+ let bytes = msg.bytes.len();+ let (records, stats) = extract_one(&msg);+ drop(msg);+ if done_tx+ .send(Done::Message {+ seq,+ bytes,+ records,+ stats,+ })+ .is_err()+ {+ return;+ }+ });+ match spawned {+ Ok(_) => parsers += 1,+ Err(error) => {+ spawn_error = Some(error);+ break;}+ }+ }+ let splitter_spawn = if parsers == 0 {+ // Nobody would drain the job queue; dropping the splitter (and+ // with it `job_tx` and the reader) is the clean way out.+ drop(splitter);+ Err(spawn_error.unwrap_or_else(|| {+ std::io::Error::other("no mailbox parser threads were requested")+ }))+ } else {+ std::thread::Builder::new()+ .name("xerj-mbox-split".into())+ .spawn_scoped(scope, splitter)+ .map(drop)+ };+ if let Err(error) = splitter_spawn {+ // The failed closure dropped `job_tx`, so any parsers that did+ // start see a closed queue and exit; the consumer below reads+ // this as an error at seq 0 and stops.+ let _ = done_tx.send(Done::Error {+ seq: 0,+ error: std::io::Error::new(+ error.kind(),+ format!("could not start the mailbox reader threads: {error}"),+ ),});}drop(done_tx);diff --git a/engine/crates/xerj-autoindex/src/extract/pdf.rs b/engine/crates/xerj-autoindex/src/extract/pdf.rsindex 3a41fd677..75b88ec56 100644--- a/engine/crates/xerj-autoindex/src/extract/pdf.rs+++ b/engine/crates/xerj-autoindex/src/extract/pdf.rs@@ -956,8 +956,42 @@ fn spawn_worker(path: &Path) -> Result<WorkerResponse> {})?;let stdout = child.stdout.take().expect("piped stdout");let stderr = child.stderr.take().expect("piped stderr");- let out_reader = std::thread::spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT));- let err_reader = std::thread::spawn(move || read_capped(stderr, MAX_WORKER_STDERR as usize));+ // #1122: `std::thread::spawn` panics when a thread cannot be created+ // (thread limit, memory exhaustion) and release builds are panic=abort,+ // so one failed spawn here used to abort the whole run. A failed spawn+ // now stops the worker and fails this one file, like any other worker+ // start-up failure.+ let out_reader = match std::thread::Builder::new()+ .name("xerj-pdf-stdout".into())+ .spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT))+ {+ Ok(reader) => reader,+ Err(error) => {+ terminate_worker_tree(&mut child);+ return Err(error).with_context(|| {+ format!(+ "could not start the PDF parser's output reader for {}",+ path.display()+ )+ });+ }+ };+ let err_reader = match std::thread::Builder::new()+ .name("xerj-pdf-stderr".into())+ .spawn(move || read_capped(stderr, MAX_WORKER_STDERR as usize))+ {+ Ok(reader) => reader,+ Err(error) => {+ terminate_worker_tree(&mut child);+ let _ = out_reader.join();+ return Err(error).with_context(|| {+ format!(+ "could not start the PDF parser's error reader for {}",+ path.display()+ )+ });+ }+ };let started = Instant::now();let timeout = Duration::from_secs(WORKER_TIMEOUT_SECS.load(Ordering::Relaxed));diff --git a/engine/crates/xerj-autoindex/src/extract/xls.rs b/engine/crates/xerj-autoindex/src/extract/xls.rsindex 4aacf6368..aa787c51e 100644--- a/engine/crates/xerj-autoindex/src/extract/xls.rs+++ b/engine/crates/xerj-autoindex/src/extract/xls.rs@@ -358,7 +358,26 @@ fn run_worker(let stdout = child.stdout.take().expect("piped stdout");// Read on a thread so the wall-clock watchdog below can act even if the// worker stalls with the pipe open.- let reader = std::thread::spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT));+ //+ // #1122: `std::thread::spawn` panics when a thread cannot be created and+ // release builds are panic=abort; a failed spawn stops the worker and+ // fails this one file instead of aborting the run.+ let reader = match std::thread::Builder::new()+ .name("xerj-xls-stdout".into())+ .spawn(move || read_capped(stdout, MAX_WORKER_OUTPUT))+ {+ Ok(reader) => reader,+ Err(error) => {+ kill_group(pid);+ let _ = child.wait();+ return Err(error).with_context(|| {+ format!(+ "could not start the .xls reader's output thread for {}",+ path.display()+ )+ });+ }+ };let started = std::time::Instant::now();let status = loop {