Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api,autoindex): degrade when a scoped thread or update body cannot be allocated (#1122) (#2322)

xerj-org5da900186fbd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 82fbb460b..ff8ba3f3d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -85,6 +85,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Scoped autoindex threads and `POST /{index}/_update/{id}` no longer abort
+ the process when a thread stack or a large body cannot be allocated
+ (#1122). Indexing workers, sync replay, object-watch fetches, and mailbox
+ parsers start with `pthread_create` and do not map an alternate signal
+ stack. A spawn the OS refuses is an error: the job does not run, workers
+ that already started keep going, and the caller runs the work on the
+ calling thread or records the failure. An update body over two 8 MiB
+ documents plus 256 KiB of envelope is refused before `serde_json`, and
+ the existing document is left as it was.
+
- An idle node whose resident set sits flat at or above the memory watermark
no longer drains every 2 seconds for the rest of the process. The breaker
releases once RSS has not fallen by more than 4 MiB for 2 seconds, and does
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index cc56d5e8f..cde14221a 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -4501,6 +4501,126 @@ mod oversized_single_doc {
}
}
+#[cfg(test)]
+mod oversized_update {
+ use super::*;
+ use axum::body::{to_bytes, Body};
+ use axum::http::Request;
+ use tower::ServiceExt;
+ use xerj_common::config::{Config, WalSync};
+ use xerj_common::document_limit::MAX_UPDATE_BODY_BYTES;
+ use xerj_common::metrics::Metrics;
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ #[test]
+ fn an_oversized_update_body_is_refused_before_it_is_parsed() {
+ let body = vec![b'x'; MAX_UPDATE_BODY_BYTES + 1];
+ let err = parse_update_body(&body).unwrap_err();
+ let text = err.to_string();
+ assert!(text.contains(&MAX_UPDATE_BODY_BYTES.to_string()), "{text}");
+ assert!(text.contains("not parsed"), "{text}");
+ assert!(text.contains("not changed"), "{text}");
+ assert!(
+ !text.contains("invalid JSON"),
+ "the size check must run before parse: {text}"
+ );
+ let small = parse_update_body(br#"{"doc":{"name":"ada"}}"#).unwrap();
+ assert_eq!(small.doc.unwrap()["name"], "ada");
+ }
+
+ #[tokio::test]
+ async fn post_update_refuses_an_oversized_body_and_a_small_update_applies() {
+ let app = crate::router::build_es_compat_router(test_state());
+ let created = app
+ .clone()
+ .oneshot(
+ Request::put("/upd/_doc/1")
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"name":"ada","n":1}"#))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(created.status(), StatusCode::CREATED);
+
+ // Valid JSON, over the envelope. A parse-first path would accept
+ // the syntax and answer with the document ceiling (or index it).
+ // The size refusal names the update limit and does not say the
+ // body was invalid JSON.
+ let mut big = Vec::new();
+ big.extend_from_slice(br#"{"doc":{"blob":""#);
+ big.extend(std::iter::repeat_n(b'a', MAX_UPDATE_BODY_BYTES));
+ big.extend_from_slice(br#""}}"#);
+ let refused = app
+ .clone()
+ .oneshot(
+ Request::post("/upd/_update/1")
+ .header("content-type", "application/json")
+ .body(Body::from(big))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(refused.status(), StatusCode::BAD_REQUEST);
+ let bytes = to_bytes(refused.into_body(), usize::MAX).await.unwrap();
+ let text = String::from_utf8(bytes.to_vec()).unwrap();
+ assert!(text.contains(&MAX_UPDATE_BODY_BYTES.to_string()), "{text}");
+ assert!(text.contains("not parsed"), "{text}");
+ assert!(text.contains("not changed"), "{text}");
+ assert!(
+ !text.contains("invalid JSON"),
+ "parsed before the size check: {text}"
+ );
+
+ let fetched = app
+ .clone()
+ .oneshot(Request::get("/upd/_doc/1").body(Body::empty()).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(fetched.status(), StatusCode::OK);
+ let bytes = to_bytes(fetched.into_body(), usize::MAX).await.unwrap();
+ let doc: Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(doc["_source"]["name"], json!("ada"));
+ assert_eq!(doc["_source"]["n"], json!(1));
+
+ let updated = app
+ .clone()
+ .oneshot(
+ Request::post("/upd/_update/1")
+ .header("content-type", "application/json")
+ .body(Body::from(r#"{"doc":{"name":"ada2"}}"#))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(updated.status(), StatusCode::OK);
+
+ let fetched = app
+ .oneshot(Request::get("/upd/_doc/1").body(Body::empty()).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(fetched.status(), StatusCode::OK);
+ let bytes = to_bytes(fetched.into_body(), usize::MAX).await.unwrap();
+ let doc: Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(doc["_source"]["name"], json!("ada2"));
+ assert_eq!(
+ doc["_source"]["n"],
+ json!(1),
+ "fields the update did not name stay"
+ );
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// GET /{index}/_doc/{id}?_source_includes=f1,f2&_source_excludes=f3
// ─────────────────────────────────────────────────────────────────────────────
@@ -22984,12 +23104,43 @@ async fn build_update_get_field(
}
}
+/// Parse an `_update` body from raw bytes.
+///
+/// The size check runs before `serde_json`. The route used to take
+/// `Json<EsUpdateBody>`, so axum built the DOM before the handler. A body
+/// between this envelope and `limits.max_body_bytes` (100 MiB) was parsed
+/// even when it could not be applied, and that parse is an infallible
+/// allocation (`panic = "abort"`, #1122). Bytes that are not JSON and are
+/// also over the limit report the size refusal, not a parse error.
+pub(crate) fn parse_update_body(body: &[u8]) -> Result<EsUpdateBody, xerj_common::XerjError> {
+ if xerj_common::document_limit::update_body_too_large(body.len()) {
+ let len = body.len();
+ tracing::warn!(
+ bytes = len,
+ limit = xerj_common::document_limit::MAX_UPDATE_BODY_BYTES,
+ "update body refused before parse; the existing document was not changed (#1122)"
+ );
+ return Err(xerj_common::XerjError::invalid_document_json(
+ xerj_common::document_limit::update_body_refusal(len),
+ ));
+ }
+ serde_json::from_slice(body).map_err(|e| {
+ xerj_common::XerjError::invalid_document_json(format!("invalid JSON body: {e}"))
+ })
+}
+
pub async fn update_doc(
State(state): State<AppState>,
Path((index, id)): Path<(String, String)>,
Query(query_params): Query<UpdateDocParams>,
- Json(body): Json<EsUpdateBody>,
+ body: bytes::Bytes,
) -> impl IntoResponse {
+ // Before `get_or_create_index`, so a refused body does not create an
+ // index and does not read or write the document.
+ let body = match parse_update_body(&body) {
+ Ok(body) => body,
+ Err(e) => return ApiError::new(e).into_response(),
+ };
let idx = match state.engine.get_or_create_index(&index) {
Ok(i) => i,
Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),
@@ -32383,13 +32534,10 @@ mod scripted_update_publication_tests {
State(state),
Path(("script-counter".to_string(), "counter".to_string())),
Query(UpdateDocParams::default()),
- Json(EsUpdateBody {
- doc: None,
- doc_as_upsert: false,
- upsert: None,
- script: Some(json!("ctx._source.n += 1")),
- detect_noop: None,
- }),
+ bytes::Bytes::from(
+ serde_json::to_vec(&json!({"script": "ctx._source.n += 1"}))
+ .expect("script update body"),
+ ),
)
.await
.into_response()
diff --git a/engine/crates/xerj-autoindex/src/extract/mbox.rs b/engine/crates/xerj-autoindex/src/extract/mbox.rs
index f03496c4a..561b3dae4 100644
--- a/engine/crates/xerj-autoindex/src/extract/mbox.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mbox.rs
@@ -755,7 +755,9 @@ fn extract_parallel<R: BufRead + Send>(
let mut stats = ExtractStats::default();
let mut failure: Option<std::io::Error> = None;
- std::thread::scope(|scope| {
+ // #1122: see the spawn below. The scope joins before `reader` and
+ // the channels go out of scope.
+ xerj_common::thread_spawn::scope(|scope| {
let splitter_tx = done_tx.clone();
let (stop_ref, budget_ref) = (&stop, &budget);
let splitter = move || {
@@ -785,42 +787,41 @@ fn extract_parallel<R: BufRead + Send>(
let _ = splitter_tx.send(Done::End { total: seq });
// `job_tx` drops here: the parsers drain what is queued and exit.
};
- // #1122: `Scope::spawn` panics when a thread cannot be created, and
- // release builds are panic=abort. Spawn through the fallible builder:
- // fewer parsers than asked for only costs speed; none at all — or no
- // splitter — fails this mailbox through the ordinary error path below
- // instead of aborting the run.
+ // #1122: `Builder::spawn_scoped` returns `Ok`, then the child panics
+ // if it cannot map an alternate signal stack (`panic = "abort"`
+ // takes the process with it). This scope uses `pthread_create`.
+ // Fewer parsers than asked for only costs speed; none at all — or
+ // no splitter — fails this mailbox through the ordinary error path
+ // below. A failed closure is dropped unrun.
let mut parsers = 0usize;
let mut spawn_error: Option<std::io::Error> = None;
for _ in 0..workers {
let job_rx = Arc::clone(&job_rx);
let done_tx = done_tx.clone();
- let spawned = std::thread::Builder::new()
- .name("xerj-mbox-parse".into())
- .spawn_scoped(scope, move || loop {
- // Hold the lock only while waiting for a job, never while
- // parsing one: the other parsers must be able to take theirs.
- let job = match job_rx.lock() {
- Ok(rx) => rx.recv(),
- Err(_) => return,
- };
- let Ok(Job { seq, msg }) = job else { return };
- let _slot = gate().acquire();
- let bytes = msg.bytes.len();
- let (records, stats) = extract_one(&msg);
- drop(msg);
- if done_tx
- .send(Done::Message {
- seq,
- bytes,
- records,
- stats,
- })
- .is_err()
- {
- return;
- }
- });
+ let spawned = scope.spawn("xerj-mbox-parse", move || loop {
+ // Hold the lock only while waiting for a job, never while
+ // parsing one: the other parsers must be able to take theirs.
+ let job = match job_rx.lock() {
+ Ok(rx) => rx.recv(),
+ Err(_) => return,
+ };
+ let Ok(Job { seq, msg }) = job else { return };
+ let _slot = gate().acquire();
+ let bytes = msg.bytes.len();
+ let (records, stats) = extract_one(&msg);
+ drop(msg);
+ if done_tx
+ .send(Done::Message {
+ seq,
+ bytes,
+ records,
+ stats,
+ })
+ .is_err()
+ {
+ return;
+ }
+ });
match spawned {
Ok(_) => parsers += 1,
Err(error) => {
@@ -837,10 +838,7 @@ fn extract_parallel<R: BufRead + Send>(
std::io::Error::other("no mailbox parser threads were requested")
}))
} else {
- std::thread::Builder::new()
- .name("xerj-mbox-split".into())
- .spawn_scoped(scope, splitter)
- .map(drop)
+ scope.spawn("xerj-mbox-split", splitter)
};
if let Err(error) = splitter_spawn {
// The failed closure dropped `job_tx`, so any parsers that did
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index ef4668bf1..31146c40c 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -10201,19 +10201,17 @@ fn run_index_report_inner(
files_done.fetch_add(1, Ordering::Relaxed);
}
};
- // #1122: `Scope::spawn` panics when a thread cannot be created (thread
- // limit, memory exhaustion), and release builds are panic=abort, so one
- // failed spawn used to abort the whole run. Spawn through the fallible
- // builder instead: workers that did start share the queue between them,
- // the pipe stops counting the ones that never did, and if none started
- // this thread works the queue itself. Slower, never wrong.
- std::thread::scope(|scope| {
+ // #1122: `Builder::spawn_scoped` returns `Ok`, then the child panics
+ // if it cannot map an alternate signal stack. Release builds are
+ // panic=abort, so that panic takes the process down. `thread_spawn::scope`
+ // uses `pthread_create` and does not install that stack. A refusal is
+ // an error: workers that did start share the queue, the pipe stops
+ // counting the ones that never did, and if none started this thread
+ // works the queue itself. The failed closure is dropped unrun.
+ xerj_common::thread_spawn::scope(|scope| {
let mut started = 0usize;
for _ in 0..worker_count {
- match std::thread::Builder::new()
- .name("autoindex-worker".into())
- .spawn_scoped(scope, worker)
- {
+ match scope.spawn("autoindex-worker", worker) {
Ok(_) => started += 1,
Err(error) => {
pr.note(&format!(
diff --git a/engine/crates/xerj-autoindex/src/objwatch/mod.rs b/engine/crates/xerj-autoindex/src/objwatch/mod.rs
index 5c9fbcb22..f43c40655 100644
--- a/engine/crates/xerj-autoindex/src/objwatch/mod.rs
+++ b/engine/crates/xerj-autoindex/src/objwatch/mod.rs
@@ -695,17 +695,15 @@ fn fetch_batch(
let r = src.get(keys[i], max_bytes);
*slots[i].lock().unwrap_or_else(|e| e.into_inner()) = Some(r);
};
- std::thread::scope(|scope| {
+ // #1122: `Builder::spawn_scoped` returns `Ok` and the child then
+ // panics if the alternate signal stack cannot be mapped. This scope
+ // does not install that stack. The fetchers share one cursor, so a
+ // short pool only costs concurrency: stop spawning and let this
+ // thread fetch alongside whichever ones started. The failed closure
+ // is dropped unrun; `fetch` below is a separate call.
+ xerj_common::thread_spawn::scope(|scope| {
for _ in 0..FETCH_CONCURRENCY.min(keys.len()) {
- // #1122: `Scope::spawn` panics when a thread cannot be created,
- // and release builds are panic=abort. The fetchers share one
- // cursor, so a short pool only costs concurrency: stop spawning
- // and let this thread fetch alongside whichever ones started.
- if std::thread::Builder::new()
- .name("xerj-objwatch-fetch".into())
- .spawn_scoped(scope, fetch)
- .is_err()
- {
+ if scope.spawn("xerj-objwatch-fetch", fetch).is_err() {
fetch();
break;
}
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index b9b01212a..99a0d1876 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -358,10 +358,14 @@ pub(crate) fn replay_windowed<T: Sync>(
/// One dispatched operation: on its own thread, or — when that thread
/// could not be started (#1122) — already applied inline.
enum Dispatched<'scope> {
- Thread(std::thread::ScopedJoinHandle<'scope, Result<()>>),
+ Thread(xerj_common::thread_spawn::ScopedJoinHandle<'scope, Result<()>>),
Inline(Result<()>),
}
- let failure = std::thread::scope(|scope| -> Result<Option<anyhow::Error>> {
+ // #1122: `Builder::spawn_scoped` can abort the process after returning
+ // `Ok`, when the child cannot map its alternate signal stack. This
+ // scope uses `pthread_create`. A refusal applies the item on this
+ // thread; the closure that would have run it is dropped unrun.
+ let failure = xerj_common::thread_spawn::scope(|scope| -> Result<Option<anyhow::Error>> {
let mut next = items.iter();
let mut in_flight: std::collections::VecDeque<(&T, Dispatched<'_>)> =
std::collections::VecDeque::new();
@@ -374,14 +378,7 @@ pub(crate) fn replay_windowed<T: Sync>(
let Some(item) = next.next() else { break };
hooks.begin(item)?;
let dispatched = item;
- // #1122: `Scope::spawn` panics when a thread cannot be
- // created, and release builds are panic=abort. A failed spawn
- // never ran the closure, so the operation is applied inline
- // instead: the window narrows, the order and the callbacks do
- // not change.
- let slot = match std::thread::Builder::new()
- .name("xerj-sync-apply".into())
- .spawn_scoped(scope, move || apply(dispatched))
+ let slot = match scope.spawn_returning("xerj-sync-apply", move || apply(dispatched))
{
Ok(handle) => Dispatched::Thread(handle),
Err(_) => Dispatched::Inline(apply(item)),
diff --git a/engine/crates/xerj-common/src/document_limit.rs b/engine/crates/xerj-common/src/document_limit.rs
index 89700d3af..1ff92c757 100644
--- a/engine/crates/xerj-common/src/document_limit.rs
+++ b/engine/crates/xerj-common/src/document_limit.rs
@@ -21,6 +21,39 @@ use serde_json::Value;
/// skipped. A single value over this limit is not indexed.
pub const MAX_INDEXED_DOCUMENT_BYTES: usize = 8 << 20;
+/// Extra bytes an `_update` envelope may use beyond two documents at the ceiling.
+///
+/// The body can carry a partial `doc` and an `upsert`, each a full indexed
+/// document, plus a script. The engine caps a script source at 64 KiB;
+/// JSON escaping can double that, and the object keys sit beside the two
+/// bodies. 256 KiB covers that wrapping. A larger raw body is refused
+/// before `serde_json` builds a DOM (#1122).
+pub const UPDATE_ENVELOPE_SLACK_BYTES: usize = 256 * 1024;
+
+/// Largest raw `POST /{index}/_update/{id}` body that is parsed.
+///
+/// Bytes past this limit are refused before `serde_json`. A body at or
+/// under the limit is still subject to [`MAX_INDEXED_DOCUMENT_BYTES`] on
+/// the document the update would write.
+pub const MAX_UPDATE_BODY_BYTES: usize =
+ 2 * MAX_INDEXED_DOCUMENT_BYTES + UPDATE_ENVELOPE_SLACK_BYTES;
+
+/// `true` when an `_update` body is past [`MAX_UPDATE_BODY_BYTES`].
+pub fn update_body_too_large(len: usize) -> bool {
+ len > MAX_UPDATE_BODY_BYTES
+}
+
+/// Refusal text for an `_update` body of `len` bytes.
+///
+/// The text does not say the body was invalid JSON: the size check runs
+/// before parse, including when the bytes are not JSON.
+pub fn update_body_refusal(len: usize) -> String {
+ format!(
+ "update body is {len} bytes, over the {MAX_UPDATE_BODY_BYTES} byte limit; \
+ it was not parsed and the existing document was not changed"
+ )
+}
+
/// `true` when `len` is past [`MAX_INDEXED_DOCUMENT_BYTES`].
pub fn indexed_document_too_large(len: usize) -> bool {
len > MAX_INDEXED_DOCUMENT_BYTES
@@ -103,6 +136,28 @@ mod tests {
assert!(indexed_document_too_large(MAX_INDEXED_DOCUMENT_BYTES + 1));
}
+ #[test]
+ fn an_update_envelope_fits_two_documents_and_refuses_one_byte_over() {
+ assert_eq!(
+ MAX_UPDATE_BODY_BYTES,
+ 2 * MAX_INDEXED_DOCUMENT_BYTES + 256 * 1024
+ );
+ assert!(!update_body_too_large(MAX_UPDATE_BODY_BYTES));
+ assert!(update_body_too_large(MAX_UPDATE_BODY_BYTES + 1));
+ let text = update_body_refusal(MAX_UPDATE_BODY_BYTES + 1);
+ assert!(
+ text.contains(&(MAX_UPDATE_BODY_BYTES + 1).to_string()),
+ "{text}"
+ );
+ assert!(text.contains(&MAX_UPDATE_BODY_BYTES.to_string()), "{text}");
+ assert!(text.contains("not parsed"), "{text}");
+ assert!(text.contains("not changed"), "{text}");
+ assert!(
+ !text.contains("invalid JSON"),
+ "the size refusal is not a parse error: {text}"
+ );
+ }
+
#[test]
fn refusal_names_the_size_and_the_limit() {
let text = indexed_document_refusal(50 * 1024 * 1024);
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index 04afcfd34..4d5bb4802 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -38,7 +38,9 @@
//! on a failed spawn (#1122): Unix threads are created with `pthread_create`
//! so a signal-stack allocation failure is an error instead of a panic in
//! the child, the closure comes back, and a process-wide memory spawn gate
-//! skips best-effort spawns once the breaker is engaged
+//! skips best-effort spawns once the breaker is engaged. Scoped borrows
+//! use the same `pthread_create` path and are joined before the scope
+//! returns
//! - [`hub`] — Corpus Hub v2 shared types: the catalogue, pack manifest,
//! definition record, and the [`hub::Answer`] every surface renders (#1374)
diff --git a/engine/crates/xerj-common/src/thread_spawn.rs b/engine/crates/xerj-common/src/thread_spawn.rs
index 5d87e38bd..ae8960e25 100644
--- a/engine/crates/xerj-common/src/thread_spawn.rs
+++ b/engine/crates/xerj-common/src/thread_spawn.rs
@@ -36,8 +36,21 @@
//!
//! The actual spawn goes through the [`Spawn`] trait so the failure paths
//! are unit-testable without exhausting a real machine.
+//!
+//! Scoped work cannot use [`try_spawn`]. The closure borrows the caller's
+//! stack: autoindex workers, sync replay, an object-watch fetch, a mailbox
+//! splitter. [`std::thread::Builder::spawn_scoped`] still returns `Ok` and
+//! then panics in the child when the alternate signal stack cannot be
+//! mapped. [`scope`] starts those threads with the same `pthread_create`
+//! path and joins them before returning, so the borrow stays valid. A
+//! failed spawn is an [`io::Error`]. The closure is dropped unrun; the
+//! caller runs the work inline or records the error. This is not on the
+//! per-document path: one `pthread_create` per thread, then the job the
+//! caller already had.
use std::io;
+use std::marker::PhantomData;
+use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::Duration;
@@ -497,6 +510,289 @@ where
}
}
+/// A thread [`scope`] has started and not yet joined.
+struct ScopedThread {
+ handle: SpawnedThread,
+ /// Set when a [`Scope::spawn`] job panics on an unwind build. A
+ /// [`Scope::spawn_returning`] job sends its panic to the join handle
+ /// instead, so a caller that turned the panic into an error is not
+ /// panicked again when the scope ends.
+ panic: Arc<Mutex<Option<Box<dyn std::any::Any + Send>>>>,
+}
+
+impl Drop for ScopedThread {
+ fn drop(&mut self) {
+ // Join, do not detach. Detach would let the job use the caller's
+ // borrows after `scope` returned.
+ let _ = self.handle.join_inner();
+ }
+}
+
+/// Join handle for [`Scope::spawn_returning`].
+///
+/// The value is available once the job returns. The scope also joins the
+/// operating-system thread if this handle is dropped, so a borrow inside
+/// the job cannot outlive the scope. Dropping the handle discards the
+/// value, including a panic payload.
+pub struct ScopedJoinHandle<'scope, T> {
+ rx: std::sync::mpsc::Receiver<std::thread::Result<T>>,
+ _scope: PhantomData<&'scope ()>,
+}
+
+impl<T> std::fmt::Debug for ScopedJoinHandle<'_, T> {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("ScopedJoinHandle").finish_non_exhaustive()
+ }
+}
+
+impl<T> ScopedJoinHandle<'_, T> {
+ /// Wait for the job and return its value.
+ ///
+ /// `Err` is a panic from the job on an unwind build. On the release
+ /// profile (`panic = "abort"`) a panic aborts the process before this
+ /// returns. A thread that exits without sending is also `Err`.
+ pub fn join(self) -> std::thread::Result<T> {
+ self.rx.recv().unwrap_or_else(|_| {
+ Err(Box::new(io::Error::other(
+ "scoped thread ended without a result",
+ )))
+ })
+ }
+}
+
+/// The threads started inside one [`scope`] call.
+///
+/// Constructed only by [`scope`], which joins every thread before it
+/// returns, including when the body panics. Jobs borrow the caller's
+/// stack, so that join is what keeps the borrows valid.
+pub struct Scope<'scope, 'env: 'scope> {
+ threads: Mutex<Vec<ScopedThread>>,
+ _scope: PhantomData<&'scope mut &'scope ()>,
+ _env: PhantomData<&'env mut &'env ()>,
+}
+
+impl<'scope, 'env: 'scope> Scope<'scope, 'env> {
+ /// Start `f` on a helper thread named `name`.
+ ///
+ /// `Err` means `f` did not run. The caller may do the work on this
+ /// thread or skip it. The error is logged once per call.
+ pub fn spawn<F>(&self, name: &str, f: F) -> io::Result<()>
+ where
+ F: FnOnce() + Send + 'scope,
+ {
+ self.spawn_with(&StdSpawn, spawn_allowed(), name, f)
+ }
+
+ /// [`Scope::spawn`] with an explicit spawner and gate verdict.
+ ///
+ /// Tests pass a spawner that fails. Production code uses [`Scope::spawn`].
+ pub fn spawn_with<S, F>(&self, spawner: &S, allowed: bool, name: &str, f: F) -> io::Result<()>
+ where
+ S: Spawn + ?Sized,
+ F: FnOnce() + Send + 'scope,
+ {
+ if !allowed {
+ drop(f);
+ let error = gate_refused(name);
+ log_scoped_spawn_failure(name, &error);
+ return Err(error);
+ }
+ let panic = Arc::new(Mutex::new(None));
+ let slot = Arc::clone(&panic);
+ // SAFETY: `scope` joins this thread before it returns, or this
+ // function drops the closure before returning `Err`. Either way
+ // the `'scope` borrow is still alive.
+ let job: Box<dyn FnOnce() + Send + 'static> = unsafe { erase_fnonce_lifetime(Box::new(f)) };
+ self.start(
+ spawner,
+ name,
+ Box::new(move || {
+ if let Err(payload) = std::panic::catch_unwind(AssertUnwindSafe(job)) {
+ *slot.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(payload);
+ }
+ }),
+ panic,
+ )
+ }
+
+ /// Start `f` and return a handle for its value.
+ ///
+ /// Same failure contract as [`Scope::spawn`]: on `Err`, `f` did not run.
+ /// A panic on an unwind build is [`ScopedJoinHandle::join`]'s `Err`,
+ /// not a panic of the scope, so the caller can record it and keep going.
+ pub fn spawn_returning<T, F>(&self, name: &str, f: F) -> io::Result<ScopedJoinHandle<'scope, T>>
+ where
+ T: Send + 'static,
+ F: FnOnce() -> T + Send + 'scope,
+ {
+ self.spawn_returning_with(&StdSpawn, spawn_allowed(), name, f)
+ }
+
+ /// [`Scope::spawn_returning`] with an explicit spawner and gate verdict.
+ pub fn spawn_returning_with<S, T, F>(
+ &self,
+ spawner: &S,
+ allowed: bool,
+ name: &str,
+ f: F,
+ ) -> io::Result<ScopedJoinHandle<'scope, T>>
+ where
+ S: Spawn + ?Sized,
+ T: Send + 'static,
+ F: FnOnce() -> T + Send + 'scope,
+ {
+ if !allowed {
+ drop(f);
+ let error = gate_refused(name);
+ log_scoped_spawn_failure(name, &error);
+ return Err(error);
+ }
+ let (tx, rx) = std::sync::mpsc::channel();
+ // SAFETY: same as [`Scope::spawn_with`]. `T` is `'static`, so the
+ // value can sit in the channel until `join` without borrowing
+ // `'scope`. The closure itself is not `'static` and is only erased
+ // for the thread.
+ let job: Box<dyn FnOnce() -> T + Send + 'static> =
+ unsafe { erase_fnonce_returning_lifetime(Box::new(f)) };
+ self.start(
+ spawner,
+ name,
+ Box::new(move || {
+ let result = std::panic::catch_unwind(AssertUnwindSafe(job));
+ let _ = tx.send(result);
+ }),
+ Arc::new(Mutex::new(None)),
+ )?;
+ Ok(ScopedJoinHandle {
+ rx,
+ _scope: PhantomData,
+ })
+ }
+
+ fn start<S>(
+ &self,
+ spawner: &S,
+ name: &str,
+ job: Box<dyn FnOnce() + Send + 'static>,
+ panic: Arc<Mutex<Option<Box<dyn std::any::Any + Send>>>>,
+ ) -> io::Result<()>
+ where
+ S: Spawn + ?Sized,
+ {
+ match try_spawn_with(spawner, true, name, job) {
+ Ok(handle) => {
+ self.threads
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .push(ScopedThread { handle, panic });
+ Ok(())
+ }
+ Err(failure) => {
+ // The thread did not run. Dropping the job drops the
+ // caller's closure while `'scope` is still alive.
+ drop(failure.job);
+ log_scoped_spawn_failure(name, &failure.error);
+ Err(failure.error)
+ }
+ }
+ }
+
+ /// Join every thread started so far. A second call joins nothing.
+ fn finish(&self) -> Option<Box<dyn std::any::Any + Send>> {
+ let threads = {
+ let mut guard = self
+ .threads
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ std::mem::take(&mut *guard)
+ };
+ let mut panic = None;
+ for mut thread in threads {
+ // `ScopedThread`'s drop joins too. Joining here first means the
+ // panic slot is filled before we read it.
+ let _ = thread.handle.join_inner();
+ if panic.is_none() {
+ panic = thread
+ .panic
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .take();
+ }
+ }
+ panic
+ }
+}
+
+/// Run `f`, then join every thread it started.
+///
+/// A panic in `f` is resumed after the join. A panic in a
+/// [`Scope::spawn`] job is resumed after `f` returns, unless `f` itself
+/// panicked. A [`Scope::spawn_returning`] panic is only visible from
+/// [`ScopedJoinHandle::join`].
+pub fn scope<'env, F, R>(f: F) -> R
+where
+ F: for<'scope> FnOnce(&'scope Scope<'scope, 'env>) -> R,
+{
+ let scope = Scope {
+ threads: Mutex::new(Vec::new()),
+ _scope: PhantomData,
+ _env: PhantomData,
+ };
+ let result = std::panic::catch_unwind(AssertUnwindSafe(|| f(&scope)));
+ let child_panic = scope.finish();
+ match result {
+ Err(parent) => std::panic::resume_unwind(parent),
+ Ok(value) => {
+ if let Some(child) = child_panic {
+ std::panic::resume_unwind(child);
+ }
+ value
+ }
+ }
+}
+
+fn log_scoped_spawn_failure(name: &str, error: &io::Error) {
+ tracing::warn!(
+ thread = name,
+ error = %error,
+ "scoped helper thread was not started; the caller degrades and the process stays up (#1122)"
+ );
+}
+
+/// # Safety
+///
+/// The caller joins the thread that runs `job`, or drops `job`, before
+/// the `'scope` borrows end.
+unsafe fn erase_fnonce_lifetime<'scope>(
+ job: Box<dyn FnOnce() + Send + 'scope>,
+) -> Box<dyn FnOnce() + Send + 'static> {
+ // SAFETY: caller contract. The fat pointer does not depend on the
+ // lifetime. This is the same erasure `std::thread::scope` uses so a
+ // stack borrow can cross `pthread_create`; the join is what makes it
+ // sound, and that join is [`scope`]'s.
+ unsafe {
+ std::mem::transmute::<Box<dyn FnOnce() + Send + 'scope>, Box<dyn FnOnce() + Send + 'static>>(
+ job,
+ )
+ }
+}
+
+/// # Safety
+///
+/// Same as [`erase_fnonce_lifetime`]. `T` is `'static`, so the return
+/// value does not borrow `'scope`.
+unsafe fn erase_fnonce_returning_lifetime<'scope, T: Send + 'static>(
+ job: Box<dyn FnOnce() -> T + Send + 'scope>,
+) -> Box<dyn FnOnce() -> T + Send + 'static> {
+ // SAFETY: caller contract, same as [`erase_fnonce_lifetime`].
+ unsafe {
+ std::mem::transmute::<
+ Box<dyn FnOnce() -> T + Send + 'scope>,
+ Box<dyn FnOnce() -> T + Send + 'static>,
+ >(job)
+ }
+}
+
/// Linux `TASK_COMM_LEN`, including the trailing NUL. `std::thread` truncates
/// to the same width for the OS name; the full name still goes in our logs.
#[cfg(unix)]
@@ -923,6 +1219,168 @@ mod tests {
assert_eq!(spawner.calls.load(Ordering::SeqCst), 1);
}
+ #[test]
+ fn a_scoped_job_borrows_the_caller_stack_and_finishes_before_scope_returns() {
+ let value = 11u32;
+ let finished = AtomicBool::new(false);
+ scope(|s| {
+ s.spawn("autoindex-worker", || {
+ assert_eq!(value, 11, "the job sees the caller's stack");
+ finished.store(true, Ordering::SeqCst);
+ })
+ .expect("spawn");
+ });
+ assert!(
+ finished.load(Ordering::SeqCst),
+ "scope returned before the job finished"
+ );
+ }
+
+ #[test]
+ fn a_failed_scoped_spawn_does_not_run_the_job() {
+ let ran = AtomicUsize::new(0);
+ let spawner = FlakySpawn::new(1);
+ let error = scope(|s| {
+ s.spawn_with(&spawner, true, "autoindex-worker", || {
+ ran.fetch_add(1, Ordering::SeqCst);
+ })
+ .expect_err("the spawner fails")
+ });
+ assert_eq!(error.raw_os_error(), Some(libc::EAGAIN));
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 1);
+ assert_eq!(ran.load(Ordering::SeqCst), 0, "the job was dropped unrun");
+ // Indexing workers and objwatch do the work on this thread instead.
+ ran.fetch_add(1, Ordering::SeqCst);
+ assert_eq!(ran.load(Ordering::SeqCst), 1);
+ }
+
+ #[test]
+ fn a_closed_gate_does_not_call_the_scoped_spawner() {
+ let spawner = FlakySpawn::new(0);
+ let error = scope(|s| {
+ s.spawn_with(&spawner, false, "xerj-mbox-parse", || {})
+ .expect_err("gate closed")
+ });
+ assert_eq!(error.kind(), io::ErrorKind::OutOfMemory);
+ assert_eq!(spawner.calls.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn a_later_scoped_failure_does_not_cancel_a_thread_that_started() {
+ let ran = Arc::new(AtomicUsize::new(0));
+ let spawner = FailSecondSpawn::default();
+ scope(|s| {
+ let job_ran = Arc::clone(&ran);
+ s.spawn_with(&spawner, true, "xerj-objwatch-fetch", move || {
+ job_ran.fetch_add(1, Ordering::SeqCst);
+ })
+ .expect("first spawn");
+ let job_ran = Arc::clone(&ran);
+ let error = s
+ .spawn_with(&spawner, true, "xerj-objwatch-fetch", move || {
+ job_ran.fetch_add(10, Ordering::SeqCst);
+ })
+ .expect_err("second spawn");
+ assert_eq!(error.raw_os_error(), Some(libc::EAGAIN));
+ });
+ assert_eq!(ran.load(Ordering::SeqCst), 1, "only the started job ran");
+ }
+
+ #[test]
+ fn a_returning_scoped_spawn_yields_the_value_and_a_failure_does_not_apply() {
+ let ran = AtomicUsize::new(0);
+ let value = scope(|s| {
+ let handle = s
+ .spawn_returning("xerj-sync-apply", || 7u32)
+ .expect("spawn");
+ handle.join().expect("join")
+ });
+ assert_eq!(value, 7);
+
+ let error_value = scope(|s| {
+ match s.spawn_returning_with(&FlakySpawn::new(1), true, "xerj-sync-apply", || {
+ ran.fetch_add(1, Ordering::SeqCst);
+ 9u32
+ }) {
+ Ok(handle) => handle.join().expect("unexpected success"),
+ Err(error) => {
+ assert_eq!(error.raw_os_error(), Some(libc::EAGAIN));
+ assert_eq!(ran.load(Ordering::SeqCst), 0, "apply did not run");
+ // replay_windowed applies the item on this thread instead.
+ 9u32
+ }
+ }
+ });
+ assert_eq!(error_value, 9);
+ assert_eq!(ran.load(Ordering::SeqCst), 0);
+ }
+
+ #[test]
+ fn a_returning_scoped_panic_is_the_join_error_and_scope_returns() {
+ let joined = scope(|s| {
+ let handle = s
+ .spawn_returning("xerj-sync-apply", || -> u32 { panic!("apply-boom") })
+ .expect("spawn");
+ handle.join()
+ });
+ let payload = joined.expect_err("the panic is the join error");
+ let message = payload
+ .downcast_ref::<&str>()
+ .copied()
+ .or_else(|| payload.downcast_ref::<String>().map(String::as_str))
+ .unwrap_or("");
+ assert!(message.contains("apply-boom"), "{message}");
+ }
+
+ #[test]
+ #[should_panic(expected = "scoped-boom")]
+ fn a_scoped_child_panic_is_resumed_after_the_scope_joins() {
+ scope(|s| {
+ s.spawn("xerj-mbox-split", || panic!("scoped-boom"))
+ .expect("spawn");
+ });
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn a_scoped_helper_does_not_install_an_alternate_signal_stack() {
+ // Same regression as `a_spawned_helper_does_not_install_an_alternate_signal_stack`.
+ // `spawn_scoped` used to take `std::thread::Builder`, whose child
+ // mmaps a signal stack after the parent has already been told `Ok`.
+ let (tx, rx) = std::sync::mpsc::channel();
+ scope(|s| {
+ s.spawn("autoindex-worker", move || {
+ tx.send(alternate_signal_stack_flags()).unwrap();
+ })
+ .expect("spawn");
+ });
+ let (rc, flags) = rx.recv().unwrap();
+ assert_eq!(rc, 0, "sigaltstack query failed");
+ assert_ne!(
+ flags & libc::SS_DISABLE,
+ 0,
+ "scoped helper installed an alternate signal stack; an ENOMEM \
+ mmap of it panics inside the child and aborts the process (#1122)"
+ );
+ }
+
+ /// Fails the second `spawn` and lets the first through.
+ #[derive(Default)]
+ struct FailSecondSpawn {
+ calls: AtomicU32,
+ }
+
+ impl Spawn for FailSecondSpawn {
+ fn spawn(&self, name: &str, job: BoxedJob) -> io::Result<SpawnedThread> {
+ let call = self.calls.fetch_add(1, Ordering::SeqCst);
+ if call == 1 {
+ drop(job);
+ return Err(io::Error::from_raw_os_error(libc::EAGAIN));
+ }
+ StdSpawn.spawn(name, job)
+ }
+ }
+
/// `(rc, ss_flags)` from `sigaltstack`. `SS_DISABLE` set means the
/// thread has no alternate signal stack — the mapping `std::thread`
/// panics on if it cannot be created.