Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): return from a parked bulk submit once another worker shipped its bytes (#2841) (#2863)

Jake Elbirt284cf78e0060
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02f100256..22edf2e2a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -101,6 +101,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- `xerj autoindex` no longer stalls near the end of a run at high
+ `--workers`. When all 8 bulk slots were busy, a worker whose chunk crossed
+ the flush size parked; if another worker then shipped the shared buffer,
+ that chunk included, the first one woke to an empty buffer and parked again
+ every 30 s for good, so the run never finished. It now returns once the
+ bulk carrying its chunk has landed. The same loop could also return while
+ that bulk was still on the wire, so a file could be journaled complete
+ before, or even though, one of its bulks failed; it now waits. On a
+ 26-file, 28 MB scientific CSV folder, rc.100 stalled at 21 of 26 files with
+ `--workers 32` and 19 of 26 with `--workers 16`; both now finish all 26
+ files and 179,374 records (#2841).
+
- `weighted_avg` now executes field-based requests (`value` / `weight`, each
with a `field` and optional numeric `missing`) instead of returning HTTP 200
with an `unsupported aggregation type` error in the aggregations body. A
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index a82ccf681..42330c25d 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3498,9 +3498,10 @@ struct PipeState {
/// Workers that may still call [`BulkPipe::submit`]. Decremented by
/// [`BulkPipe::retire`] as each worker drains its queue.
live_workers: usize,
- /// Live workers currently parked inside `settle` (or waiting for an
- /// in-flight slot in `submit`). `live == parked` is the "no more bytes
- /// are coming" signal that flushes a residual buffer.
+ /// Live workers currently parked inside `settle`, or inside `submit`
+ /// waiting for an in-flight slot or for the bulk carrying their chunk to
+ /// land (that worker submits again once it does). `live == parked` is
+ /// the "no more bytes are coming" signal that flushes a residual buffer.
parked_workers: usize,
/// Bulk bodies currently on the wire. Capped at [`MAX_IN_FLIGHT`].
in_flight: usize,
@@ -3543,10 +3544,11 @@ const MAX_IN_FLIGHT: usize = 8;
/// The pipe lock is NOT held across the wire: a flush detaches the buffer as
/// a job, drops the lock, sends, and resolves tokens on return, so up to
/// [`MAX_IN_FLIGHT`] bulks are on the wire at once. Per-file order still
-/// holds — a file's cut chunks are sent synchronously by its own worker
-/// inside `submit`, so any later job carrying its tail ships after those
-/// completed; files interleave arbitrarily, exactly as the 32 parallel
-/// per-file buffers always did.
+/// holds: a file's cut chunk has landed before its worker's `submit`
+/// returns (sent by that worker, or by another that took the buffer while
+/// it was parked on a full wire), so any later job carrying its tail ships
+/// after those completed; files interleave arbitrarily, exactly as the 32
+/// parallel per-file buffers always did.
///
/// Durability semantics are the per-file code's, unchanged: a token settles
/// `Ok` only after EVERY bulk carrying its records was accepted (server_errors
@@ -3632,24 +3634,15 @@ where
if st.buf.len() < self.bulk_cut && st.docs < self.doc_cap {
return;
}
- // Cut crossed: ship inline. If every wire slot is busy, park until a
- // delivery frees one — backpressure, exactly like the per-file code
- // blocking inside its own bulk send.
- let job = loop {
- if let Some(job) = self.take_job(&mut st) {
- break job;
- }
- st.parked_workers += 1;
- let (guard, _) = self
- .ready
- .wait_timeout(st, std::time::Duration::from_secs(30))
- .unwrap_or_else(|p| p.into_inner());
- st = guard;
- st.parked_workers -= 1;
- };
- drop(st);
- let outcome = (self.send)(job.0);
- self.deliver(job.1, job.2, outcome);
+ // Cut crossed: ship inline, and return only once these bytes have
+ // landed. If every wire slot is busy, park until a delivery frees
+ // one: backpressure, exactly like the per-file code blocking inside
+ // its own bulk send. If another worker ships the buffer meanwhile,
+ // these bytes with it, that job was this worker's flush. Waiting for
+ // a job of its own instead parked on an empty buffer for good
+ // (#2841), and shipping someone else's bytes in its place returned
+ // while this worker's own chunk was still on the wire.
+ self.quiesce_locked(st, token);
}
/// The server's reason when this token failed only on records over the
@@ -3659,11 +3652,18 @@ where
}
/// Block until no bulk holding this file's records is buffered or in
- /// flight. A failed token settles at its first refusal, while its later
- /// chunks may still be on the wire; a purge before they land would leave
- /// a junk file half searchable.
+ /// flight, shipping the buffer while they are still in it. `submit`
+ /// flushes through this. A failed token settles at its first refusal,
+ /// while its later chunks may still be on the wire; a purge before they
+ /// land would leave a junk file half searchable.
fn quiesce(&self, token: u64) {
- let mut st = self.lock();
+ self.quiesce_locked(self.lock(), token);
+ }
+
+ /// [`Self::quiesce`] under a lock the caller already holds, so `submit`
+ /// ships the buffer it just crossed the cut with before any other
+ /// worker can add to it.
+ fn quiesce_locked<'s>(&'s self, mut st: std::sync::MutexGuard<'s, PipeState>, token: u64) {
loop {
let buffered = st.pending.contains(&token);
if !buffered && !st.in_flight_tokens.contains_key(&token) {
@@ -17273,6 +17273,202 @@ mod bulk_pipe_tests {
"{WORKERS}x{FILES_PER_WORKER} files left as {n} bodies — no coalescing happened"
);
}
+
+ /// #2841: with every wire slot busy, two workers cross the cut and park.
+ /// When a slot frees, the first one awake ships the whole buffer, the
+ /// other's bytes with it. The other wakes to an empty buffer, so it can
+ /// never take a job of its own; it must still return once its bytes were
+ /// accepted. It used to park again, every 30 s, for good, and the run
+ /// stalled at 16 and 32 workers on a 26-file corpus.
+ #[test]
+ fn a_worker_whose_bytes_another_worker_shipped_returns() {
+ use std::sync::Condvar;
+ use std::time::{Duration, Instant};
+
+ // 'static, so a worker that never returns fails this test instead
+ // of hanging it inside a thread scope.
+ let gate: &'static (Mutex<bool>, Condvar) =
+ Box::leak(Box::new((Mutex::new(false), Condvar::new())));
+ let on_wire: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));
+ let rejected: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));
+ let pipe = &*Box::leak(Box::new(BulkPipe::new(
+ move |_body| {
+ on_wire.fetch_add(1, Ordering::SeqCst);
+ let (open, cv) = gate;
+ let mut open = open.lock().unwrap();
+ while !*open {
+ open = cv.wait(open).unwrap();
+ }
+ Ok(outcome())
+ },
+ 64,
+ 5000,
+ rejected,
+ 10,
+ )));
+ let wait_until = |what: &str, ok: &dyn Fn() -> bool| {
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !ok() {
+ assert!(Instant::now() < deadline, "timed out waiting for {what}");
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ };
+
+ // Fill every wire slot with a body the gate holds.
+ for _ in 0..super::MAX_IN_FLIGHT {
+ let t = pipe.token();
+ std::thread::spawn(move || pipe.submit(t, vec![b'x'; 200], 1));
+ }
+ wait_until("every wire slot to be busy", &|| {
+ on_wire.load(Ordering::SeqCst) == super::MAX_IN_FLIGHT as u64
+ });
+
+ let (tx, rx) = std::sync::mpsc::channel();
+ let mut tokens = Vec::new();
+ for parked in 1..=2 {
+ let t = pipe.token();
+ tokens.push(t);
+ let tx = tx.clone();
+ std::thread::spawn(move || {
+ pipe.submit(t, vec![b'y'; 200], 1);
+ tx.send(t).unwrap();
+ });
+ wait_until("a worker to park on the full wire", &|| {
+ pipe.lock().parked_workers == parked
+ });
+ }
+
+ let (open, cv) = gate;
+ *open.lock().unwrap() = true;
+ cv.notify_all();
+
+ for _ in 0..2 {
+ rx.recv_timeout(Duration::from_secs(10))
+ .expect("both parked workers return once their bytes were shipped");
+ }
+ for t in tokens {
+ pipe.settle(t).expect("accepted");
+ }
+ assert_eq!(
+ on_wire.load(Ordering::SeqCst),
+ super::MAX_IN_FLIGHT as u64 + 1,
+ "the two parked workers' bytes rode one body"
+ );
+ }
+
+ /// A parked worker whose chunk another worker shipped must not return
+ /// until that chunk has landed, even when the wire frees up and the
+ /// buffer holds someone else's bytes it could ship instead. Returning
+ /// early let a file's tail settle `Ok` while its earlier chunk was still
+ /// on the wire, so a later failure of that chunk went unread.
+ #[test]
+ fn a_parked_worker_returns_only_after_its_own_chunk_lands() {
+ use std::sync::Condvar;
+ use std::time::{Duration, Instant};
+
+ type Gate = (Mutex<bool>, Condvar);
+ fn hold(gate: &Gate) {
+ let mut open = gate.0.lock().unwrap();
+ while !*open {
+ open = gate.1.wait(open).unwrap();
+ }
+ }
+ fn open(gate: &Gate) {
+ *gate.0.lock().unwrap() = true;
+ gate.1.notify_all();
+ }
+ let leak_gate =
+ || -> &'static Gate { Box::leak(Box::new((Mutex::new(false), Condvar::new()))) };
+ // Seven bodies of `x` and one of `z` fill the wire; the parked
+ // workers' shared body (it carries `a`) is held on its own gate.
+ let x_gate = leak_gate();
+ let z_gate = leak_gate();
+ let a_gate = leak_gate();
+ let on_wire: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));
+ let rejected: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));
+ let pipe = &*Box::leak(Box::new(BulkPipe::new(
+ move |body: Vec<u8>| {
+ on_wire.fetch_add(1, Ordering::SeqCst);
+ if body.contains(&b'a') {
+ hold(a_gate);
+ } else if body.first() == Some(&b'x') {
+ hold(x_gate);
+ } else if body.first() == Some(&b'z') {
+ hold(z_gate);
+ }
+ Ok(outcome())
+ },
+ 64,
+ 5000,
+ rejected,
+ 10,
+ )));
+ let wait_until = |what: &str, ok: &dyn Fn() -> bool| {
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while !ok() {
+ assert!(Instant::now() < deadline, "timed out waiting for {what}");
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ };
+
+ for (fill, byte) in [(super::MAX_IN_FLIGHT - 1, b'x'), (1, b'z')] {
+ let busy = pipe.lock().in_flight + fill;
+ for _ in 0..fill {
+ let t = pipe.token();
+ std::thread::spawn(move || pipe.submit(t, vec![byte; 200], 1));
+ }
+ wait_until("the wire to fill", &|| pipe.lock().in_flight == busy);
+ }
+ assert_eq!(pipe.lock().in_flight, super::MAX_IN_FLIGHT);
+
+ let (tx, rx) = std::sync::mpsc::channel();
+ let mut tokens = Vec::new();
+ for (parked, byte) in [(1, b'a'), (2, b'b')] {
+ let t = pipe.token();
+ tokens.push(t);
+ let tx = tx.clone();
+ std::thread::spawn(move || {
+ pipe.submit(t, vec![byte; 200], 1);
+ tx.send(t).unwrap();
+ });
+ wait_until("a worker to park on the full wire", &|| {
+ pipe.lock().parked_workers == parked
+ });
+ }
+
+ // One slot frees: one parked worker ships both chunks, held on the
+ // wire; the other parks again with its chunk in flight.
+ open(z_gate);
+ wait_until("the shared body to go out", &|| {
+ on_wire.load(Ordering::SeqCst) == super::MAX_IN_FLIGHT as u64 + 1
+ });
+ wait_until("the other worker to park again", &|| {
+ pipe.lock().parked_workers == 1
+ });
+
+ // Another file's sub-cut chunk waits in the buffer, then the rest of
+ // the wire frees up.
+ let c = pipe.token();
+ tokens.push(c);
+ pipe.submit(c, vec![b'c'; 10], 1);
+ open(x_gate);
+ wait_until("the x bodies to land", &|| pipe.lock().in_flight == 1);
+ std::thread::sleep(Duration::from_millis(200));
+ assert!(
+ rx.try_recv().is_err(),
+ "a worker returned while its chunk was still on the wire"
+ );
+
+ open(a_gate);
+ for _ in 0..2 {
+ rx.recv_timeout(Duration::from_secs(10))
+ .expect("both workers return once their chunk has landed");
+ }
+ pipe.retire();
+ for t in tokens {
+ pipe.settle(t).expect("accepted");
+ }
+ }
}
#[cfg(test)]