AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): return from a parked bulk submit once another worker shipped its bytes (#2841) (#2863)
Jake Elbirt284cf78e0060
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 02f100256..22edf2e2a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -101,6 +101,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `xerj autoindex` no longer stalls near the end of a run at high+ `--workers`. When all 8 bulk slots were busy, a worker whose chunk crossed+ the flush size parked; if another worker then shipped the shared buffer,+ that chunk included, the first one woke to an empty buffer and parked again+ every 30 s for good, so the run never finished. It now returns once the+ bulk carrying its chunk has landed. The same loop could also return while+ that bulk was still on the wire, so a file could be journaled complete+ before, or even though, one of its bulks failed; it now waits. On a+ 26-file, 28 MB scientific CSV folder, rc.100 stalled at 21 of 26 files with+ `--workers 32` and 19 of 26 with `--workers 16`; both now finish all 26+ files and 179,374 records (#2841).+- `weighted_avg` now executes field-based requests (`value` / `weight`, eachwith a `field` and optional numeric `missing`) instead of returning HTTP 200with an `unsupported aggregation type` error in the aggregations body. Adiff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex a82ccf681..42330c25d 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -3498,9 +3498,10 @@ struct PipeState {/// Workers that may still call [`BulkPipe::submit`]. Decremented by/// [`BulkPipe::retire`] as each worker drains its queue.live_workers: usize,- /// Live workers currently parked inside `settle` (or waiting for an- /// in-flight slot in `submit`). `live == parked` is the "no more bytes- /// are coming" signal that flushes a residual buffer.+ /// Live workers currently parked inside `settle`, or inside `submit`+ /// waiting for an in-flight slot or for the bulk carrying their chunk to+ /// land (that worker submits again once it does). `live == parked` is+ /// the "no more bytes are coming" signal that flushes a residual buffer.parked_workers: usize,/// Bulk bodies currently on the wire. Capped at [`MAX_IN_FLIGHT`].in_flight: usize,@@ -3543,10 +3544,11 @@ const MAX_IN_FLIGHT: usize = 8;/// The pipe lock is NOT held across the wire: a flush detaches the buffer as/// a job, drops the lock, sends, and resolves tokens on return, so up to/// [`MAX_IN_FLIGHT`] bulks are on the wire at once. Per-file order still-/// holds — a file's cut chunks are sent synchronously by its own worker-/// inside `submit`, so any later job carrying its tail ships after those-/// completed; files interleave arbitrarily, exactly as the 32 parallel-/// per-file buffers always did.+/// holds: a file's cut chunk has landed before its worker's `submit`+/// returns (sent by that worker, or by another that took the buffer while+/// it was parked on a full wire), so any later job carrying its tail ships+/// after those completed; files interleave arbitrarily, exactly as the 32+/// parallel per-file buffers always did.////// Durability semantics are the per-file code's, unchanged: a token settles/// `Ok` only after EVERY bulk carrying its records was accepted (server_errors@@ -3632,24 +3634,15 @@ whereif st.buf.len() < self.bulk_cut && st.docs < self.doc_cap {return;}- // Cut crossed: ship inline. If every wire slot is busy, park until a- // delivery frees one — backpressure, exactly like the per-file code- // blocking inside its own bulk send.- let job = loop {- if let Some(job) = self.take_job(&mut st) {- break job;- }- st.parked_workers += 1;- let (guard, _) = self- .ready- .wait_timeout(st, std::time::Duration::from_secs(30))- .unwrap_or_else(|p| p.into_inner());- st = guard;- st.parked_workers -= 1;- };- drop(st);- let outcome = (self.send)(job.0);- self.deliver(job.1, job.2, outcome);+ // Cut crossed: ship inline, and return only once these bytes have+ // landed. If every wire slot is busy, park until a delivery frees+ // one: backpressure, exactly like the per-file code blocking inside+ // its own bulk send. If another worker ships the buffer meanwhile,+ // these bytes with it, that job was this worker's flush. Waiting for+ // a job of its own instead parked on an empty buffer for good+ // (#2841), and shipping someone else's bytes in its place returned+ // while this worker's own chunk was still on the wire.+ self.quiesce_locked(st, token);}/// The server's reason when this token failed only on records over the@@ -3659,11 +3652,18 @@ where}/// Block until no bulk holding this file's records is buffered or in- /// flight. A failed token settles at its first refusal, while its later- /// chunks may still be on the wire; a purge before they land would leave- /// a junk file half searchable.+ /// flight, shipping the buffer while they are still in it. `submit`+ /// flushes through this. A failed token settles at its first refusal,+ /// while its later chunks may still be on the wire; a purge before they+ /// land would leave a junk file half searchable.fn quiesce(&self, token: u64) {- let mut st = self.lock();+ self.quiesce_locked(self.lock(), token);+ }++ /// [`Self::quiesce`] under a lock the caller already holds, so `submit`+ /// ships the buffer it just crossed the cut with before any other+ /// worker can add to it.+ fn quiesce_locked<'s>(&'s self, mut st: std::sync::MutexGuard<'s, PipeState>, token: u64) {loop {let buffered = st.pending.contains(&token);if !buffered && !st.in_flight_tokens.contains_key(&token) {@@ -17273,6 +17273,202 @@ mod bulk_pipe_tests {"{WORKERS}x{FILES_PER_WORKER} files left as {n} bodies — no coalescing happened");}++ /// #2841: with every wire slot busy, two workers cross the cut and park.+ /// When a slot frees, the first one awake ships the whole buffer, the+ /// other's bytes with it. The other wakes to an empty buffer, so it can+ /// never take a job of its own; it must still return once its bytes were+ /// accepted. It used to park again, every 30 s, for good, and the run+ /// stalled at 16 and 32 workers on a 26-file corpus.+ #[test]+ fn a_worker_whose_bytes_another_worker_shipped_returns() {+ use std::sync::Condvar;+ use std::time::{Duration, Instant};++ // 'static, so a worker that never returns fails this test instead+ // of hanging it inside a thread scope.+ let gate: &'static (Mutex<bool>, Condvar) =+ Box::leak(Box::new((Mutex::new(false), Condvar::new())));+ let on_wire: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));+ let rejected: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));+ let pipe = &*Box::leak(Box::new(BulkPipe::new(+ move |_body| {+ on_wire.fetch_add(1, Ordering::SeqCst);+ let (open, cv) = gate;+ let mut open = open.lock().unwrap();+ while !*open {+ open = cv.wait(open).unwrap();+ }+ Ok(outcome())+ },+ 64,+ 5000,+ rejected,+ 10,+ )));+ let wait_until = |what: &str, ok: &dyn Fn() -> bool| {+ let deadline = Instant::now() + Duration::from_secs(10);+ while !ok() {+ assert!(Instant::now() < deadline, "timed out waiting for {what}");+ std::thread::sleep(Duration::from_millis(5));+ }+ };++ // Fill every wire slot with a body the gate holds.+ for _ in 0..super::MAX_IN_FLIGHT {+ let t = pipe.token();+ std::thread::spawn(move || pipe.submit(t, vec![b'x'; 200], 1));+ }+ wait_until("every wire slot to be busy", &|| {+ on_wire.load(Ordering::SeqCst) == super::MAX_IN_FLIGHT as u64+ });++ let (tx, rx) = std::sync::mpsc::channel();+ let mut tokens = Vec::new();+ for parked in 1..=2 {+ let t = pipe.token();+ tokens.push(t);+ let tx = tx.clone();+ std::thread::spawn(move || {+ pipe.submit(t, vec![b'y'; 200], 1);+ tx.send(t).unwrap();+ });+ wait_until("a worker to park on the full wire", &|| {+ pipe.lock().parked_workers == parked+ });+ }++ let (open, cv) = gate;+ *open.lock().unwrap() = true;+ cv.notify_all();++ for _ in 0..2 {+ rx.recv_timeout(Duration::from_secs(10))+ .expect("both parked workers return once their bytes were shipped");+ }+ for t in tokens {+ pipe.settle(t).expect("accepted");+ }+ assert_eq!(+ on_wire.load(Ordering::SeqCst),+ super::MAX_IN_FLIGHT as u64 + 1,+ "the two parked workers' bytes rode one body"+ );+ }++ /// A parked worker whose chunk another worker shipped must not return+ /// until that chunk has landed, even when the wire frees up and the+ /// buffer holds someone else's bytes it could ship instead. Returning+ /// early let a file's tail settle `Ok` while its earlier chunk was still+ /// on the wire, so a later failure of that chunk went unread.+ #[test]+ fn a_parked_worker_returns_only_after_its_own_chunk_lands() {+ use std::sync::Condvar;+ use std::time::{Duration, Instant};++ type Gate = (Mutex<bool>, Condvar);+ fn hold(gate: &Gate) {+ let mut open = gate.0.lock().unwrap();+ while !*open {+ open = gate.1.wait(open).unwrap();+ }+ }+ fn open(gate: &Gate) {+ *gate.0.lock().unwrap() = true;+ gate.1.notify_all();+ }+ let leak_gate =+ || -> &'static Gate { Box::leak(Box::new((Mutex::new(false), Condvar::new()))) };+ // Seven bodies of `x` and one of `z` fill the wire; the parked+ // workers' shared body (it carries `a`) is held on its own gate.+ let x_gate = leak_gate();+ let z_gate = leak_gate();+ let a_gate = leak_gate();+ let on_wire: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));+ let rejected: &'static AtomicU64 = Box::leak(Box::new(AtomicU64::new(0)));+ let pipe = &*Box::leak(Box::new(BulkPipe::new(+ move |body: Vec<u8>| {+ on_wire.fetch_add(1, Ordering::SeqCst);+ if body.contains(&b'a') {+ hold(a_gate);+ } else if body.first() == Some(&b'x') {+ hold(x_gate);+ } else if body.first() == Some(&b'z') {+ hold(z_gate);+ }+ Ok(outcome())+ },+ 64,+ 5000,+ rejected,+ 10,+ )));+ let wait_until = |what: &str, ok: &dyn Fn() -> bool| {+ let deadline = Instant::now() + Duration::from_secs(10);+ while !ok() {+ assert!(Instant::now() < deadline, "timed out waiting for {what}");+ std::thread::sleep(Duration::from_millis(5));+ }+ };++ for (fill, byte) in [(super::MAX_IN_FLIGHT - 1, b'x'), (1, b'z')] {+ let busy = pipe.lock().in_flight + fill;+ for _ in 0..fill {+ let t = pipe.token();+ std::thread::spawn(move || pipe.submit(t, vec![byte; 200], 1));+ }+ wait_until("the wire to fill", &|| pipe.lock().in_flight == busy);+ }+ assert_eq!(pipe.lock().in_flight, super::MAX_IN_FLIGHT);++ let (tx, rx) = std::sync::mpsc::channel();+ let mut tokens = Vec::new();+ for (parked, byte) in [(1, b'a'), (2, b'b')] {+ let t = pipe.token();+ tokens.push(t);+ let tx = tx.clone();+ std::thread::spawn(move || {+ pipe.submit(t, vec![byte; 200], 1);+ tx.send(t).unwrap();+ });+ wait_until("a worker to park on the full wire", &|| {+ pipe.lock().parked_workers == parked+ });+ }++ // One slot frees: one parked worker ships both chunks, held on the+ // wire; the other parks again with its chunk in flight.+ open(z_gate);+ wait_until("the shared body to go out", &|| {+ on_wire.load(Ordering::SeqCst) == super::MAX_IN_FLIGHT as u64 + 1+ });+ wait_until("the other worker to park again", &|| {+ pipe.lock().parked_workers == 1+ });++ // Another file's sub-cut chunk waits in the buffer, then the rest of+ // the wire frees up.+ let c = pipe.token();+ tokens.push(c);+ pipe.submit(c, vec![b'c'; 10], 1);+ open(x_gate);+ wait_until("the x bodies to land", &|| pipe.lock().in_flight == 1);+ std::thread::sleep(Duration::from_millis(200));+ assert!(+ rx.try_recv().is_err(),+ "a worker returned while its chunk was still on the wire"+ );++ open(a_gate);+ for _ in 0..2 {+ rx.recv_timeout(Duration::from_secs(10))+ .expect("both workers return once their chunk has landed");+ }+ pipe.retire();+ for t in tokens {+ pipe.settle(t).expect("accepted");+ }+ }}#[cfg(test)]