Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): heartbeat while an extract slot is full (#1657)

xerj-org1e682dcc1b5a
diff --git a/engine/crates/xerj-autoindex/src/extract/mbox.rs b/engine/crates/xerj-autoindex/src/extract/mbox.rs
index 905fadfb3..f03496c4a 100644
--- a/engine/crates/xerj-autoindex/src/extract/mbox.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mbox.rs
@@ -525,10 +525,15 @@ impl Gate {
}
fn acquire(&self) -> GatePermit<'_> {
- let mut state = self.state.lock().expect("mbox gate poisoned");
- while state.0 >= state.1 {
- state = self.ready.wait(state).expect("mbox gate poisoned");
- }
+ let mut state = super::wait_until_slot(
+ "mbox parse",
+ &self.state,
+ &self.ready,
+ super::SLOT_HEARTBEAT,
+ |state| state.0 >= state.1,
+ |state| (state.0, state.1),
+ |line| eprintln!("{line}"),
+ );
state.0 += 1;
GatePermit(self)
}
@@ -564,10 +569,15 @@ impl Budget {
}
fn acquire(&self, bytes: usize) {
- let mut used = self.used.lock().expect("mbox budget poisoned");
- while *used > 0 && used.saturating_add(bytes) > self.cap {
- used = self.freed.wait(used).expect("mbox budget poisoned");
- }
+ let mut used = super::wait_until_slot(
+ "mbox byte budget",
+ &self.used,
+ &self.freed,
+ super::SLOT_HEARTBEAT,
+ |used| *used > 0 && used.saturating_add(bytes) > self.cap,
+ |used| (*used, self.cap),
+ |line| eprintln!("{line}"),
+ );
*used = used.saturating_add(bytes);
}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 7e3162691..7904d23c3 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -34,6 +34,8 @@ use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use std::io::{BufRead, BufReader, Read};
use std::path::Path;
+use std::sync::{Condvar, Mutex};
+use std::time::{Duration, Instant};
/// Where a record's FIELD NAMES came from — the input to dataset clustering.
///
@@ -814,6 +816,57 @@ pub(crate) fn emit_document_with_fields(
true
}
+/// How often a full extract slot says it is still waiting. The wait is not
+/// failed: a legal PDF or mailbox parse can hold its slot for a long time.
+/// The line exists so a leaked permit is not a silent futex.
+pub(crate) const SLOT_HEARTBEAT: Duration = Duration::from_secs(15);
+
+pub(crate) fn slot_wait_line(kind: &str, in_use: usize, limit: usize, waited: Duration) -> String {
+ format!(
+ "autoindex: still waiting for a {kind} slot (in_use={in_use} limit={limit}) after {}s",
+ waited.as_secs()
+ )
+}
+
+/// Block until `full` is false. Each wait is sliced so a notify is rechecked
+/// and a stuck slot prints [`slot_wait_line`] on `heartbeat` cadence. The
+/// caller adds the permit after this returns; this function does not.
+pub(crate) fn wait_until_slot<'a, T>(
+ kind: &str,
+ lock: &'a Mutex<T>,
+ ready: &Condvar,
+ heartbeat: Duration,
+ full: impl Fn(&T) -> bool,
+ counts: impl Fn(&T) -> (usize, usize),
+ mut note: impl FnMut(&str),
+) -> std::sync::MutexGuard<'a, T> {
+ let heartbeat = heartbeat.max(Duration::from_millis(1));
+ let mut guard = lock.lock().expect("extract slot lock poisoned");
+ let started = Instant::now();
+ let mut next_note = started + heartbeat;
+ while full(&guard) {
+ let now = Instant::now();
+ if now >= next_note {
+ let (in_use, limit) = counts(&guard);
+ note(&slot_wait_line(
+ kind,
+ in_use,
+ limit,
+ now.duration_since(started),
+ ));
+ next_note = now + heartbeat;
+ }
+ let slice = next_note
+ .saturating_duration_since(Instant::now())
+ .min(Duration::from_secs(5));
+ let (next, _) = ready
+ .wait_timeout(guard, slice)
+ .expect("extract slot lock poisoned");
+ guard = next;
+ }
+ guard
+}
+
#[cfg(test)]
mod section_tests {
use super::*;
@@ -1234,3 +1287,65 @@ mod section_tests {
assert!(stats2.truncated, "N>cap must set truncated");
}
}
+
+#[cfg(test)]
+mod slot_tests {
+ use super::*;
+ use std::sync::Arc;
+ use std::thread;
+ use std::time::Duration;
+
+ #[test]
+ fn slot_wait_line_names_the_slot_and_how_long() {
+ let line = slot_wait_line("pdf worker", 4, 4, Duration::from_secs(45));
+ assert_eq!(
+ line,
+ "autoindex: still waiting for a pdf worker slot (in_use=4 limit=4) after 45s"
+ );
+ }
+
+ /// A full slot used to park on `Condvar::wait` with no timeout and no
+ /// line. The waiter must say so, then proceed when the holder releases.
+ #[test]
+ fn a_full_extract_slot_heartbeats_and_then_proceeds() {
+ let lock = Arc::new(Mutex::new((1usize, 1usize)));
+ let ready = Arc::new(Condvar::new());
+ let notes = Arc::new(Mutex::new(Vec::new()));
+ let lock2 = Arc::clone(&lock);
+ let ready2 = Arc::clone(&ready);
+ let notes2 = Arc::clone(&notes);
+ let waiter = thread::spawn(move || {
+ let guard = wait_until_slot(
+ "pdf worker",
+ &lock2,
+ &ready2,
+ Duration::from_millis(20),
+ |state| state.0 >= state.1,
+ |state| (state.0, state.1),
+ |line| notes2.lock().expect("notes").push(line.to_string()),
+ );
+ guard.0
+ });
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while notes.lock().expect("notes").is_empty() {
+ assert!(
+ Instant::now() < deadline,
+ "a full slot must heartbeat before the holder releases it"
+ );
+ thread::sleep(Duration::from_millis(10));
+ }
+ {
+ let mut guard = lock.lock().expect("lock");
+ guard.0 = 0;
+ ready.notify_all();
+ }
+ assert_eq!(waiter.join().expect("waiter"), 0);
+ let lines = notes.lock().expect("notes");
+ assert!(
+ lines[0].contains("pdf worker")
+ && lines[0].contains("in_use=1")
+ && lines[0].contains("limit=1"),
+ "{lines:?}"
+ );
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/pdf.rs b/engine/crates/xerj-autoindex/src/extract/pdf.rs
index 271e1c358..10b717f27 100644
--- a/engine/crates/xerj-autoindex/src/extract/pdf.rs
+++ b/engine/crates/xerj-autoindex/src/extract/pdf.rs
@@ -1404,10 +1404,15 @@ impl WorkerGate {
}
fn acquire(&self) -> WorkerPermit<'_> {
- let mut state = self.state.lock().expect("PDF worker gate poisoned");
- while state.0 >= state.1 {
- state = self.ready.wait(state).expect("PDF worker gate poisoned");
- }
+ let mut state = super::wait_until_slot(
+ "pdf worker",
+ &self.state,
+ &self.ready,
+ super::SLOT_HEARTBEAT,
+ |state| state.0 >= state.1,
+ |state| (state.0, state.1),
+ |line| eprintln!("{line}"),
+ );
state.0 += 1;
WorkerPermit(self)
}