Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(#1023): linearize matching_ids_sorted against flush publications

xerj-orgcec66fa0b48b
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 1cd47bf48..d1dc4656d 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -26395,8 +26395,11 @@ async fn run_delete_by_query(
// (252 pages on the 252k-doc measurement corpus at the default
// `scroll_size`), which dominated the wall time once per-page `_source`
// hydration was gone. `None` — any other query shape, or a segment
- // without a complete id index — runs the paged arm unchanged.
- if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query) {
+ // without a complete id index — runs the paged arm unchanged. The
+ // collection is admission-bracketed against concurrent flush
+ // publications (#1023), so a purge storm cannot collect an empty set out
+ // of another call's drain→publish window and report a clean no-op.
+ if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query).await {
// ES `max_docs`: stop after N documents; `total` then reports the
// processed count (the same rule the paged arm applies).
let matched = ids.len() as u64;
@@ -26848,9 +26851,14 @@ async fn run_update_by_query(
// per-segment `_id` maps (#950), then transformed per id in
// `scroll_size` batches — one O(N) id materialisation, no
// per-page stored-section re-scan, no `_source` hydration outside
- // the per-id transform.
+ // the per-id transform. The collection is admission-bracketed
+ // against concurrent flush publications (#1023): without the
+ // bracket, sixteen concurrent runners each flushed, the one
+ // flush carrying the doc held the drain→publish window open,
+ // and the other fifteen collected EMPTY match sets and
+ // reported total:0/updated:0/failures:[] — a silent no-op.
if script.is_some() {
- if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query) {
+ if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query).await {
// ES `max_docs`: stop after N documents; `total` then
// reports the processed count (the same rule the paged
// arm applies).
@@ -27256,6 +27264,58 @@ mod scripted_update_publication_tests {
assert_eq!(idx.get_document("counter").await.unwrap().unwrap()["n"], 16);
}
+
+ /// #1023 delete twin: `run_delete_by_query` flushes then collects the
+ /// match set through the SAME `matching_ids_sorted` enumeration, so it
+ /// shares the update arm's hole — pre-fix, a concurrent purge storm
+ /// collected empty sets out of another runner's drain→publish window and
+ /// reported clean no-ops. No failures may be swallowed, the doc must be
+ /// purged, and at least one runner must report it (a runner collecting
+ /// after another's delete legitimately reports `total:0` — ES's
+ /// start-of-run collection semantics; the deterministic lost-work proof
+ /// is the update twin above).
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn concurrent_delete_by_query_purges_the_document() {
+ let state = test_state();
+ let idx = state
+ .engine
+ .get_or_create_index("query-purge-counter")
+ .unwrap();
+ idx.index_document(Some("counter".into()), json!({"n": 0}))
+ .await
+ .unwrap();
+
+ let start = Arc::new(tokio::sync::Barrier::new(17));
+ let mut tasks = Vec::new();
+ for _ in 0..16 {
+ let idx = Arc::clone(&idx);
+ let start = Arc::clone(&start);
+ tasks.push(tokio::spawn(async move {
+ start.wait().await;
+ run_delete_by_query(
+ &idx,
+ &json!({ "match_all": {} }),
+ None,
+ DEFAULT_BY_QUERY_SCROLL_SIZE as usize,
+ )
+ .await
+ }));
+ }
+ start.wait().await;
+ let mut seen: u64 = 0;
+ for task in tasks {
+ let out = task.await.unwrap();
+ assert_eq!(out["failures"], json!([]), "no swallowed purge errors");
+ seen += out["total"].as_u64().unwrap_or(0);
+ }
+ assert!(seen >= 1, "at least one runner must have matched the doc");
+
+ assert_eq!(
+ idx.get_document("counter").await.unwrap(),
+ None,
+ "the doc must actually be purged, not just reported"
+ );
+ }
}
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index c5b8567b7..07d936f6d 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -17819,7 +17819,25 @@ impl Index {
/// which answers every query. Liveness and byte-order match that loop
/// exactly: a missing version-map entry counts as live, a deleted entry
/// is skipped, and ids sort as byte strings (the `_id` sort key's order).
- pub fn matching_ids_sorted(&self, query: &QueryNode) -> Option<Vec<String>> {
+ ///
+ /// Linearized against concurrent publications (the #1023 CI regression):
+ /// the enumeration reads only the snapshot's segments, so without reader
+ /// admission a doc sitting in a flush's drain→publish window — drained
+ /// out of the memtable, not yet in a published segment — was invisible,
+ /// and sixteen concurrent `update_by_query` calls collected FIFTEEN empty
+ /// match sets and reported `total:0 / updated:0 / failures:[]` while one
+ /// lone increment landed (`n == 1`, expected 16). The enumeration now
+ /// takes the same `CollectionPublication` reader-admission bracket
+ /// `get_document` uses (acquire token, capture, validate, retry): the
+ /// flush writer bracket provably spans drain→publish (`do_flush_shard`
+ /// begins the guard before the memtable drain and commits it in the
+ /// finalize worker after the snapshot publish), so a validated
+ /// enumeration cannot land inside that window. On `Poisoned` the answer
+ /// is `None` — the caller's paged arm runs `search()`, which surfaces the
+ /// poison as a real error instead of a silent no-op.
+ pub async fn matching_ids_sorted(&self, query: &QueryNode) -> Option<Vec<String>> {
+ use crate::collection_publication::ReadAdmission;
+
// Membership resolved against a set: an `ids` query can carry tens of
// thousands of values, and a per-doc linear scan would reintroduce
// exactly the O(N × k) blowup this method exists to remove.
@@ -17828,29 +17846,60 @@ impl Index {
QueryNode::Ids { values } => Some(values.iter().map(String::as_str).collect()),
_ => return None,
};
- let snap = self.store.snapshot();
- let mut ids: Vec<String> =
- Vec::with_capacity(snap.segments.iter().map(|m| m.doc_count as usize).sum());
- for meta in snap.segments.iter() {
- let map = self.id_pos_map_for(meta.id.as_str(), meta.doc_count)?;
- for id in map.keys() {
- if let Some(set) = &values {
- if !set.contains(id.as_str()) {
- continue;
- }
+ // `get_document`'s admission shape exactly: wait out any active
+ // publication bracket (flush, merge, or write), enumerate, and retry
+ // the whole enumeration if a bracket opened while we read. Retrying
+ // is safe — the enumeration has no side effects.
+ let ids = loop {
+ let token = loop {
+ let notified = self.collection_publication.notified();
+ tokio::pin!(notified);
+ notified.as_mut().enable();
+ match self.collection_publication.try_admit_reader() {
+ ReadAdmission::Admitted(token) => break token,
+ ReadAdmission::Poisoned => return None,
+ ReadAdmission::WriterActive => notified.await,
}
- if let Some(ver) = self.store.version_map.get(id) {
- if ver.deleted {
- continue;
+ };
+ let snap = self.store.snapshot();
+ let mut ids: Vec<String> =
+ Vec::with_capacity(snap.segments.iter().map(|m| m.doc_count as usize).sum());
+ let mut unsupported = false;
+ for meta in snap.segments.iter() {
+ let map = match self.id_pos_map_for(meta.id.as_str(), meta.doc_count) {
+ Some(map) => map,
+ None => {
+ unsupported = true;
+ break;
+ }
+ };
+ for id in map.keys() {
+ if let Some(set) = &values {
+ if !set.contains(id.as_str()) {
+ continue;
+ }
}
+ if let Some(ver) = self.store.version_map.get(id) {
+ if ver.deleted {
+ continue;
+ }
+ }
+ ids.push(id.clone());
}
- ids.push(id.clone());
}
- }
- ids.sort_unstable();
- // Same contract as the paged loop's `seen_ids`: an id resolves to one
- // delete even if a legacy multi-segment layout lists it twice.
- ids.dedup();
+ if unsupported {
+ return None;
+ }
+ ids.sort_unstable();
+ // Same contract as the paged loop's `seen_ids`: an id resolves to one
+ // delete even if a legacy multi-segment layout lists it twice.
+ ids.dedup();
+ match self.collection_publication.validate_reader(token) {
+ ReadAdmission::Admitted(_) => break ids,
+ ReadAdmission::Poisoned => return None,
+ ReadAdmission::WriterActive => continue,
+ }
+ };
Some(ids)
}
@@ -17880,7 +17929,11 @@ impl Index {
/// scan arms apply (version-map tombstone skip, superseded-copy skip, one
/// id per walk, memtable copy preferred over a stale segment copy).
///
- /// Consistency under a concurrent publication: the segment snapshot holds
+ /// Consistency under a concurrent publication: the memtable capture and
+ /// the segment snapshot are taken inside ONE `CollectionPublication`
+ /// reader bracket, validated before the walk starts (#1023 — previously
+ /// a doc drained from the memtable after the capture and unpublished at
+ /// the snapshot was silently missed). The segment snapshot then holds
/// its read lease for the whole walk (retired segment files persist), so
/// a mid-walk flush or merge can neither orphan a byte range nor feed an
/// id twice — a drained-then-flushed doc is either still in this
@@ -17910,13 +17963,60 @@ impl Index {
let needs_id_injection = query_needs_id_injection(&resolved_query);
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
+ // ── Capture bracket (#1023): both captures under ONE admission ──────
+ //
+ // The memtable capture and the segment snapshot were previously taken
+ // apart, and the gap between them was the narrow variant of the hole
+ // `matching_ids_sorted`'s admission loop closes: a doc drained out of
+ // the memtable after the capture but unpublished at the snapshot sat
+ // in neither and was silently missed. Both captures are now taken
+ // inside one `CollectionPublication` reader bracket and validated
+ // BEFORE the walk starts — a drain happens only inside a publication
+ // bracket, and a flush's publish completes before its bracket drops,
+ // so a validated pair cannot straddle a drain→publish window. On
+ // `WriterActive` only the captures are retried (they are cheap);
+ // `f` has side effects, so the walk itself is never re-run.
+ let ghost_filter = self.store.version_map.ghost_events() > 0;
+ let (mem_docs, snap) = {
+ use crate::collection_publication::ReadAdmission;
+
+ loop {
+ let token = loop {
+ let notified = self.collection_publication.notified();
+ tokio::pin!(notified);
+ notified.as_mut().enable();
+ match self.collection_publication.try_admit_reader() {
+ ReadAdmission::Admitted(token) => break Some(token),
+ // Poisoned: nothing consistent is left to wait for —
+ // every collection-wide reader on this index errors
+ // from here on, and the fold has no error channel, so
+ // walk the captures best-effort (the pre-fix
+ // behaviour) rather than silently folding nothing.
+ ReadAdmission::Poisoned => break None,
+ ReadAdmission::WriterActive => notified.await,
+ }
+ };
+ let mem_docs = self.memtable.all_docs_with_seq_arc();
+ let snap = self.store.snapshot();
+ let raced = token.is_some_and(|token| {
+ matches!(
+ self.collection_publication.validate_reader(token),
+ ReadAdmission::WriterActive
+ )
+ });
+ if raced {
+ continue;
+ }
+ break (mem_docs, snap);
+ }
+ };
+
// ── Memtable: one Arc-shared snapshot, matched per doc ──────────────
// Deletes remove from the memtable, so `docs` is live by construction;
// the ghost-window check (below) only guards the window where the
// live copy of an id moved to a segment between snapshot and now —
// the segment walk covers that id instead.
- let ghost_filter = self.store.version_map.ghost_events() > 0;
- for (_seq_no, doc_id, source) in self.memtable.all_docs_with_seq_arc() {
+ for (_seq_no, doc_id, source) in mem_docs {
if ghost_filter {
if let Some(ver) = self.store.version_map.get(&doc_id) {
if ver.deleted
@@ -17954,8 +18054,10 @@ impl Index {
}
// ── Segments: one stored-section walk per segment, one doc parsed
- // at a time. The scan arms' brace-walk, reduced to its bones.
- let snap = self.store.snapshot();
+ // at a time. The scan arms' brace-walk, reduced to its bones. The
+ // snapshot is the admission bracket's capture above — NOT a fresh
+ // one — so the walk reads exactly the segment set the bracket
+ // validated.
for meta in snap.segments.iter() {
let seg_id = meta.id.clone();
// Same cache discipline as the unsorted search path: reuse the
@@ -18170,7 +18272,7 @@ impl Index {
// over unflushed docs could skip or duplicate ids) — the same
// precondition `reindex` establishes.
self.flush().await?;
- if let Some(ids) = self.matching_ids_sorted(&query) {
+ if let Some(ids) = self.matching_ids_sorted(&query).await {
let total = ids.len() as u64;
let mut deleted = 0u64;
for chunk in ids.chunks(DELETE_BY_QUERY_PAGE) {
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 7ee3054ae..2850b3b16 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -2684,6 +2684,101 @@ async fn test_delete_by_query_purges_past_ten_thousand() {
assert_eq!(result.total.value, 0, "index must be empty after the purge");
}
+/// #1023 (CI regression on PR #1023's branch): `matching_ids_sorted`
+/// enumerated only `store.snapshot().segments` with no `CollectionPublication`
+/// reader admission, so while another caller's flush held its drain→publish
+/// window open — the doc drained out of the memtable but not yet in a
+/// published segment — the id was invisible and the collector answered `[]`.
+/// Sixteen concurrent `_update_by_query` calls each flushed first; the one
+/// flush carrying the doc held the window, the other fifteen collected EMPTY
+/// match sets and silently no-opped (`total:0/updated:0/failures:[]`), so the
+/// final assertion read `n == 1` where 16 was expected. This is the
+/// engine-level shape of that race: every task flushes then collects over a
+/// one-doc index, and EVERY collector must see the doc — not just the task
+/// whose flush happened to carry it. Fails before the fix with the losers
+/// collecting `[]`.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_matching_ids_sorted_linearized_against_concurrent_flush() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ engine.create_index("match-race", Schema::empty()).unwrap();
+ let idx = engine.get_index("match-race").unwrap();
+
+ idx.index_document(Some("counter".into()), json!({"n": 0}))
+ .await
+ .unwrap();
+
+ const TASKS: usize = 8;
+ let start = std::sync::Arc::new(tokio::sync::Barrier::new(TASKS));
+ let mut tasks = Vec::new();
+ for _ in 0..TASKS {
+ let idx = std::sync::Arc::clone(&idx);
+ let start = std::sync::Arc::clone(&start);
+ tasks.push(tokio::spawn(async move {
+ start.wait().await;
+ // Same precondition the by-query runners establish: flush, then
+ // collect. The loser flushes race the winner's drain.
+ idx.flush().await.unwrap();
+ idx.matching_ids_sorted(&QueryNode::MatchAll).await
+ }));
+ }
+ for (i, task) in tasks.into_iter().enumerate() {
+ let ids = task.await.unwrap().expect("match_all is a supported shape");
+ assert_eq!(
+ ids,
+ vec!["counter".to_string()],
+ "collector {i} must see the pre-existing doc, not just the task \
+ whose flush carried it"
+ );
+ }
+}
+
+/// #1023: the flush-vs-collector race above, answered through the engine's
+/// own `delete_by_query` (flush + collect + delete). Guards that the
+/// admission change cannot make a concurrent purge storm deadlock, error, or
+/// lose the document: no failures, the doc is purged, and at least one
+/// runner reports it. (The deterministic lost-work proof is the update
+/// twin — a runner that collects after another's delete legitimately reports
+/// `total:0`, ES's start-of-run collection semantics.)
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn test_concurrent_delete_by_query_purges_the_document() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ engine.create_index("dbq-race", Schema::empty()).unwrap();
+ let idx = engine.get_index("dbq-race").unwrap();
+
+ idx.index_document(Some("counter".into()), json!({"n": 0}))
+ .await
+ .unwrap();
+
+ const TASKS: usize = 8;
+ let start = std::sync::Arc::new(tokio::sync::Barrier::new(TASKS));
+ let mut tasks = Vec::new();
+ for _ in 0..TASKS {
+ let idx = std::sync::Arc::clone(&idx);
+ let start = std::sync::Arc::clone(&start);
+ tasks.push(tokio::spawn(async move {
+ start.wait().await;
+ idx.delete_by_query(QueryNode::MatchAll).await
+ }));
+ }
+ let mut seen: u64 = 0;
+ for task in tasks {
+ let (total, deleted) = task.await.unwrap().unwrap();
+ assert_eq!(deleted, total, "no partial purges");
+ seen += total;
+ }
+ assert!(seen >= 1, "at least one runner must have matched the doc");
+
+ assert_eq!(
+ idx.get_document("counter").await.unwrap(),
+ None,
+ "the doc must actually be purged"
+ );
+}
+
// ── 10. multi_match query ─────────────────────────────────────────────────────
#[tokio::test]