AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(#1023): linearize matching_ids_sorted against flush publications
xerj-orgcec66fa0b48b
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 1cd47bf48..d1dc4656d 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -26395,8 +26395,11 @@ async fn run_delete_by_query(// (252 pages on the 252k-doc measurement corpus at the default// `scroll_size`), which dominated the wall time once per-page `_source`// hydration was gone. `None` — any other query shape, or a segment- // without a complete id index — runs the paged arm unchanged.- if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query) {+ // without a complete id index — runs the paged arm unchanged. The+ // collection is admission-bracketed against concurrent flush+ // publications (#1023), so a purge storm cannot collect an empty set out+ // of another call's drain→publish window and report a clean no-op.+ if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query).await {// ES `max_docs`: stop after N documents; `total` then reports the// processed count (the same rule the paged arm applies).let matched = ids.len() as u64;@@ -26848,9 +26851,14 @@ async fn run_update_by_query(// per-segment `_id` maps (#950), then transformed per id in// `scroll_size` batches — one O(N) id materialisation, no// per-page stored-section re-scan, no `_source` hydration outside- // the per-id transform.+ // the per-id transform. The collection is admission-bracketed+ // against concurrent flush publications (#1023): without the+ // bracket, sixteen concurrent runners each flushed, the one+ // flush carrying the doc held the drain→publish window open,+ // and the other fifteen collected EMPTY match sets and+ // reported total:0/updated:0/failures:[] — a silent no-op.if script.is_some() {- if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query) {+ if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query).await {// ES `max_docs`: stop after N documents; `total` then// reports the processed count (the same rule the paged// arm applies).@@ -27256,6 +27264,58 @@ mod scripted_update_publication_tests {assert_eq!(idx.get_document("counter").await.unwrap().unwrap()["n"], 16);}++ /// #1023 delete twin: `run_delete_by_query` flushes then collects the+ /// match set through the SAME `matching_ids_sorted` enumeration, so it+ /// shares the update arm's hole — pre-fix, a concurrent purge storm+ /// collected empty sets out of another runner's drain→publish window and+ /// reported clean no-ops. No failures may be swallowed, the doc must be+ /// purged, and at least one runner must report it (a runner collecting+ /// after another's delete legitimately reports `total:0` — ES's+ /// start-of-run collection semantics; the deterministic lost-work proof+ /// is the update twin above).+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_delete_by_query_purges_the_document() {+ let state = test_state();+ let idx = state+ .engine+ .get_or_create_index("query-purge-counter")+ .unwrap();+ idx.index_document(Some("counter".into()), json!({"n": 0}))+ .await+ .unwrap();++ let start = Arc::new(tokio::sync::Barrier::new(17));+ let mut tasks = Vec::new();+ for _ in 0..16 {+ let idx = Arc::clone(&idx);+ let start = Arc::clone(&start);+ tasks.push(tokio::spawn(async move {+ start.wait().await;+ run_delete_by_query(+ &idx,+ &json!({ "match_all": {} }),+ None,+ DEFAULT_BY_QUERY_SCROLL_SIZE as usize,+ )+ .await+ }));+ }+ start.wait().await;+ let mut seen: u64 = 0;+ for task in tasks {+ let out = task.await.unwrap();+ assert_eq!(out["failures"], json!([]), "no swallowed purge errors");+ seen += out["total"].as_u64().unwrap_or(0);+ }+ assert!(seen >= 1, "at least one runner must have matched the doc");++ assert_eq!(+ idx.get_document("counter").await.unwrap(),+ None,+ "the doc must actually be purged, not just reported"+ );+ }}// ─────────────────────────────────────────────────────────────────────────────diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex c5b8567b7..07d936f6d 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -17819,7 +17819,25 @@ impl Index {/// which answers every query. Liveness and byte-order match that loop/// exactly: a missing version-map entry counts as live, a deleted entry/// is skipped, and ids sort as byte strings (the `_id` sort key's order).- pub fn matching_ids_sorted(&self, query: &QueryNode) -> Option<Vec<String>> {+ ///+ /// Linearized against concurrent publications (the #1023 CI regression):+ /// the enumeration reads only the snapshot's segments, so without reader+ /// admission a doc sitting in a flush's drain→publish window — drained+ /// out of the memtable, not yet in a published segment — was invisible,+ /// and sixteen concurrent `update_by_query` calls collected FIFTEEN empty+ /// match sets and reported `total:0 / updated:0 / failures:[]` while one+ /// lone increment landed (`n == 1`, expected 16). The enumeration now+ /// takes the same `CollectionPublication` reader-admission bracket+ /// `get_document` uses (acquire token, capture, validate, retry): the+ /// flush writer bracket provably spans drain→publish (`do_flush_shard`+ /// begins the guard before the memtable drain and commits it in the+ /// finalize worker after the snapshot publish), so a validated+ /// enumeration cannot land inside that window. On `Poisoned` the answer+ /// is `None` — the caller's paged arm runs `search()`, which surfaces the+ /// poison as a real error instead of a silent no-op.+ pub async fn matching_ids_sorted(&self, query: &QueryNode) -> Option<Vec<String>> {+ use crate::collection_publication::ReadAdmission;+// Membership resolved against a set: an `ids` query can carry tens of// thousands of values, and a per-doc linear scan would reintroduce// exactly the O(N × k) blowup this method exists to remove.@@ -17828,29 +17846,60 @@ impl Index {QueryNode::Ids { values } => Some(values.iter().map(String::as_str).collect()),_ => return None,};- let snap = self.store.snapshot();- let mut ids: Vec<String> =- Vec::with_capacity(snap.segments.iter().map(|m| m.doc_count as usize).sum());- for meta in snap.segments.iter() {- let map = self.id_pos_map_for(meta.id.as_str(), meta.doc_count)?;- for id in map.keys() {- if let Some(set) = &values {- if !set.contains(id.as_str()) {- continue;- }+ // `get_document`'s admission shape exactly: wait out any active+ // publication bracket (flush, merge, or write), enumerate, and retry+ // the whole enumeration if a bracket opened while we read. Retrying+ // is safe — the enumeration has no side effects.+ let ids = loop {+ let token = loop {+ let notified = self.collection_publication.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();+ match self.collection_publication.try_admit_reader() {+ ReadAdmission::Admitted(token) => break token,+ ReadAdmission::Poisoned => return None,+ ReadAdmission::WriterActive => notified.await,}- if let Some(ver) = self.store.version_map.get(id) {- if ver.deleted {- continue;+ };+ let snap = self.store.snapshot();+ let mut ids: Vec<String> =+ Vec::with_capacity(snap.segments.iter().map(|m| m.doc_count as usize).sum());+ let mut unsupported = false;+ for meta in snap.segments.iter() {+ let map = match self.id_pos_map_for(meta.id.as_str(), meta.doc_count) {+ Some(map) => map,+ None => {+ unsupported = true;+ break;+ }+ };+ for id in map.keys() {+ if let Some(set) = &values {+ if !set.contains(id.as_str()) {+ continue;+ }}+ if let Some(ver) = self.store.version_map.get(id) {+ if ver.deleted {+ continue;+ }+ }+ ids.push(id.clone());}- ids.push(id.clone());}- }- ids.sort_unstable();- // Same contract as the paged loop's `seen_ids`: an id resolves to one- // delete even if a legacy multi-segment layout lists it twice.- ids.dedup();+ if unsupported {+ return None;+ }+ ids.sort_unstable();+ // Same contract as the paged loop's `seen_ids`: an id resolves to one+ // delete even if a legacy multi-segment layout lists it twice.+ ids.dedup();+ match self.collection_publication.validate_reader(token) {+ ReadAdmission::Admitted(_) => break ids,+ ReadAdmission::Poisoned => return None,+ ReadAdmission::WriterActive => continue,+ }+ };Some(ids)}@@ -17880,7 +17929,11 @@ impl Index {/// scan arms apply (version-map tombstone skip, superseded-copy skip, one/// id per walk, memtable copy preferred over a stale segment copy).///- /// Consistency under a concurrent publication: the segment snapshot holds+ /// Consistency under a concurrent publication: the memtable capture and+ /// the segment snapshot are taken inside ONE `CollectionPublication`+ /// reader bracket, validated before the walk starts (#1023 — previously+ /// a doc drained from the memtable after the capture and unpublished at+ /// the snapshot was silently missed). The segment snapshot then holds/// its read lease for the whole walk (retired segment files persist), so/// a mid-walk flush or merge can neither orphan a byte range nor feed an/// id twice — a drained-then-flushed doc is either still in this@@ -17910,13 +17963,60 @@ impl Index {let needs_id_injection = query_needs_id_injection(&resolved_query);let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();+ // ── Capture bracket (#1023): both captures under ONE admission ──────+ //+ // The memtable capture and the segment snapshot were previously taken+ // apart, and the gap between them was the narrow variant of the hole+ // `matching_ids_sorted`'s admission loop closes: a doc drained out of+ // the memtable after the capture but unpublished at the snapshot sat+ // in neither and was silently missed. Both captures are now taken+ // inside one `CollectionPublication` reader bracket and validated+ // BEFORE the walk starts — a drain happens only inside a publication+ // bracket, and a flush's publish completes before its bracket drops,+ // so a validated pair cannot straddle a drain→publish window. On+ // `WriterActive` only the captures are retried (they are cheap);+ // `f` has side effects, so the walk itself is never re-run.+ let ghost_filter = self.store.version_map.ghost_events() > 0;+ let (mem_docs, snap) = {+ use crate::collection_publication::ReadAdmission;++ loop {+ let token = loop {+ let notified = self.collection_publication.notified();+ tokio::pin!(notified);+ notified.as_mut().enable();+ match self.collection_publication.try_admit_reader() {+ ReadAdmission::Admitted(token) => break Some(token),+ // Poisoned: nothing consistent is left to wait for —+ // every collection-wide reader on this index errors+ // from here on, and the fold has no error channel, so+ // walk the captures best-effort (the pre-fix+ // behaviour) rather than silently folding nothing.+ ReadAdmission::Poisoned => break None,+ ReadAdmission::WriterActive => notified.await,+ }+ };+ let mem_docs = self.memtable.all_docs_with_seq_arc();+ let snap = self.store.snapshot();+ let raced = token.is_some_and(|token| {+ matches!(+ self.collection_publication.validate_reader(token),+ ReadAdmission::WriterActive+ )+ });+ if raced {+ continue;+ }+ break (mem_docs, snap);+ }+ };+// ── Memtable: one Arc-shared snapshot, matched per doc ──────────────// Deletes remove from the memtable, so `docs` is live by construction;// the ghost-window check (below) only guards the window where the// live copy of an id moved to a segment between snapshot and now —// the segment walk covers that id instead.- let ghost_filter = self.store.version_map.ghost_events() > 0;- for (_seq_no, doc_id, source) in self.memtable.all_docs_with_seq_arc() {+ for (_seq_no, doc_id, source) in mem_docs {if ghost_filter {if let Some(ver) = self.store.version_map.get(&doc_id) {if ver.deleted@@ -17954,8 +18054,10 @@ impl Index {}// ── Segments: one stored-section walk per segment, one doc parsed- // at a time. The scan arms' brace-walk, reduced to its bones.- let snap = self.store.snapshot();+ // at a time. The scan arms' brace-walk, reduced to its bones. The+ // snapshot is the admission bracket's capture above — NOT a fresh+ // one — so the walk reads exactly the segment set the bracket+ // validated.for meta in snap.segments.iter() {let seg_id = meta.id.clone();// Same cache discipline as the unsorted search path: reuse the@@ -18170,7 +18272,7 @@ impl Index {// over unflushed docs could skip or duplicate ids) — the same// precondition `reindex` establishes.self.flush().await?;- if let Some(ids) = self.matching_ids_sorted(&query) {+ if let Some(ids) = self.matching_ids_sorted(&query).await {let total = ids.len() as u64;let mut deleted = 0u64;for chunk in ids.chunks(DELETE_BY_QUERY_PAGE) {diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 7ee3054ae..2850b3b16 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -2684,6 +2684,101 @@ async fn test_delete_by_query_purges_past_ten_thousand() {assert_eq!(result.total.value, 0, "index must be empty after the purge");}+/// #1023 (CI regression on PR #1023's branch): `matching_ids_sorted`+/// enumerated only `store.snapshot().segments` with no `CollectionPublication`+/// reader admission, so while another caller's flush held its drain→publish+/// window open — the doc drained out of the memtable but not yet in a+/// published segment — the id was invisible and the collector answered `[]`.+/// Sixteen concurrent `_update_by_query` calls each flushed first; the one+/// flush carrying the doc held the window, the other fifteen collected EMPTY+/// match sets and silently no-opped (`total:0/updated:0/failures:[]`), so the+/// final assertion read `n == 1` where 16 was expected. This is the+/// engine-level shape of that race: every task flushes then collects over a+/// one-doc index, and EVERY collector must see the doc — not just the task+/// whose flush happened to carry it. Fails before the fix with the losers+/// collecting `[]`.+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn test_matching_ids_sorted_linearized_against_concurrent_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ engine.create_index("match-race", Schema::empty()).unwrap();+ let idx = engine.get_index("match-race").unwrap();++ idx.index_document(Some("counter".into()), json!({"n": 0}))+ .await+ .unwrap();++ const TASKS: usize = 8;+ let start = std::sync::Arc::new(tokio::sync::Barrier::new(TASKS));+ let mut tasks = Vec::new();+ for _ in 0..TASKS {+ let idx = std::sync::Arc::clone(&idx);+ let start = std::sync::Arc::clone(&start);+ tasks.push(tokio::spawn(async move {+ start.wait().await;+ // Same precondition the by-query runners establish: flush, then+ // collect. The loser flushes race the winner's drain.+ idx.flush().await.unwrap();+ idx.matching_ids_sorted(&QueryNode::MatchAll).await+ }));+ }+ for (i, task) in tasks.into_iter().enumerate() {+ let ids = task.await.unwrap().expect("match_all is a supported shape");+ assert_eq!(+ ids,+ vec!["counter".to_string()],+ "collector {i} must see the pre-existing doc, not just the task \+ whose flush carried it"+ );+ }+}++/// #1023: the flush-vs-collector race above, answered through the engine's+/// own `delete_by_query` (flush + collect + delete). Guards that the+/// admission change cannot make a concurrent purge storm deadlock, error, or+/// lose the document: no failures, the doc is purged, and at least one+/// runner reports it. (The deterministic lost-work proof is the update+/// twin — a runner that collects after another's delete legitimately reports+/// `total:0`, ES's start-of-run collection semantics.)+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]+async fn test_concurrent_delete_by_query_purges_the_document() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ engine.create_index("dbq-race", Schema::empty()).unwrap();+ let idx = engine.get_index("dbq-race").unwrap();++ idx.index_document(Some("counter".into()), json!({"n": 0}))+ .await+ .unwrap();++ const TASKS: usize = 8;+ let start = std::sync::Arc::new(tokio::sync::Barrier::new(TASKS));+ let mut tasks = Vec::new();+ for _ in 0..TASKS {+ let idx = std::sync::Arc::clone(&idx);+ let start = std::sync::Arc::clone(&start);+ tasks.push(tokio::spawn(async move {+ start.wait().await;+ idx.delete_by_query(QueryNode::MatchAll).await+ }));+ }+ let mut seen: u64 = 0;+ for task in tasks {+ let (total, deleted) = task.await.unwrap().unwrap();+ assert_eq!(deleted, total, "no partial purges");+ seen += total;+ }+ assert!(seen >= 1, "at least one runner must have matched the doc");++ assert_eq!(+ idx.get_document("counter").await.unwrap(),+ None,+ "the doc must actually be purged"+ );+}+// ── 10. multi_match query ─────────────────────────────────────────────────────#[tokio::test]