Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api): delete a one-page _delete_by_query match set without a flush (#2782)

xerj-orgd4a53fa2ed6c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7a04d35d8..3020fa116 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -82,6 +82,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
separate 48 B heap allocation, other ids up to 24 bytes are stored inline,
and the tombstone flag shares the version word. On the 2.46 M-entry perf-gate
corpus the map drops from about 347 MB to 268 MB.
+- A `_delete_by_query` whose matches fit in one scroll page no longer flushes
+ the whole memtable before deleting. `autoindex --fresh` sends one such
+ delete-before-replace per file, and each one used to cut a new tiny segment
+ out of every other worker's pending bulks (about 1,050 segments on the
+ #2640 corpus). Larger deletes keep the flush-then-page path (#2640).
- A search no longer hangs forever when a cold segment's stored fields are
being warmed in the background. A search-pool thread that waited for the
warm's decode could block the decode it waited for, so every later search
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 01ca88c84..b6d2715bb 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -32721,6 +32721,76 @@ async fn by_query_matches_anything(
Ok(results.total.value > 0)
}
+/// What [`by_query_single_page`] found.
+enum SinglePage {
+ /// Nothing matches: the #1222 no-op.
+ Empty,
+ /// Every match, in one page. `total` was exact and equal to the page.
+ Complete(Vec<String>),
+ /// More than one page, or a total that is not exact: the flush-first
+ /// paged runner owns it.
+ Larger,
+}
+
+/// #2640: the #1222 probe, widened to return the match set when it fits in
+/// one page.
+///
+/// The by-query runner flushes before paging because `_id`-keyset paging is
+/// only correct over on-disk segments (#1019). A selector whose whole match
+/// set comes back in ONE page with an exact `hits.total` does not page, so
+/// that precondition has nothing to protect. And the flush cost is not the
+/// selector's to pay: it drains the WHOLE memtable, every concurrent writer's
+/// fresh documents included, into a new segment. `autoindex --fresh` fires
+/// one delete-before-replace per file while its workers bulk the
+/// replacements, so each matching delete cut a tiny segment out of the other
+/// workers' writes. 1,050 of them on the #2640 corpus, about 19 s of the
+/// 26.7 s run after #2703.
+///
+/// Same search path and same strictness as `by_query_matches_anything`:
+/// memtable documents count, an unanswered probe propagates, and a script
+/// limit refuses the run.
+async fn by_query_single_page(
+ idx: &std::sync::Arc<xerj_engine::Index>,
+ query: &Value,
+ page: usize,
+) -> Result<SinglePage, Value> {
+ let mut probe = match xerj_query::parse_request(&json!({
+ "query": query,
+ "size": page,
+ "track_total_hits": true,
+ "_source": false,
+ })) {
+ Ok(r) => r,
+ Err(e) => {
+ return Err(
+ ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())).into_value(),
+ )
+ }
+ };
+ probe.ids_only = true;
+ let results = match idx.search(&probe).await {
+ Ok(r) => r,
+ Err(e) => return Err(ApiError::new(xerj_common::XerjError::from(e)).into_value()),
+ };
+ if let Some(reason) = &results.script_failure {
+ return Err(script_limit_error_value(reason));
+ }
+ if results.total.value == 0 {
+ return Ok(SinglePage::Empty);
+ }
+ let exact = results.total.relation == xerj_query::executor::TotalHitsRelation::Eq;
+ if !exact || results.hits.len() as u64 != results.total.value {
+ return Ok(SinglePage::Larger);
+ }
+ let mut ids: Vec<String> = results.hits.into_iter().map(|h| h.id).collect();
+ ids.sort_unstable();
+ ids.dedup();
+ if ids.len() as u64 != results.total.value {
+ return Ok(SinglePage::Larger);
+ }
+ Ok(SinglePage::Complete(ids))
+}
+
fn record_deleted(deleted_ids: &Option<std::sync::Arc<std::sync::Mutex<Vec<String>>>>, id: &str) {
let Some(ids) = deleted_ids else {
return;
@@ -32791,9 +32861,57 @@ pub(crate) async fn run_delete_by_query_recorded(
// #1222: no match, no flush (see `by_query_matches_anything`). The zero
// response mirrors what both arms below already return for a no-match
// run over a flushed index: `batches: 0` with `total: 0`.
- match by_query_matches_anything(idx, query).await {
- Ok(true) => {}
- Ok(false) => {
+ //
+ // #2640: a small match set, no flush either (see
+ // `by_query_single_page`). The probe reads the whole match set in one
+ // exact page through the ordinary search path, memtable included, so
+ // there is nothing to page over and the flush precondition has no job.
+ let single_page = if max_docs.is_none() {
+ by_query_single_page(idx, query, scroll_size).await
+ } else {
+ by_query_matches_anything(idx, query).await.map(|any| {
+ if any {
+ SinglePage::Larger
+ } else {
+ SinglePage::Empty
+ }
+ })
+ };
+ match single_page {
+ Ok(SinglePage::Larger) => {}
+ Ok(SinglePage::Complete(ids)) => {
+ let total = ids.len() as u64;
+ let mut deleted: u64 = 0;
+ let mut failures: Vec<Value> = Vec::new();
+ for id in &ids {
+ match idx.delete_document(id).await {
+ Ok(_) => {
+ deleted += 1;
+ record_deleted(&deleted_ids, id);
+ }
+ Err(e) => {
+ failures.push(json!({
+ "id": id,
+ "cause": { "reason": e.to_string() },
+ }));
+ }
+ }
+ }
+ return json!({
+ "took": started.elapsed().as_millis() as u64,
+ "timed_out": false,
+ "total": total,
+ "deleted": deleted,
+ "batches": 1,
+ "version_conflicts": 0,
+ "noops": 0,
+ "failures": failures,
+ "throttled_millis": 0,
+ "requests_per_second": -1,
+ "throttled_until_millis": 0,
+ });
+ }
+ Ok(SinglePage::Empty) => {
return json!({
"took": started.elapsed().as_millis() as u64,
"timed_out": false,
@@ -34016,11 +34134,10 @@ mod scripted_update_publication_tests {
}
/// The #1222 gate must not weaken the real run: a matching
- /// `_delete_by_query` still flushes, purges, and reports ES totals —
- /// the unflushed matching doc is visible to the gate's probe (ordinary
- /// search path), so it takes the flush arm exactly as before.
+ /// `_delete_by_query` still purges and reports ES totals — the unflushed
+ /// matching doc is visible to the gate's probe (ordinary search path).
#[tokio::test]
- async fn matching_delete_by_query_still_flushes_and_purges() {
+ async fn matching_delete_by_query_still_purges() {
let state = test_state();
let idx = state
.engine
@@ -34054,6 +34171,81 @@ mod scripted_update_publication_tests {
"the non-matching doc must survive"
);
}
+
+ /// #2640: a match set that fits in one page is deleted without the
+ /// flush. `autoindex --fresh` sends one delete-before-replace per file
+ /// while other workers bulk; pre-fix every matching delete drained THEIR
+ /// unflushed writes into a new segment (one per file). The match set
+ /// here spans a segment and the memtable, and both copies must go, while
+ /// the bystander's unflushed write stays in the memtable.
+ #[tokio::test]
+ async fn single_page_delete_by_query_purges_without_flushing() {
+ let state = test_state();
+ let idx = state
+ .engine
+ .get_or_create_index("byquery-single-page")
+ .unwrap();
+ idx.index_document(Some("old".into()), json!({"file": "a"}))
+ .await
+ .unwrap();
+ idx.flush().await.unwrap();
+ idx.index_document(Some("new".into()), json!({"file": "a"}))
+ .await
+ .unwrap();
+ idx.index_document(Some("bystander".into()), json!({"file": "b"}))
+ .await
+ .unwrap();
+ assert!(idx.memtable_bytes() > 0, "precondition: writes unflushed");
+
+ let out = run_delete_by_query(
+ &idx,
+ &json!({ "term": { "file": "a" } }),
+ None,
+ DEFAULT_BY_QUERY_SCROLL_SIZE as usize,
+ )
+ .await;
+ assert_eq!(out["failures"], json!([]));
+ assert_eq!(out["total"], json!(2));
+ assert_eq!(out["deleted"], json!(2));
+ assert_eq!(out["batches"], json!(1));
+ assert_eq!(idx.get_document("old").await.unwrap(), None);
+ assert_eq!(idx.get_document("new").await.unwrap(), None);
+ assert!(idx.get_document("bystander").await.unwrap().is_some());
+ assert!(
+ idx.memtable_bytes() > 0,
+ "a one-page delete must not drain other writers' memtable \
+ documents into a segment (#2640)"
+ );
+ }
+
+ /// The #2640 fast path covers ONE complete page only. A match set larger
+ /// than `scroll_size` still takes the flush-first paged runner and is
+ /// removed in full.
+ #[tokio::test]
+ async fn multi_page_delete_by_query_still_purges_everything() {
+ let state = test_state();
+ let idx = state
+ .engine
+ .get_or_create_index("byquery-multi-page")
+ .unwrap();
+ for i in 0..5 {
+ idx.index_document(Some(format!("m{i}")), json!({"file": "a"}))
+ .await
+ .unwrap();
+ }
+ idx.index_document(Some("keep".into()), json!({"file": "b"}))
+ .await
+ .unwrap();
+
+ let out = run_delete_by_query(&idx, &json!({ "term": { "file": "a" } }), None, 2).await;
+ assert_eq!(out["failures"], json!([]));
+ assert_eq!(out["total"], json!(5));
+ assert_eq!(out["deleted"], json!(5));
+ for i in 0..5 {
+ assert_eq!(idx.get_document(&format!("m{i}")).await.unwrap(), None);
+ }
+ assert!(idx.get_document("keep").await.unwrap().is_some());
+ }
}
// ─────────────────────────────────────────────────────────────────────────────