AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api): delete a one-page _delete_by_query match set without a flush (#2782)
xerj-orgd4a53fa2ed6c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7a04d35d8..3020fa116 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -82,6 +82,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0separate 48 B heap allocation, other ids up to 24 bytes are stored inline,and the tombstone flag shares the version word. On the 2.46 M-entry perf-gatecorpus the map drops from about 347 MB to 268 MB.+- A `_delete_by_query` whose matches fit in one scroll page no longer flushes+ the whole memtable before deleting. `autoindex --fresh` sends one such+ delete-before-replace per file, and each one used to cut a new tiny segment+ out of every other worker's pending bulks (about 1,050 segments on the+ #2640 corpus). Larger deletes keep the flush-then-page path (#2640).- A search no longer hangs forever when a cold segment's stored fields arebeing warmed in the background. A search-pool thread that waited for thewarm's decode could block the decode it waited for, so every later searchdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 01ca88c84..b6d2715bb 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -32721,6 +32721,76 @@ async fn by_query_matches_anything(Ok(results.total.value > 0)}+/// What [`by_query_single_page`] found.+enum SinglePage {+ /// Nothing matches: the #1222 no-op.+ Empty,+ /// Every match, in one page. `total` was exact and equal to the page.+ Complete(Vec<String>),+ /// More than one page, or a total that is not exact: the flush-first+ /// paged runner owns it.+ Larger,+}++/// #2640: the #1222 probe, widened to return the match set when it fits in+/// one page.+///+/// The by-query runner flushes before paging because `_id`-keyset paging is+/// only correct over on-disk segments (#1019). A selector whose whole match+/// set comes back in ONE page with an exact `hits.total` does not page, so+/// that precondition has nothing to protect. And the flush cost is not the+/// selector's to pay: it drains the WHOLE memtable, every concurrent writer's+/// fresh documents included, into a new segment. `autoindex --fresh` fires+/// one delete-before-replace per file while its workers bulk the+/// replacements, so each matching delete cut a tiny segment out of the other+/// workers' writes. 1,050 of them on the #2640 corpus, about 19 s of the+/// 26.7 s run after #2703.+///+/// Same search path and same strictness as `by_query_matches_anything`:+/// memtable documents count, an unanswered probe propagates, and a script+/// limit refuses the run.+async fn by_query_single_page(+ idx: &std::sync::Arc<xerj_engine::Index>,+ query: &Value,+ page: usize,+) -> Result<SinglePage, Value> {+ let mut probe = match xerj_query::parse_request(&json!({+ "query": query,+ "size": page,+ "track_total_hits": true,+ "_source": false,+ })) {+ Ok(r) => r,+ Err(e) => {+ return Err(+ ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())).into_value(),+ )+ }+ };+ probe.ids_only = true;+ let results = match idx.search(&probe).await {+ Ok(r) => r,+ Err(e) => return Err(ApiError::new(xerj_common::XerjError::from(e)).into_value()),+ };+ if let Some(reason) = &results.script_failure {+ return Err(script_limit_error_value(reason));+ }+ if results.total.value == 0 {+ return Ok(SinglePage::Empty);+ }+ let exact = results.total.relation == xerj_query::executor::TotalHitsRelation::Eq;+ if !exact || results.hits.len() as u64 != results.total.value {+ return Ok(SinglePage::Larger);+ }+ let mut ids: Vec<String> = results.hits.into_iter().map(|h| h.id).collect();+ ids.sort_unstable();+ ids.dedup();+ if ids.len() as u64 != results.total.value {+ return Ok(SinglePage::Larger);+ }+ Ok(SinglePage::Complete(ids))+}+fn record_deleted(deleted_ids: &Option<std::sync::Arc<std::sync::Mutex<Vec<String>>>>, id: &str) {let Some(ids) = deleted_ids else {return;@@ -32791,9 +32861,57 @@ pub(crate) async fn run_delete_by_query_recorded(// #1222: no match, no flush (see `by_query_matches_anything`). The zero// response mirrors what both arms below already return for a no-match// run over a flushed index: `batches: 0` with `total: 0`.- match by_query_matches_anything(idx, query).await {- Ok(true) => {}- Ok(false) => {+ //+ // #2640: a small match set, no flush either (see+ // `by_query_single_page`). The probe reads the whole match set in one+ // exact page through the ordinary search path, memtable included, so+ // there is nothing to page over and the flush precondition has no job.+ let single_page = if max_docs.is_none() {+ by_query_single_page(idx, query, scroll_size).await+ } else {+ by_query_matches_anything(idx, query).await.map(|any| {+ if any {+ SinglePage::Larger+ } else {+ SinglePage::Empty+ }+ })+ };+ match single_page {+ Ok(SinglePage::Larger) => {}+ Ok(SinglePage::Complete(ids)) => {+ let total = ids.len() as u64;+ let mut deleted: u64 = 0;+ let mut failures: Vec<Value> = Vec::new();+ for id in &ids {+ match idx.delete_document(id).await {+ Ok(_) => {+ deleted += 1;+ record_deleted(&deleted_ids, id);+ }+ Err(e) => {+ failures.push(json!({+ "id": id,+ "cause": { "reason": e.to_string() },+ }));+ }+ }+ }+ return json!({+ "took": started.elapsed().as_millis() as u64,+ "timed_out": false,+ "total": total,+ "deleted": deleted,+ "batches": 1,+ "version_conflicts": 0,+ "noops": 0,+ "failures": failures,+ "throttled_millis": 0,+ "requests_per_second": -1,+ "throttled_until_millis": 0,+ });+ }+ Ok(SinglePage::Empty) => {return json!({"took": started.elapsed().as_millis() as u64,"timed_out": false,@@ -34016,11 +34134,10 @@ mod scripted_update_publication_tests {}/// The #1222 gate must not weaken the real run: a matching- /// `_delete_by_query` still flushes, purges, and reports ES totals —- /// the unflushed matching doc is visible to the gate's probe (ordinary- /// search path), so it takes the flush arm exactly as before.+ /// `_delete_by_query` still purges and reports ES totals — the unflushed+ /// matching doc is visible to the gate's probe (ordinary search path).#[tokio::test]- async fn matching_delete_by_query_still_flushes_and_purges() {+ async fn matching_delete_by_query_still_purges() {let state = test_state();let idx = state.engine@@ -34054,6 +34171,81 @@ mod scripted_update_publication_tests {"the non-matching doc must survive");}++ /// #2640: a match set that fits in one page is deleted without the+ /// flush. `autoindex --fresh` sends one delete-before-replace per file+ /// while other workers bulk; pre-fix every matching delete drained THEIR+ /// unflushed writes into a new segment (one per file). The match set+ /// here spans a segment and the memtable, and both copies must go, while+ /// the bystander's unflushed write stays in the memtable.+ #[tokio::test]+ async fn single_page_delete_by_query_purges_without_flushing() {+ let state = test_state();+ let idx = state+ .engine+ .get_or_create_index("byquery-single-page")+ .unwrap();+ idx.index_document(Some("old".into()), json!({"file": "a"}))+ .await+ .unwrap();+ idx.flush().await.unwrap();+ idx.index_document(Some("new".into()), json!({"file": "a"}))+ .await+ .unwrap();+ idx.index_document(Some("bystander".into()), json!({"file": "b"}))+ .await+ .unwrap();+ assert!(idx.memtable_bytes() > 0, "precondition: writes unflushed");++ let out = run_delete_by_query(+ &idx,+ &json!({ "term": { "file": "a" } }),+ None,+ DEFAULT_BY_QUERY_SCROLL_SIZE as usize,+ )+ .await;+ assert_eq!(out["failures"], json!([]));+ assert_eq!(out["total"], json!(2));+ assert_eq!(out["deleted"], json!(2));+ assert_eq!(out["batches"], json!(1));+ assert_eq!(idx.get_document("old").await.unwrap(), None);+ assert_eq!(idx.get_document("new").await.unwrap(), None);+ assert!(idx.get_document("bystander").await.unwrap().is_some());+ assert!(+ idx.memtable_bytes() > 0,+ "a one-page delete must not drain other writers' memtable \+ documents into a segment (#2640)"+ );+ }++ /// The #2640 fast path covers ONE complete page only. A match set larger+ /// than `scroll_size` still takes the flush-first paged runner and is+ /// removed in full.+ #[tokio::test]+ async fn multi_page_delete_by_query_still_purges_everything() {+ let state = test_state();+ let idx = state+ .engine+ .get_or_create_index("byquery-multi-page")+ .unwrap();+ for i in 0..5 {+ idx.index_document(Some(format!("m{i}")), json!({"file": "a"}))+ .await+ .unwrap();+ }+ idx.index_document(Some("keep".into()), json!({"file": "b"}))+ .await+ .unwrap();++ let out = run_delete_by_query(&idx, &json!({ "term": { "file": "a" } }), None, 2).await;+ assert_eq!(out["failures"], json!([]));+ assert_eq!(out["total"], json!(5));+ assert_eq!(out["deleted"], json!(5));+ for i in 0..5 {+ assert_eq!(idx.get_document(&format!("m{i}")).await.unwrap(), None);+ }+ assert!(idx.get_document("keep").await.unwrap().is_some());+ }}// ─────────────────────────────────────────────────────────────────────────────