Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(#1019): _delete_by_query pages the whole match set, ids-only

xerj-org64c698ec5565
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 9bbe1d331..615cf888d 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -26072,6 +26072,17 @@ fn explain_query_node(q: &xerj_query::ast::QueryNode) -> (String, Vec<Value>) {
#[derive(Debug, Deserialize)]
pub struct DeleteByQueryBody {
pub query: Value,
+ /// ES `max_docs`: "The maximum number of documents to process. Defaults
+ /// to all documents." `None` = purge every match.
+ #[serde(default)]
+ pub max_docs: Option<u64>,
+ /// ES `scroll_size`: "The size of the scroll request that powers the
+ /// operation." Default 1 000, clamped to `1..=10_000` (the per-request
+ /// `max_result_window` ceiling — paging is keyset, so a small page never
+ /// hits the deep-paging wall). `max_docs <= scroll_size` naturally
+ /// degenerates to the documented single-batch fast path.
+ #[serde(default)]
+ pub scroll_size: Option<u64>,
}
/// Resolve a by-query selector to the concrete member indices it must touch and
@@ -26198,14 +26209,22 @@ pub async fn delete_by_query(
Err(e) => return ApiError::new(e).into_response(),
};
- // Run a match-all-sized search using the provided query.
- let search_body_val = json!({ "query": effective_query, "size": 10000, "from": 0 });
- let search_req = match xerj_query::parse_request(&search_body_val)
- .map_err(|e| xerj_common::XerjError::invalid_query(e.to_string()))
- {
- Ok(r) => r,
- Err(e) => return ApiError::new(e).into_response(),
- };
+ // Validate the (alias-filter-ANDed) query once, up front, so a malformed
+ // body is an immediate 400 before a task is registered or any member is
+ // touched — the runner below re-parses per batch as the cursor advances.
+ let scroll_size = body
+ .scroll_size
+ .unwrap_or(DEFAULT_BY_QUERY_SCROLL_SIZE)
+ .clamp(1, 10_000) as usize;
+ let validation_body = json!({
+ "query": effective_query,
+ "size": scroll_size,
+ "from": 0,
+ "sort": [{ "_id": "asc" }],
+ });
+ if let Err(e) = xerj_query::parse_request(&validation_body) {
+ return ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())).into_response();
+ }
// ES defaults `wait_for_completion` to true (synchronous); only an
// explicit `false` switches to the async `{"task": "node:id"}` form.
@@ -26219,6 +26238,8 @@ pub async fn delete_by_query(
let task_key = handle.key().to_string();
let spawned_key = task_key.clone();
let tasks = state.tasks.clone();
+ let task_query = effective_query.clone();
+ let task_max_docs = body.max_docs;
// Detached, so no `index_guard` rule — and none is needed: the task
// owns the already-authorized `Index` handles this request resolved,
// and an `Index` cannot name another index (it holds no `Engine`). See
@@ -26227,7 +26248,8 @@ pub async fn delete_by_query(
tokio::spawn(async move {
let mut results = Vec::with_capacity(indices.len());
for idx in &indices {
- results.push(run_delete_by_query(idx, &search_req).await);
+ results
+ .push(run_delete_by_query(idx, &task_query, task_max_docs, scroll_size).await);
}
tasks.complete(&spawned_key, aggregate_by_query_results(results, "deleted"));
drop(handle);
@@ -26237,7 +26259,7 @@ pub async fn delete_by_query(
let mut results = Vec::with_capacity(indices.len());
for idx in &indices {
- results.push(run_delete_by_query(idx, &search_req).await);
+ results.push(run_delete_by_query(idx, &effective_query, body.max_docs, scroll_size).await);
}
drop(handle);
by_query_response(aggregate_by_query_results(results, "deleted"))
@@ -26268,37 +26290,224 @@ fn by_query_response(body: Value) -> Response {
(status, Json(body)).into_response()
}
+/// ES `scroll_size` default for the by-query write paths ("The size of the
+/// scroll request that powers the operation").
+const DEFAULT_BY_QUERY_SCROLL_SIZE: u64 = 1_000;
+
+/// One member's paginated `_delete_by_query` run (#1019).
+///
+/// Shaped like the `_reindex` loop: flush the member first — `_id`-keyset
+/// paging is only correct over on-disk segments (the memtable's field-sort
+/// path does not order by `_id` reliably, so paging over unflushed docs
+/// could skip or duplicate ids), and a failed flush aborts the delete rather
+/// than under-deleting — then pull `scroll_size` pages sorted `_id: asc`
+/// with a `search_after` cursor until an empty page or `max_docs`, deleting
+/// per id. XERJ has no batch-delete API; per-id is the liveness-authoritative
+/// semantic everywhere else, and ES's per-batch bulk is an internal detail.
+///
+/// Every page runs with the INTERNAL ids-only projection
+/// (`SearchRequest::ids_only`, set here and never deserialisable from the
+/// wire): the runner reads nothing from the hits but `hit.id`, so the search
+/// phase skips `_source` materialisation entirely — previously the single
+/// `size: 10_000` shot hydrated 10 100 full `Value` trees just to read ids,
+/// the dominant RSS transient of a purge (the #950 follow-up headroom).
+///
+/// ES semantics matched: `total` is the exact match count when unlimited
+/// (the collection snapshot / first page's `hits.total`, taken before any
+/// deletion) or the processed count under `max_docs`; `batches` is the real
+/// batch count; `noops`/`version_conflicts` are always 0 for this runner (an
+/// unconditional per-id delete cannot conflict).
+///
+/// Two arms, same counters: `match_all` / `ids` selectors take the
+/// single-pass id-set collection (`Index::matching_ids_sorted`) and delete it
+/// in `scroll_size` batches; every other query shape takes the `_id`-keyset
+/// paged loop below, whose pages run with the ids-only projection.
async fn run_delete_by_query(
- idx: &xerj_engine::Index,
- search_req: &xerj_query::SearchRequest,
+ idx: &std::sync::Arc<xerj_engine::Index>,
+ query: &Value,
+ max_docs: Option<u64>,
+ scroll_size: usize,
) -> Value {
let started = Instant::now();
- let results = match idx.search(search_req).await {
- Ok(r) => r,
- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_value(),
- };
- // A script resource limit makes matching fail-closed, so the selection is
- // a subset of what the caller asked to delete. Deleting that subset and
- // reporting success is both destructive and wrong; refuse instead.
- if let Some(reason) = &results.script_failure {
- return script_limit_error_value(reason);
+ // Flush precondition (see doc comment). Propagated, not swallowed, for
+ // the same reason as `reindex`: the flush is a precondition of the
+ // paging being complete, so a failed flush must not be followed by a
+ // purge that reports a `total` over an index it could not read
+ // consistently.
+ if let Err(e) = idx.flush().await {
+ return ApiError::new(xerj_common::XerjError::internal(format!(
+ "delete_by_query was not attempted: flushing the index failed ({e}); the purge \
+ needs a flushed index to page over it without skipping documents"
+ )))
+ .into_value();
}
- let total = results.hits.len() as u64;
- let mut deleted = 0u64;
+ // Parse once for the fast-path eligibility check below. The handler
+ // already validated the (page-1-shaped) query, so this cannot fail for a
+ // request that got here; the paged arm re-parses per page only because it
+ // rebuilds the body to attach `search_after`.
+ let parsed_query = match xerj_query::parse_request(&json!({ "query": query })) {
+ Ok(r) => r.query,
+ Err(e) => {
+ return ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())).into_value()
+ }
+ };
+
+ // #1019 single-pass fast path: for `match_all` / `ids` selectors the
+ // whole live match set is collected once from the cached per-segment
+ // `_id` maps (#950), then deleted in `scroll_size` batches — one O(N)
+ // pass. The paged arm below re-scans every stored section once per page
+ // (252 pages on the 252k-doc measurement corpus at the default
+ // `scroll_size`), which dominated the wall time once per-page `_source`
+ // hydration was gone. `None` — any other query shape, or a segment
+ // without a complete id index — runs the paged arm unchanged.
+ if let Some(mut ids) = idx.matching_ids_sorted(&parsed_query) {
+ // ES `max_docs`: stop after N documents; `total` then reports the
+ // processed count (the same rule the paged arm applies).
+ let matched = ids.len() as u64;
+ if let Some(cap) = max_docs {
+ ids.truncate(cap as usize);
+ }
+ let mut processed: u64 = 0;
+ let mut deleted: u64 = 0;
+ let mut batches: u64 = 0;
+ let mut failures: Vec<Value> = Vec::new();
+ for chunk in ids.chunks(scroll_size) {
+ batches += 1;
+ for id in chunk {
+ processed += 1;
+ match idx.delete_document(id).await {
+ Ok(_) => deleted += 1,
+ Err(e) => {
+ failures.push(json!({
+ "id": id,
+ "cause": { "reason": e.to_string() },
+ }));
+ }
+ }
+ }
+ }
+ let total = if max_docs.is_none() {
+ // Never under-report if a concurrent writer extended the match
+ // set past the collection snapshot mid-purge.
+ matched.max(deleted)
+ } else {
+ processed
+ };
+ let took = started.elapsed().as_millis() as u64;
+ return json!({
+ "took": took,
+ "timed_out": false,
+ "total": total,
+ "deleted": deleted,
+ "batches": batches,
+ "version_conflicts": 0,
+ "noops": 0,
+ "failures": failures,
+ "throttled_millis": 0,
+ "requests_per_second": -1,
+ "throttled_until_millis": 0,
+ });
+ }
+
+ let mut total: u64 = 0;
+ let mut processed: u64 = 0;
+ let mut deleted: u64 = 0;
+ let mut batches: u64 = 0;
let mut failures: Vec<Value> = Vec::new();
+ // Keyset cursor: the last id of the previous page (the sole sort key is
+ // `_id: asc`). `None` on the first page.
+ let mut search_after: Option<String> = None;
+ // Safety backstop only, mirroring reindex's `max_total`: with keyset
+ // paging every page is a fresh top-N far below `max_result_window`, so
+ // this just bounds a runaway loop.
+ let max_total: u64 = 10_000_000;
- for hit in results.hits {
- match idx.delete_document(&hit.id).await {
- Ok(_) => deleted += 1,
- Err(e) => {
- failures.push(json!({
- "id": hit.id,
- "cause": { "reason": e.to_string() },
- }));
+ loop {
+ let page = match max_docs {
+ Some(cap) => (scroll_size as u64).min(cap.saturating_sub(processed)) as usize,
+ None => scroll_size,
+ };
+ if page == 0 {
+ break; // max_docs reached
+ }
+
+ let mut search_body_val = json!({
+ "query": query,
+ "size": page,
+ "sort": [{ "_id": "asc" }],
+ // Belt only — the ids-only projection below is the load-bearing
+ // `_source` cut (`_source: false` alone keeps raw sources for
+ // fields/highlight resolution).
+ "_source": false,
+ });
+ if let Some(ref cursor) = search_after {
+ search_body_val["search_after"] = json!([cursor]);
+ }
+ let mut search_req = match xerj_query::parse_request(&search_body_val)
+ .map_err(|e| xerj_common::XerjError::invalid_query(e.to_string()))
+ {
+ Ok(r) => r,
+ Err(e) => return ApiError::new(e).into_value(),
+ };
+ // Internal-only projection hint — `parse_request` can never set it.
+ search_req.ids_only = true;
+
+ let results = match idx.search(&search_req).await {
+ Ok(r) => r,
+ Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_value(),
+ };
+ // A script resource limit makes matching fail-closed, so the
+ // selection is a subset of what the caller asked to delete — checked
+ // PER BATCH, before a single id of the page is deleted. Deleting
+ // that subset and reporting success is both destructive and wrong;
+ // refuse instead.
+ if let Some(reason) = &results.script_failure {
+ return script_limit_error_value(reason);
+ }
+
+ if results.hits.is_empty() {
+ break;
+ }
+ batches += 1;
+ // Page 1's `hits.total` is the EXACT pre-delete match count (later
+ // pages run over an already-shrunk live set). Under `max_docs` ES
+ // reports the processed count instead.
+ if max_docs.is_none() && total == 0 {
+ total = results.total.value;
+ }
+
+ // Capture the cursor before the loop consumes `results.hits`.
+ let last_id = results.hits.last().map(|h| h.id.clone());
+ for hit in results.hits {
+ processed += 1;
+ match idx.delete_document(&hit.id).await {
+ Ok(_) => deleted += 1,
+ Err(e) => {
+ failures.push(json!({
+ "id": hit.id,
+ "cause": { "reason": e.to_string() },
+ }));
+ }
}
}
+
+ match last_id {
+ Some(id) => search_after = Some(id),
+ None => break,
+ }
+ if processed >= max_total {
+ break;
+ }
+ }
+
+ if max_docs.is_none() {
+ // Never under-report if a concurrent writer extended the match set
+ // past page 1's snapshot count mid-purge.
+ total = total.max(processed);
+ } else {
+ total = processed;
}
let took = started.elapsed().as_millis() as u64;
@@ -26307,7 +26516,7 @@ async fn run_delete_by_query(
"timed_out": false,
"total": total,
"deleted": deleted,
- "batches": 1,
+ "batches": batches,
"version_conflicts": 0,
"noops": 0,
"failures": failures,
diff --git a/engine/crates/xerj-api/tests/delete_by_query_purges_entire_index.rs b/engine/crates/xerj-api/tests/delete_by_query_purges_entire_index.rs
new file mode 100644
index 000000000..1f8071c42
--- /dev/null
+++ b/engine/crates/xerj-api/tests/delete_by_query_purges_entire_index.rs
@@ -0,0 +1,242 @@
+//! Issue #1019: `_delete_by_query` truncates at 10 000 documents per call.
+//!
+//! The handler built ONE search body — `{"query": …, "size": 10000, "from":
+//! 0}` — deleted whatever came back, and reported `total` as the page length
+//! with `batches` hardcoded to 1. An index with more than `max_result_window`
+//! matching documents was therefore silently under-deleted by a single call
+//! (the autoindex client works around it by looping up to 1 000 delete
+//! passes, each paying the full search again). ES semantics: delete-by-query
+//! has NO default document cap — it processes the entire matching result set
+//! in `scroll_size` batches (default 1 000) unless `max_docs` limits it, and
+//! `total` is the exact match count, not the truncated page length.
+//!
+//! These tests go through the real HTTP route. They fail before the fix at
+//! `total`/`deleted` == 10 000 (and `_count` == 15 000 afterwards).
+//!
+//! Elasticsearch is referenced for wire semantics only (approach-only per the
+//! licence rules); no ES code is reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+/// Seed `n` docs into `index` via `_bulk` (chunked so no single request body
+/// is enormous). Ids are zero-padded so lexicographic `_id` order is
+/// deterministic — the paginated runner pages by `_id` keyset.
+async fn seed_bulk(app: &axum::Router, index: &str, n: u64) {
+ let (st, _) = call(
+ app,
+ "PUT",
+ &format!("/{index}"),
+ json!({"mappings": {"properties": {"v": {"type": "long"}}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create {index}");
+
+ let width = n.to_string().len();
+ let chunk = 2_500u64;
+ let mut start = 0u64;
+ while start < n {
+ let end = (start + chunk).min(n);
+ let mut ndjson = String::new();
+ for i in start..end {
+ ndjson.push_str(&format!(
+ "{{\"index\":{{\"_id\":\"d{i:0width$}\"}}}}\n{{\"v\":{i}}}\n",
+ width = width
+ ));
+ }
+ let response = app
+ .clone()
+ .oneshot(
+ Request::post(format!("/{index}/_bulk"))
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(ndjson))
+ .unwrap(),
+ )
+ .await
+ .expect("bulk request");
+ assert_eq!(
+ response.status(),
+ StatusCode::OK,
+ "bulk chunk {start}..{end}"
+ );
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let body: Value = serde_json::from_slice(&bytes).expect("bulk body");
+ assert_eq!(
+ body["errors"],
+ json!(false),
+ "bulk chunk {start}..{end}: {body}"
+ );
+ start = end;
+ }
+ let (st, _) = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;
+ assert_eq!(st, StatusCode::OK, "refresh {index}");
+}
+
+async fn count(app: &axum::Router, index: &str) -> u64 {
+ let (st, body) = call(
+ app,
+ "POST",
+ &format!("/{index}/_count"),
+ json!({"query": {"match_all": {}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "_count {index}");
+ body["count"].as_u64().unwrap_or(u64::MAX)
+}
+
+/// One `_delete_by_query` over 25 000 matching docs must delete ALL of them —
+/// no 10k truncation, `total` the exact match count, `batches` the real page
+/// count. Fails before the fix at total/deleted == 10 000.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn delete_by_query_purges_past_ten_thousand_in_one_call() {
+ let (app, _dir) = app().await;
+ const N: u64 = 25_000;
+ seed_bulk(&app, "dbq-big", N).await;
+ assert_eq!(count(&app, "dbq-big").await, N, "seeded");
+
+ let (st, body) = call(
+ &app,
+ "POST",
+ "/dbq-big/_delete_by_query",
+ json!({"query": {"match_all": {}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+ assert_eq!(
+ body["total"],
+ json!(N),
+ "total must be the exact match count, not the page length: {body}"
+ );
+ assert_eq!(
+ body["deleted"],
+ json!(N),
+ "every matching doc deleted by ONE call: {body}"
+ );
+ let batches = body["batches"].as_u64().expect("batches");
+ assert!(batches >= 3, "a 25k purge at the default scroll_size of 1000 must pull real batches, got {batches}: {body}");
+ assert_eq!(body["failures"], json!([]), "{body}");
+
+ assert_eq!(
+ count(&app, "dbq-big").await,
+ 0,
+ "index must be empty after the purge"
+ );
+}
+
+/// ES `max_docs`: "The maximum number of documents to process. Defaults to
+/// all documents." Exactly `max_docs` documents are deleted, `total` reports
+/// the processed count, and a `max_docs <= scroll_size` run is a single
+/// batch (the documented no-scroll fast path).
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn delete_by_query_honors_max_docs() {
+ let (app, _dir) = app().await;
+ seed_bulk(&app, "dbq-max", 20).await;
+
+ let (st, body) = call(
+ &app,
+ "POST",
+ "/dbq-max/_delete_by_query",
+ json!({"query": {"match_all": {}}, "max_docs": 7}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+ assert_eq!(
+ body["total"],
+ json!(7),
+ "with max_docs set, total is the processed count: {body}"
+ );
+ assert_eq!(body["deleted"], json!(7), "{body}");
+ assert_eq!(
+ body["batches"],
+ json!(1),
+ "max_docs <= scroll_size is a single batch: {body}"
+ );
+
+ assert_eq!(count(&app, "dbq-max").await, 13, "exactly max_docs deleted");
+}
+
+/// ES `scroll_size` (default 1 000): the size of the batch that powers the
+/// operation — `batches` is the number of batches actually pulled.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn delete_by_query_reports_real_batch_count() {
+ let (app, _dir) = app().await;
+ seed_bulk(&app, "dbq-scroll", 10).await;
+
+ let (st, body) = call(
+ &app,
+ "POST",
+ "/dbq-scroll/_delete_by_query",
+ json!({"query": {"match_all": {}}, "scroll_size": 3}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+ assert_eq!(body["total"], json!(10), "{body}");
+ assert_eq!(body["deleted"], json!(10), "{body}");
+ assert_eq!(
+ body["batches"],
+ json!(4),
+ "10 docs at scroll_size 3 = 4 batches: {body}"
+ );
+
+ assert_eq!(count(&app, "dbq-scroll").await, 0);
+}
+
+/// A selective query must page exactly like match_all: every matching doc
+/// deleted, non-matching untouched, total the exact match count.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn delete_by_query_paginates_selective_queries() {
+ let (app, _dir) = app().await;
+ seed_bulk(&app, "dbq-sel", 30).await;
+
+ let (st, body) = call(
+ &app,
+ "POST",
+ "/dbq-sel/_delete_by_query",
+ json!({"query": {"range": {"v": {"gte": 10}}}, "scroll_size": 4}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{body}");
+ assert_eq!(body["total"], json!(20), "docs with v >= 10: {body}");
+ assert_eq!(body["deleted"], json!(20), "{body}");
+ assert_eq!(
+ body["batches"],
+ json!(5),
+ "20 matches at scroll_size 4 = 5 batches: {body}"
+ );
+
+ assert_eq!(count(&app, "dbq-sel").await, 10, "docs with v < 10 survive");
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 975fa29ea..c24bbd605 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1590,6 +1590,93 @@ mod flush_publication_recovery_tests {
drop(engine);
}
+ /// #1019: the delete-by-query search phase must not materialise
+ /// `_source`. An `ids_only` page (sort confined to `_id`, nothing that
+ /// reads fields) returns hits with `source: Value::Null` — the direct
+ /// observable of the projection, because `_source: false` alone would
+ /// NOT do this (`apply_source_filter_measured` keeps the raw source for
+ /// fields/highlight resolution, so a non-null source here means the page
+ /// hydrated `Value` trees again). Also pins the keyset contract the
+ /// paginated runners rely on: `search_after` pages over a `_id: asc`
+ /// sort partition the corpus with no gaps, no dupes, exact total.
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn ids_only_pages_skip_source_materialisation() {
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ engine
+ .create_index("ids-only-pages", slash_field_schema())
+ .unwrap();
+ let idx = engine.get_index("ids-only-pages").unwrap();
+ idx.abort_background_tasks();
+ const N: u64 = 300;
+ for i in 0..N {
+ idx.index_document(
+ Some(format!("d{i:0width$}", width = 4)),
+ json!({"bad/field": "needle"}),
+ )
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+ let mut seen: Vec<String> = Vec::new();
+ let mut first_total: Option<u64> = None;
+ let mut cursor: Option<Vec<Value>> = None;
+ loop {
+ let req = SearchRequest {
+ query: QueryNode::MatchAll,
+ from: 0,
+ size: 50,
+ sort: vec![xerj_query::sort::SortField {
+ field: "_id".to_string(),
+ order: xerj_query::sort::SortOrder::Asc,
+ mode: xerj_query::sort::SortMode::default(),
+ missing: xerj_query::sort::SortMissing::default(),
+ format: None,
+ unmapped_type: None,
+ numeric_type: None,
+ }],
+ search_after: cursor,
+ ids_only: true,
+ ..SearchRequest::default()
+ };
+ let r = idx.search(&req).await.unwrap();
+ assert!(r.hits.len() <= 50);
+ if r.hits.is_empty() {
+ break;
+ }
+ if first_total.is_none() {
+ first_total = Some(r.total.value);
+ }
+ for h in &r.hits {
+ assert!(
+ h.source.is_null(),
+ "#1019: ids-only hit must not carry a hydrated _source"
+ );
+ seen.push(h.id.clone());
+ }
+ let last = r.hits.last().unwrap().id.clone();
+ cursor = Some(vec![Value::String(last)]);
+ }
+ assert_eq!(
+ first_total,
+ Some(N),
+ "page-1 total must be the exact match count"
+ );
+ assert_eq!(seen.len() as u64, N, "pages must cover every doc");
+ let mut sorted = seen.clone();
+ sorted.sort();
+ assert_eq!(
+ seen, sorted,
+ "keyset pages must arrive _id-ascending across the whole scan"
+ );
+ sorted.dedup();
+ assert_eq!(sorted.len() as u64, N, "no duplicate ids across pages");
+ drop(idx);
+ drop(engine);
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn slash_field_flush_publishes_and_restarts_searchable() {
let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
@@ -16209,6 +16296,7 @@ impl Index {
profile: false,
leaf_ts_field: None,
savings: request.savings,
+ ids_only: false,
})
.collect();
let mut leg_outcomes: Vec<Option<Result<SearchResult>>> =
@@ -16382,6 +16470,7 @@ impl Index {
min_score: None,
leaf_ts_field: None,
savings: request.savings,
+ ids_only: false,
};
let plain = filter.is_none()
&& boost.is_none()
@@ -17711,25 +17800,137 @@ impl Index {
Ok(Some(resp))
}
+ /// The complete live `_id` match set for `match_all` / `ids` queries,
+ /// ascending — the #1019 single-pass alternative to paging a purge.
+ ///
+ /// Reads each flushed segment's cached `_id → position` map (`#950`'s
+ /// `id_pos_map_for`, which decodes the `_id` column alone) and the
+ /// version map, so the whole match set costs one O(N) id
+ /// materialisation (~48 B per match: `String` + vec slot) instead of the
+ /// paged loop's one full stored-section re-scan per page (O(N × pages)
+ /// over the corpus — the dominant cost of a default-size purge once the
+ /// per-page `_source` hydration was gone).
+ ///
+ /// Caller MUST flush first (same precondition as the paged loop): the
+ /// memtable is not represented in any segment's id map.
+ ///
+ /// `None` = this query shape (or a segment without a complete id index)
+ /// is not served here — the caller falls back to the paged keyset loop,
+ /// which answers every query. Liveness and byte-order match that loop
+ /// exactly: a missing version-map entry counts as live, a deleted entry
+ /// is skipped, and ids sort as byte strings (the `_id` sort key's order).
+ pub fn matching_ids_sorted(&self, query: &QueryNode) -> Option<Vec<String>> {
+ // Membership resolved against a set: an `ids` query can carry tens of
+ // thousands of values, and a per-doc linear scan would reintroduce
+ // exactly the O(N × k) blowup this method exists to remove.
+ let values: Option<std::collections::HashSet<&str>> = match query {
+ QueryNode::MatchAll => None,
+ QueryNode::Ids { values } => Some(values.iter().map(String::as_str).collect()),
+ _ => return None,
+ };
+ let snap = self.store.snapshot();
+ let mut ids: Vec<String> =
+ Vec::with_capacity(snap.segments.iter().map(|m| m.doc_count as usize).sum());
+ for meta in snap.segments.iter() {
+ let map = self.id_pos_map_for(meta.id.as_str(), meta.doc_count)?;
+ for id in map.keys() {
+ if let Some(set) = &values {
+ if !set.contains(id.as_str()) {
+ continue;
+ }
+ }
+ if let Some(ver) = self.store.version_map.get(id) {
+ if ver.deleted {
+ continue;
+ }
+ }
+ ids.push(id.clone());
+ }
+ }
+ ids.sort_unstable();
+ // Same contract as the paged loop's `seen_ids`: an id resolves to one
+ // delete even if a legacy multi-segment layout lists it twice.
+ ids.dedup();
+ Some(ids)
+ }
+
/// Delete all documents matching the given query.
///
- /// Runs a search limited to 10 000 hits, then deletes each matched document.
+ /// Collects the whole match set in one pass (`matching_ids_sorted`) and
+ /// deletes it in `DELETE_BY_QUERY_PAGE` batches when the query shape
+ /// allows; anything else pages `_id`-keyset batches with the internal
+ /// ids-only projection (the `_reindex` loop's shape). Either way a purge
+ /// over more than `max_result_window` documents deletes EVERY match —
+ /// the old single-shot `size: 10_000` search silently truncated at one
+ /// page and returned a `(total, deleted)` pair that disagreed (#1019).
/// Returns `(total_matched, deleted_count)`.
- pub async fn delete_by_query(&self, query: QueryNode) -> Result<(u64, u64)> {
- let req = SearchRequest {
- query,
- from: 0,
- size: 10_000,
- ..SearchRequest::default()
- };
- let results = self.search(&req).await?;
- let total = results.total.value;
+ pub async fn delete_by_query(self: &Arc<Self>, query: QueryNode) -> Result<(u64, u64)> {
+ // Flush first: the single-pass id collection and `_id`-keyset paging
+ // are both only correct over on-disk segments (the memtable's
+ // field-sort path does not order by `_id` reliably, so either path
+ // over unflushed docs could skip or duplicate ids) — the same
+ // precondition `reindex` establishes.
+ self.flush().await?;
+ if let Some(ids) = self.matching_ids_sorted(&query) {
+ let total = ids.len() as u64;
+ let mut deleted = 0u64;
+ for chunk in ids.chunks(DELETE_BY_QUERY_PAGE) {
+ for id in chunk {
+ if self.delete_document(id).await? {
+ deleted += 1;
+ }
+ }
+ }
+ // Never under-report if a concurrent writer extended the match
+ // set past the collection snapshot mid-purge.
+ return Ok((total.max(deleted), deleted));
+ }
+ let mut total = 0u64;
let mut deleted = 0u64;
- for hit in results.hits {
- if self.delete_document(&hit.id).await? {
- deleted += 1;
+ // Keyset cursor: `[last_id]` of the previous page (the sole sort key
+ // is `_id: asc`). `None` on the first page.
+ let mut search_after: Option<Vec<Value>> = None;
+ loop {
+ let req = SearchRequest {
+ query: query.clone(),
+ from: 0,
+ size: DELETE_BY_QUERY_PAGE,
+ sort: vec![xerj_query::sort::SortField {
+ field: "_id".to_string(),
+ order: xerj_query::sort::SortOrder::Asc,
+ mode: xerj_query::sort::SortMode::default(),
+ missing: xerj_query::sort::SortMissing::default(),
+ format: None,
+ unmapped_type: None,
+ numeric_type: None,
+ }],
+ search_after: search_after.clone(),
+ ids_only: true,
+ ..SearchRequest::default()
+ };
+ let results = self.search(&req).await?;
+ if results.hits.is_empty() {
+ break;
+ }
+ // Page 1's `hits.total` is the EXACT pre-delete match count
+ // (later pages run over an already-shrunk live set).
+ if total == 0 {
+ total = results.total.value;
+ }
+ let last_id = results.hits.last().map(|h| h.id.clone());
+ for hit in results.hits {
+ if self.delete_document(&hit.id).await? {
+ deleted += 1;
+ }
+ }
+ match last_id {
+ Some(id) => search_after = Some(vec![Value::String(id)]),
+ None => break,
}
}
+ // Never under-report if a concurrent writer extended the match set
+ // past page 1's snapshot count mid-purge.
+ total = total.max(deleted);
Ok((total, deleted))
}
@@ -18829,6 +19030,36 @@ impl Index {
|| request.min_score.is_some();
let count_only: bool = !need_sources_for_post;
+ // #1019 — ids-only projection eligibility. `request.ids_only` is set
+ // solely by engine-controlled callers (the `_delete_by_query` runner
+ // and the engine-level purge loop); `parse_request` can never produce
+ // it, so it cannot arrive from the wire. It is honoured only when the
+ // request provably needs nothing beyond hit ids: sort keys confined
+ // to `_id`/`_doc` (a real field sort or a `_score` ordering would
+ // have to read `_source` — via `compute_sort_values` or the per-doc
+ // scorer — to rank), and none of the page post-passes that consume
+ // sources (`fields` / `script_fields` / `highlight` / `explain` /
+ // `collapse` / `rescore` / `aggs` / `min_score`). Any other shape
+ // silently degrades to a full request — correctness first. The
+ // non-empty-sort requirement also means the columnar
+ // `scored_columnar` path (whose `scored_fast_plan` gate demands an
+ // EMPTY sort and no `search_after`) is unreachable in ids-only mode,
+ // so its Phase-5 hydration needs no branch of its own.
+ let ids_only: bool = request.ids_only
+ && !request.sort.is_empty()
+ && request
+ .sort
+ .iter()
+ .all(|sf| sf.field == "_id" || sf.is_doc_order())
+ && request.fields.is_empty()
+ && request.script_fields.is_none()
+ && request.highlight.is_none()
+ && !request.explain
+ && request.collapse.is_none()
+ && request.rescore.is_empty()
+ && request.aggs.is_none()
+ && request.min_score.is_none();
+
// --- Memtable search ---
// `all_hits` is the bounded collector of fully-hydrated hits (source +
// score). Capacity is `materialisation_limit`; anything beyond that
@@ -21126,6 +21357,7 @@ impl Index {
&cand,
query,
false,
+ ids_only,
sort_topk.as_mut().expect("gated on is_some"),
&mut seen_ids,
&mut discard,
@@ -21811,6 +22043,7 @@ impl Index {
sort_cand.as_ref().expect("gated on is_some"),
query,
is_match_all,
+ ids_only,
sort_topk.as_mut().expect("gated on is_some"),
&mut seen_ids,
&mut total_count,
@@ -21840,6 +22073,7 @@ impl Index {
&cached_bytes,
query,
is_match_all,
+ ids_only,
count_only,
scan_limit,
count_authoritative,
@@ -21893,6 +22127,7 @@ impl Index {
pf,
query,
is_match_all,
+ ids_only,
count_only,
scan_limit,
count_authoritative,
@@ -21908,6 +22143,7 @@ impl Index {
&warm_slices.bytes,
query,
is_match_all,
+ ids_only,
count_only,
scan_limit,
count_authoritative,
@@ -21992,6 +22228,7 @@ impl Index {
&stored_bytes,
query,
is_match_all,
+ ids_only,
count_only,
scan_limit,
count_authoritative,
@@ -29375,6 +29612,9 @@ impl Index {
// score now comes from `scorer` (which closes over the query).
_query: &QueryNode,
is_match_all: bool,
+ // #1019 — ids-only projection (delete-by-query pages): see the
+ // admission block below.
+ ids_only: bool,
topk: &mut SortTopK,
seen_ids: &mut HashSet<String>,
total_count: &mut u64,
@@ -29399,6 +29639,58 @@ impl Index {
let Some(slice) = slices.bytes.get(start as usize..end as usize) else {
continue;
};
+ // #1019 — ids-only hydration. Candidates are guaranteed query
+ // matches by construction (match_all, or positions drawn from
+ // the query's own dv/FTS match set), and the eligibility gate
+ // in `search_inner` confined the sort to `_id`/`_doc`, so
+ // nothing here needs the `_source`: extract the id and offer
+ // it with `source: Value::Null` — no `Value` tree, no clone.
+ // The version-map liveness filter is kept verbatim; the
+ // superseded-copy (`_seq_no`) skip is deliberately dropped (a
+ // stale copy carries the SAME id, and id-set membership is all
+ // this projection may decide). `extract_stored_id` → `None`
+ // (escape-bearing layout) falls through to the full parse.
+ if ids_only {
+ if let Some(id) = extract_stored_id(slice) {
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
+ if let Some(ver) = self.store.version_map.get(&id) {
+ if ver.deleted {
+ continue;
+ }
+ let (s, v) = self.hit_seq_version(&id, &ver);
+ hit_seq_no = s;
+ hit_version = v;
+ }
+ *total_count += 1;
+ if seen_ids.contains(&id) {
+ continue;
+ }
+ let key =
+ compute_sort_values(&Value::Null, 1.0, &id, topk.fields.as_slice(), self);
+ if !topk.would_admit(&key) {
+ continue;
+ }
+ seen_ids.insert(id.clone());
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
+ topk.offer_keyed(
+ Hit {
+ id,
+ score: 1.0,
+ source: Value::Null,
+ seq_no: hit_seq_no,
+ version: hit_version,
+ sort: key,
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ },
+ seq,
+ );
+ continue;
+ }
+ }
let mut doc_buf = slice.to_vec();
let doc: Value = match simd_json::serde::from_slice(&mut doc_buf) {
Ok(v) => v,
@@ -29432,7 +29724,10 @@ impl Index {
// Candidates are guaranteed query matches by construction
// (match_all, or positions drawn from the query's own dv/FTS
// match set) — only the SCORE needs the per-query path.
- let score = if is_match_all {
+ // #1019 — under the ids-only projection the score is unused
+ // (sort confined to `_id`/`_doc`), so the scorer is skipped and
+ // the hit carries `source: Value::Null` instead of a clone.
+ let score = if is_match_all || ids_only {
1.0
} else {
scorer(source_ref, id_ref)
@@ -29448,7 +29743,11 @@ impl Index {
Hit {
id,
score,
- source: source_ref.clone(),
+ source: if ids_only {
+ Value::Null
+ } else {
+ source_ref.clone()
+ },
seq_no: hit_seq_no,
version: hit_version,
sort: key,
@@ -29489,6 +29788,14 @@ impl Index {
pre_filter: &PrefilterSet,
query: &QueryNode,
is_match_all: bool,
+ // #1019 — ids-only projection: this arm must still PARSE each
+ // candidate (the pre-filter is only a superset, so the full query
+ // is re-tested against `_source`), but under the projection the
+ // admitted hit carries `source: Value::Null` and skips the scorer.
+ // Unreachable for the delete loops themselves (they sort by `_id`,
+ // which routes them to the sorted-candidate / scan arms); kept
+ // uniform so no ids-only admission can clone a source tree.
+ ids_only: bool,
count_only: bool,
materialisation_limit: usize,
count_authoritative: bool,
@@ -29580,8 +29887,15 @@ impl Index {
if seen_ids.contains(&id) {
continue;
}
- let source = doc.get("_source").cloned().unwrap_or_else(|| doc.clone());
- let score = if is_match_all {
+ // #1019 — ids-only projection: no source clone, no scorer (the
+ // eligibility argument is documented on the sort-topk branch of
+ // `scan_stored_section_into`).
+ let source = if ids_only {
+ Value::Null
+ } else {
+ doc.get("_source").cloned().unwrap_or_else(|| doc.clone())
+ };
+ let score = if is_match_all || ids_only {
1.0
} else {
scorer(&source, &id)
@@ -29867,6 +30181,10 @@ impl Index {
stored_bytes: &[u8],
query: &QueryNode,
is_match_all: bool,
+ // #1019 — ids-only projection: the caller reads nothing but `hit.id`
+ // (delete-by-query / engine purge pages); sort keys were gated to
+ // `_id`/`_doc` in `search_inner`. See the admission block below.
+ ids_only: bool,
count_only: bool,
materialisation_limit: usize,
count_authoritative: bool,
@@ -29922,6 +30240,28 @@ impl Index {
// `query_needs_id_injection`'s doc comment.
let needs_id_injection = query_needs_id_injection(query);
+ // #1019 — precomputed cursor for the ids-only fast decline in the
+ // admission block below: `(cursor, ascending)` when the sort is a
+ // sole `_id` key with a string `search_after` (every delete-by-query
+ // page after the first). Any other shape declines through the normal
+ // `would_admit` path.
+ let ids_only_cursor: Option<(String, bool)> = if !ids_only {
+ None
+ } else {
+ sort_topk.as_ref().and_then(|t| {
+ if t.fields.len() == 1 && t.fields[0].field == "_id" {
+ let asc = t.fields[0].order == xerj_query::sort::SortOrder::Asc;
+ t.after
+ .as_ref()
+ .and_then(|a| a.first())
+ .and_then(Value::as_str)
+ .map(|c| (c.to_string(), asc))
+ } else {
+ None
+ }
+ })
+ };
+
loop {
// F1b — IN-SEGMENT early stop. The between-segment F1 break in
// `search_inner` skips segments once the page is full and the
@@ -30058,6 +30398,137 @@ impl Index {
let doc_slice = &bytes[start..end];
+ // #1019 — ids-only admission. This scan feeds a caller that
+ // reads nothing but `hit.id`, the sort keys are `_id`/`_doc`
+ // only (gated in `search_inner`), and the query decides
+ // membership without a `_source` (match_all, or an `ids`
+ // clause compared by string). Extract the id straight off the
+ // stored slice — BORROWED, no per-doc allocation — and admit
+ // WITHOUT building the `Value` tree: the full-tree parse of
+ // every stored doc was the dominant RSS transient and wall
+ // time of a whole-corpus purge (issue #1019 — the residual
+ // the #950 fix recorded as follow-up headroom). The version-map
+ // liveness filter is kept verbatim (a tombstoned id must not be
+ // admitted); the superseded-copy (`_seq_no`) skip is
+ // deliberately dropped because a stale copy carries the SAME
+ // id — id-set membership is all this projection may decide.
+ // `extract_stored_id_str` returns `None` for escape-bearing or
+ // unusual layouts; those (and queries that need `_source` to
+ // test membership) fall through to the full parse below, which
+ // admits with the same ids-only source cut.
+ //
+ // Cursor decline comes FIRST, before the liveness lookup and
+ // every allocation: a delete-by-query purge walks this scan
+ // once per `scroll_size` page, and every page after the first
+ // re-visits all previously deleted docs — with the id-ordered
+ // cursor compared as raw bytes up front, a declined doc costs
+ // one prefix scan + one memcmp and nothing else (63.5 M such
+ // visits measured on the 252k-doc purge benchmark before this
+ // ordering). Below-cursor docs are also skipped out of
+ // `total_count`; the only ids-only callers are the delete
+ // loops, which read `hits.total` off page 1 (cursor `None`).
+ let ids_only_admit: Option<&str> = if !ids_only {
+ None
+ } else {
+ extract_stored_id_str(doc_slice)
+ };
+ if let Some(id) = ids_only_admit {
+ if let Some((ref cursor, asc)) = ids_only_cursor {
+ let ord = id.as_bytes().cmp(cursor.as_bytes());
+ let at_or_below = if asc {
+ ord != std::cmp::Ordering::Greater
+ } else {
+ ord != std::cmp::Ordering::Less
+ };
+ if at_or_below {
+ continue;
+ }
+ }
+ // Membership without a `_source`. `None` = this query
+ // needs the full parse to decide → fall through.
+ let member: Option<bool> = match query {
+ QueryNode::MatchAll => Some(true),
+ QueryNode::Ids { values } => Some(values.iter().any(|v| v == id)),
+ _ => None,
+ };
+ if member == Some(false) {
+ continue;
+ }
+ if member == Some(true) {
+ let mut hit_seq_no = None;
+ let mut hit_version = None;
+ if let Some(ver) = self.store.version_map.get(id) {
+ if ver.deleted {
+ continue;
+ }
+ let (s, v) = self.hit_seq_version(id, &ver);
+ hit_seq_no = s;
+ hit_version = v;
+ }
+ *total_count += 1;
+ *dbg_admitted += 1;
+ if count_only {
+ continue;
+ }
+ if let Some(topk) = sort_topk.as_deref_mut() {
+ if seen_ids.contains(id) {
+ continue;
+ }
+ let key = compute_sort_values(
+ &Value::Null,
+ 1.0,
+ id,
+ topk.fields.as_slice(),
+ self,
+ );
+ if !topk.would_admit(&key) {
+ continue;
+ }
+ seen_ids.insert(id.to_string());
+ let seq = hit_seq_no.unwrap_or(u64::MAX);
+ topk.offer_keyed(
+ Hit {
+ id: id.to_string(),
+ score: 1.0,
+ source: Value::Null,
+ seq_no: hit_seq_no,
+ version: hit_version,
+ sort: key,
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ },
+ seq,
+ );
+ continue;
+ }
+ if all_hits.len() >= materialisation_limit {
+ if count_authoritative {
+ break;
+ }
+ continue;
+ }
+ if seen_ids.contains(id) {
+ continue;
+ }
+ seen_ids.insert(id.to_string());
+ all_hits.push(Hit {
+ id: id.to_string(),
+ score: 1.0,
+ source: Value::Null,
+ seq_no: hit_seq_no,
+ version: hit_version,
+ sort: Vec::new(),
+ explain: None,
+ highlight: None,
+ matched_queries: Vec::new(),
+ passage: None,
+ });
+ continue;
+ }
+ }
+
let mut doc_buf = doc_slice.to_vec();
let doc: Value = match simd_json::serde::from_slice(&mut doc_buf) {
Ok(v) => v,
@@ -30150,7 +30621,14 @@ impl Index {
continue;
}
let source_ref = doc.get("_source").unwrap_or(&doc);
- let score = if is_match_all {
+ // #1019 — under the ids-only projection the score is unused
+ // (the eligibility gate confined the sort to `_id`/`_doc`,
+ // neither of which reads `_source` to rank), so the scorer
+ // is skipped and the hit carries `source: Value::Null`
+ // instead of a clone of the parsed tree — the per-hit clone
+ // is what piled up in the bounded heap across a
+ // whole-corpus purge.
+ let score = if is_match_all || ids_only {
1.0
} else {
scorer(source_ref, id_ref)
@@ -30167,7 +30645,11 @@ impl Index {
Hit {
id,
score,
- source: source_ref.clone(),
+ source: if ids_only {
+ Value::Null
+ } else {
+ source_ref.clone()
+ },
seq_no: hit_seq_no,
version: hit_version,
sort: key,
@@ -30203,8 +30685,14 @@ impl Index {
continue;
}
- let source = doc.get("_source").cloned().unwrap_or_else(|| doc.clone());
- let score = if is_match_all {
+ // #1019 — ids-only projection: no source clone, no scorer (see
+ // the sort-topk branch above for the eligibility argument).
+ let source = if ids_only {
+ Value::Null
+ } else {
+ doc.get("_source").cloned().unwrap_or_else(|| doc.clone())
+ };
+ let score = if is_match_all || ids_only {
1.0
} else {
scorer(&source, &id)
@@ -35644,6 +36132,13 @@ fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {
/// prevent.
const MAX_PER_INDEX_WAL_SHARDS: u64 = 256;
+/// Batch size for the engine-level `Index::delete_by_query` keyset loop
+/// (#1019) — ES's `scroll_size` default ("The size of the scroll request
+/// that powers the operation"). Paging is keyset (`search_after` on a
+/// `_id: asc` sort), so a small page never approaches the
+/// `max_result_window` deep-paging wall.
+const DELETE_BY_QUERY_PAGE: usize = 1_000;
+
/// A concurrent map that exists **once per index** (#873).
///
/// `DashMap::new()` sizes its shard array from the host core count —
@@ -45373,13 +45868,14 @@ impl PrefilterSet {
/// however many complete objects it found (caller compares against the
/// segment doc count to reject malformed sections).
/// Extract the `_id` string value from a stored-doc slice
-/// (`{"_id":"<value>",...}`) without a full JSON parse. Returns `None` when
-/// the slice does not begin with an `_id` string key, or when the value
-/// contains a JSON escape (`\`), in which case the caller falls back to a full
-/// parse. This is the hot path for building the per-segment id→position map:
+/// (`{"_id":"<value>",...}`) without a full JSON parse, borrowing from the
+/// slice. Returns `None` when the slice does not begin with an `_id` string
+/// key, or when the value contains a JSON escape (`\`), in which case the
+/// caller falls back to a full parse. This is the hot path for building the
+/// per-segment id→position map and for the #1019 ids-only scan admission:
/// the common append-only corpus has escape-free ids, so it stays a cheap
/// prefix scan.
-fn extract_stored_id(slice: &[u8]) -> Option<String> {
+fn extract_stored_id_str(slice: &[u8]) -> Option<&str> {
let n = slice.len();
let mut i = 0usize;
// Leading whitespace + opening brace.
@@ -45418,9 +45914,7 @@ fn extract_stored_id(slice: &[u8]) -> Option<String> {
match slice[i] {
b'\\' => return None, // escape present → let the caller full-parse
b'"' => {
- return std::str::from_utf8(&slice[val_start..i])
- .ok()
- .map(str::to_string)
+ return std::str::from_utf8(&slice[val_start..i]).ok();
}
_ => i += 1,
}
@@ -45428,6 +45922,11 @@ fn extract_stored_id(slice: &[u8]) -> Option<String> {
None
}
+/// Owned form of [`extract_stored_id_str`] for callers that keep the id.
+fn extract_stored_id(slice: &[u8]) -> Option<String> {
+ extract_stored_id_str(slice).map(str::to_string)
+}
+
/// Exact set of stored positions matching a `term`/`terms` value list in one
/// segment, resolved from the segment's doc-values columns. Keyword fields
/// enumerate positions carrying any of the terms' ordinals; numeric fields
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 4263e4169..7ee3054ae 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -2645,6 +2645,45 @@ async fn test_delete_by_query() {
assert_eq!(result.hits[0].id, "q3");
}
+/// #1019: the engine-level `delete_by_query` had the same 10 000-doc
+/// single-shot cap as the HTTP handler — it built one `SearchRequest` with
+/// `size: 10_000` and deleted only the returned page. Over a corpus larger
+/// than `max_result_window` it under-deleted while reporting an exact
+/// `total`, so the `(total, deleted)` pair disagreed and docs survived.
+/// Fails before the fix at `(12000, 10000)`.
+#[tokio::test]
+async fn test_delete_by_query_purges_past_ten_thousand() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ engine.create_index("dbq-12k", Schema::empty()).unwrap();
+ let idx = engine.get_index("dbq-12k").unwrap();
+
+ const N: u64 = 12_000;
+ let width = N.to_string().len();
+ for i in 0..N {
+ idx.index_document(
+ Some(format!("d{i:0width$}", width = width)),
+ json!({"v": i}),
+ )
+ .await
+ .unwrap();
+ }
+
+ let (total, deleted) = idx.delete_by_query(QueryNode::MatchAll).await.unwrap();
+ assert_eq!(total, N, "total is the exact match count");
+ assert_eq!(
+ deleted, N,
+ "every matching doc deleted, not just the first 10k page"
+ );
+
+ let result = idx
+ .search(&make_search(json!({"match_all": {}})))
+ .await
+ .unwrap();
+ assert_eq!(result.total.value, 0, "index must be empty after the purge");
+}
+
// ── 10. multi_match query ─────────────────────────────────────────────────────
#[tokio::test]
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index d07587979..3c48aea56 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -1252,6 +1252,21 @@ pub struct SearchRequest {
/// lookups, suggesters) never pay for a number nobody will read.
#[serde(skip)]
pub savings: SavingsMode,
+
+ /// INTERNAL (never on the wire — `serde(skip)`): the caller reads nothing
+ /// from the hits except their `id`s (`_delete_by_query` pages, the
+ /// engine-level purge loop). When the request shape needs nothing beyond
+ /// ids — sort keys confined to `_id`/`_doc`, no `fields` /
+ /// `script_fields` / `highlight` / `explain` / `collapse` / `rescore` /
+ /// `aggs` / `min_score` — the executor's hydration sites extract the
+ /// stored `_id` without building the `_source` `Value` tree and the
+ /// returned hits carry `source: Value::Null` (#1019). Any shape the
+ /// engine's eligibility gate rejects runs exactly as a full request.
+ ///
+ /// Set ONLY by engine-controlled callers; `parse_request` never sets it,
+ /// so it can never arrive from the wire (same contract as `savings`).
+ #[serde(skip)]
+ pub ids_only: bool,
}
/// Whether — and how precisely — to measure withheld `_source` bytes.
@@ -1303,6 +1318,7 @@ impl Default for SearchRequest {
min_score: None,
leaf_ts_field: None,
savings: SavingsMode::Off,
+ ids_only: false,
}
}
}
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index 6a4d6773e..e67d78040 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -365,6 +365,8 @@ pub fn parse_request(body: &Value) -> Result<SearchRequest> {
// Never parsed off the wire body: the ES-compat `_search` handler
// sets this from `?savings=`.
savings: SavingsMode::Off,
+ // #1019: ids-only is engine-internal and never arrives from the wire.
+ ids_only: false,
})
}
diff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rs
index c80fda1a4..0cfaf2a9d 100644
--- a/engine/tests/es-compat-yaml/src/main.rs
+++ b/engine/tests/es-compat-yaml/src/main.rs
@@ -1170,6 +1170,9 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
("POST".into(), path, body)
}
"reindex" => ("POST".into(), "/_reindex".into(), body),
+ // #1019 — pins the paginated delete-by-query contract (exact total,
+ // real batch count, max_docs).
+ "delete_by_query" => ("POST".into(), format!("/{}/_delete_by_query", index), body),
"scroll" => {
let mut path = "/_search/scroll".to_string();
let mut qp = Vec::new();
diff --git a/engine/tests/es-compat-yaml/yaml/delete_by_query/10_multipage_purge.yml b/engine/tests/es-compat-yaml/yaml/delete_by_query/10_multipage_purge.yml
new file mode 100644
index 000000000..c939bf7c4
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/delete_by_query/10_multipage_purge.yml
@@ -0,0 +1,76 @@
+---
+"delete_by_query pages through the whole index in batches":
+ # Seed 9 docs; scroll_size 2 forces 5 batches, so the run must advance a
+ # keyset cursor across pages instead of stopping at the first batch
+ # (#1019: the runner used to fire ONE size=10000 search and truncate).
+ - do:
+ index: { index: dbq_src, id: "1", body: { n: 1 } }
+ - do:
+ index: { index: dbq_src, id: "2", body: { n: 2 } }
+ - do:
+ index: { index: dbq_src, id: "3", body: { n: 3 } }
+ - do:
+ index: { index: dbq_src, id: "4", body: { n: 4 } }
+ - do:
+ index: { index: dbq_src, id: "5", body: { n: 5 } }
+ - do:
+ index: { index: dbq_src, id: "6", body: { n: 6 } }
+ - do:
+ index: { index: dbq_src, id: "7", body: { n: 7 } }
+ - do:
+ index: { index: dbq_src, id: "8", body: { n: 8 } }
+ - do:
+ index: { index: dbq_src, id: "9", refresh: true, body: { n: 9 } }
+
+ - do:
+ delete_by_query:
+ index: dbq_src
+ body:
+ query: { match_all: {} }
+ scroll_size: 2
+
+ # total is the exact match count (not the first page's length), deleted
+ # every match, batches the real page count.
+ - match: { total: 9 }
+ - match: { deleted: 9 }
+ - match: { batches: 5 }
+ - match: { failures: [] }
+
+ - do:
+ count:
+ index: dbq_src
+ body:
+ query: { match_all: {} }
+ - match: { count: 0 }
+
+---
+"delete_by_query honors max_docs":
+ - do:
+ index: { index: dbq_max, id: "1", body: { n: 1 } }
+ - do:
+ index: { index: dbq_max, id: "2", body: { n: 2 } }
+ - do:
+ index: { index: dbq_max, id: "3", body: { n: 3 } }
+ - do:
+ index: { index: dbq_max, id: "4", body: { n: 4 } }
+ - do:
+ index: { index: dbq_max, id: "5", refresh: true, body: { n: 5 } }
+
+ # ES: max_docs <= scroll_size means a single batch, no scroll.
+ - do:
+ delete_by_query:
+ index: dbq_max
+ body:
+ query: { match_all: {} }
+ max_docs: 2
+
+ - match: { total: 2 }
+ - match: { deleted: 2 }
+ - match: { batches: 1 }
+
+ - do:
+ count:
+ index: dbq_max
+ body:
+ query: { match_all: {} }
+ - match: { count: 3 }