AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): fan _delete_by_query/_update_by_query out to every alias member (#450)
xerj-org71d77913d55a
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 0179b001d..fd5e0d906 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -24363,19 +24363,125 @@ pub struct DeleteByQueryBody {pub query: Value,}+/// Resolve a by-query selector to the concrete member indices it must touch and+/// the effective query to run against each.+///+/// The destructive by-query handlers used to call `Engine::get_index`, which+/// resolves an alias to `aliased.first()` — one member. A `_delete_by_query`+/// through a 3-member alias therefore deleted a third of the corpus and+/// reported a clean success (#450). This fans the selector out to every member+/// the read paths already see (`resolve_index_selector`, #433) and — crucially+/// for a destructive path — ANDs in any alias `filter` the way `search_impl`+/// does, so a *filtered* alias deletes only its slice, not the whole backing+/// index (the #524 hazard).+async fn resolve_by_query_targets(+ state: &AppState,+ spec: &str,+ user_query: Value,+) -> std::result::Result<(Vec<std::sync::Arc<xerj_engine::Index>>, Value), xerj_common::XerjError> {+ let member_names = resolve_index_selector(state, spec).await;++ // Alias filters are keyed on the name AS WRITTEN (the alias, not its+ // expanded members) — mirror `search_impl`'s alias-filter path exactly.+ let mut alias_filters: Vec<Value> = Vec::new();+ for part in spec.split(',').map(str::trim).filter(|s| !s.is_empty()) {+ if let Some(entry) = state.engine.aliases.get(part) {+ for backing in entry.value().iter() {+ if let Some(meta) = state.engine.index_alias_metadata.get(backing) {+ if let Some(filter) = meta.get(part).and_then(|v| v.get("filter")).cloned() {+ alias_filters.push(filter);+ }+ }+ }+ }+ }+ let effective_query = if alias_filters.is_empty() {+ user_query+ } else {+ json!({ "bool": { "must": [user_query], "filter": alias_filters } })+ };++ let mut indices = Vec::with_capacity(member_names.len());+ for name in &member_names {+ indices.push(+ state+ .engine+ .get_index(name)+ .map_err(xerj_common::XerjError::from)?,+ );+ }+ Ok((indices, effective_query))+}++/// Combine the per-member results of a fanned-out by-query into one response.+///+/// A single member (the common case: a plain index, or a one-member alias) is+/// returned byte-for-byte unchanged so the existing wire format never drifts.+/// Multiple members are summed field-by-field with their `failures` arrays+/// concatenated; if any member produced an error body (it carries a `status`),+/// that error is surfaced rather than a clean aggregate — a destructive+/// by-query that partially failed must not report success.+fn aggregate_by_query_results(mut results: Vec<Value>, count_key: &str) -> Value {+ if results.len() == 1 {+ return results.pop().unwrap();+ }+ if let Some(err) = results.iter().find(|v| v.get("status").is_some()) {+ return err.clone();+ }+ let mut took = 0u64;+ let mut timed_out = false;+ let mut total = 0u64;+ let mut affected = 0u64;+ let mut batches = 0u64;+ let mut version_conflicts = 0u64;+ let mut noops = 0u64;+ let mut failures: Vec<Value> = Vec::new();+ for r in &results {+ took += r.get("took").and_then(Value::as_u64).unwrap_or(0);+ timed_out |= r.get("timed_out").and_then(Value::as_bool).unwrap_or(false);+ total += r.get("total").and_then(Value::as_u64).unwrap_or(0);+ affected += r.get(count_key).and_then(Value::as_u64).unwrap_or(0);+ batches += r.get("batches").and_then(Value::as_u64).unwrap_or(0);+ version_conflicts += r+ .get("version_conflicts")+ .and_then(Value::as_u64)+ .unwrap_or(0);+ noops += r.get("noops").and_then(Value::as_u64).unwrap_or(0);+ if let Some(f) = r.get("failures").and_then(Value::as_array) {+ failures.extend(f.iter().cloned());+ }+ }+ let mut out = serde_json::Map::new();+ out.insert("took".into(), json!(took));+ out.insert("timed_out".into(), json!(timed_out));+ out.insert("total".into(), json!(total));+ out.insert(count_key.to_string(), json!(affected));+ out.insert("batches".into(), json!(batches));+ out.insert("version_conflicts".into(), json!(version_conflicts));+ out.insert("noops".into(), json!(noops));+ out.insert("failures".into(), json!(failures));+ out.insert("throttled_millis".into(), json!(0));+ out.insert("requests_per_second".into(), json!(-1));+ out.insert("throttled_until_millis".into(), json!(0));+ Value::Object(out)+}+pub async fn delete_by_query(State(state): State<AppState>,Path(index): Path<String>,Query(params): Query<HashMap<String, String>>,Json(body): Json<DeleteByQueryBody>,) -> impl IntoResponse {- let idx = match state.engine.get_index(&index) {- Ok(i) => i,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),- };+ // Fan the selector out to every member of a multi-index alias (#450), with+ // any alias filter already ANDed into the query.+ let (indices, effective_query) =+ match resolve_by_query_targets(&state, &index, body.query.clone()).await {+ Ok(t) => t,+ Err(e) => return ApiError::new(e).into_response(),+ };// Run a match-all-sized search using the provided query.- let search_body_val = json!({ "query": body.query, "size": 10000, "from": 0 });+ let search_body_val = json!({ "query": effective_query, "size": 10000, "from": 0 });let search_req = match xerj_query::parse_request(&search_body_val).map_err(|e| xerj_common::XerjError::invalid_query(e.to_string())){@@ -24396,21 +24502,27 @@ pub async fn delete_by_query(let spawned_key = task_key.clone();let tasks = state.tasks.clone();// Detached, so no `index_guard` rule — and none is needed: the task- // owns `idx`, the single already-authorized `Index` handle this request- // resolved, and an `Index` cannot name another index (it holds no- // `Engine`). See `xerj_engine::index_guard` for the contract a spawn- // that *can* reach `Engine::get_index` has to follow instead.+ // owns the already-authorized `Index` handles this request resolved,+ // and an `Index` cannot name another index (it holds no `Engine`). See+ // `xerj_engine::index_guard` for the contract a spawn that *can* reach+ // `Engine::get_index` has to follow instead.tokio::spawn(async move {- let response = run_delete_by_query(&idx, &search_req).await;- tasks.complete(&spawned_key, response);+ let mut results = Vec::with_capacity(indices.len());+ for idx in &indices {+ results.push(run_delete_by_query(idx, &search_req).await);+ }+ tasks.complete(&spawned_key, aggregate_by_query_results(results, "deleted"));drop(handle);});return Json(json!({ "task": task_key })).into_response();}- let response = run_delete_by_query(&idx, &search_req).await;+ let mut results = Vec::with_capacity(indices.len());+ for idx in &indices {+ results.push(run_delete_by_query(idx, &search_req).await);+ }drop(handle);- by_query_response(response)+ by_query_response(aggregate_by_query_results(results, "deleted"))}/// Give a `_delete_by_query` / `_update_by_query` result the HTTP status its@@ -24504,13 +24616,15 @@ pub async fn update_by_query(Query(params): Query<HashMap<String, String>>,Json(body): Json<UpdateByQueryBody>,) -> impl IntoResponse {- let idx = match state.engine.get_index(&index) {- Ok(i) => i,- Err(e) => return ApiError::new(xerj_common::XerjError::from(e)).into_response(),- };-let query_val = body.query.clone().unwrap_or(json!({ "match_all": {} }));- let search_body_val = json!({ "query": query_val, "size": 10000, "from": 0 });+ // Fan the selector out to every member of a multi-index alias (#450), with+ // any alias filter already ANDed into the query.+ let (indices, effective_query) = match resolve_by_query_targets(&state, &index, query_val).await+ {+ Ok(t) => t,+ Err(e) => return ApiError::new(e).into_response(),+ };+ let search_body_val = json!({ "query": effective_query, "size": 10000, "from": 0 });let search_req = match xerj_query::parse_request(&search_body_val).map_err(|e| xerj_common::XerjError::invalid_query(e.to_string())){@@ -24572,18 +24686,26 @@ pub async fn update_by_query(let spawned_key = task_key.clone();let tasks = state.tasks.clone();// Same reasoning as the `_delete_by_query` spawn above: the task owns- // one already-authorized `Index` handle and can reach no other index.+ // the already-authorized `Index` handles and can reach no other index.tokio::spawn(async move {- let response = run_update_by_query(&idx, &search_req, script, pipeline).await;- tasks.complete(&spawned_key, response);+ let mut results = Vec::with_capacity(indices.len());+ for idx in &indices {+ results.push(+ run_update_by_query(idx, &search_req, script.clone(), pipeline.clone()).await,+ );+ }+ tasks.complete(&spawned_key, aggregate_by_query_results(results, "updated"));drop(handle);});return Json(json!({ "task": task_key })).into_response();}- let response = run_update_by_query(&idx, &search_req, script, pipeline).await;+ let mut results = Vec::with_capacity(indices.len());+ for idx in &indices {+ results.push(run_update_by_query(idx, &search_req, script.clone(), pipeline.clone()).await);+ }drop(handle);- by_query_response(response)+ by_query_response(aggregate_by_query_results(results, "updated"))}/// Wall-clock ceiling for the Painless work one scripted-update request maydiff --git a/engine/crates/xerj-api/tests/by_query_through_alias_touches_every_member.rs b/engine/crates/xerj-api/tests/by_query_through_alias_touches_every_member.rsindex 1e769a701..f8a0706ef 100644--- a/engine/crates/xerj-api/tests/by_query_through_alias_touches_every_member.rs+++ b/engine/crates/xerj-api/tests/by_query_through_alias_touches_every_member.rs@@ -97,7 +97,9 @@ async fn seed(app: &axum::Router) {async fn count(app: &axum::Router, index: &str) -> u64 {let (_, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;- body.get("count").and_then(Value::as_u64).unwrap_or(u64::MAX)+ body.get("count")+ .and_then(Value::as_u64)+ .unwrap_or(u64::MAX)}/// The #450 bug: a `_delete_by_query` through a 3-member alias must delete@@ -142,3 +144,40 @@ async fn delete_by_query_through_a_multi_index_alias_deletes_every_member() {);}}++/// The `_update_by_query` sibling: it shares the same `get_index → first`+/// defect, so an update through the alias must reindex every member's matching+/// documents, not just the first member's.+#[tokio::test]+async fn update_by_query_through_a_multi_index_alias_updates_every_member() {+ let (app, _dir) = app().await;+ seed(&app).await;++ let total = (MEMBERS.len() * PER_INDEX) as u64;++ let (status, body) = call(+ &app,+ "POST",+ "/tri/_update_by_query",+ json!({"query": {"match_all": {}}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "_update_by_query status: {body}");+ assert_eq!(+ body.get("updated").and_then(Value::as_u64),+ Some(total),+ "_update_by_query through a {}-member alias must touch every member's docs, \+ not just the first member's (#450): {body}",+ MEMBERS.len()+ );++ // Every member is still fully present (update reindexes in place; nothing+ // is lost) — the point is that the count is honest about the whole corpus.+ for member in MEMBERS {+ assert_eq!(+ count(&app, member).await,+ PER_INDEX as u64,+ "member {member} document count changed unexpectedly after _update_by_query"+ );+ }+}