AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(es-compat): a multi-index alias answered as an alias over one member
xerj-orgaf5ff14cd60c
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 5410cfc45..d25a97e48 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -8605,7 +8605,17 @@ async fn search_impl(.map(|n| resolve_date_math_index(n.trim())).collect();let raw_names: Vec<&str> = resolved_names_owned.iter().map(|s| s.as_str()).collect();- let needs_resolve = raw_names.iter().any(|n| *n == "_all" || n.contains('*'));+ // An ALIAS also needs resolving, and used not to be counted here — which is+ // why a multi-index alias behaved as an alias over one member on every read+ // path (#433). `_count` and plain `_search` truncated the same way: three+ // indices of 40 documents behind one alias answered 40, from the first+ // member, with no error and a `_count` that agreed with the short result.+ // The `_index`-stamping problem reported alongside it is downstream of this+ // — the hits could not carry the right index because the other indices were+ // never searched.+ let needs_resolve = raw_names+ .iter()+ .any(|n| *n == "_all" || n.contains('*') || state.engine.aliases.contains_key(*n));let index_names: Vec<String> = if needs_resolve {let all = state.engine.list_indices().await;let all_names: Vec<String> = all.into_iter().map(|i| i.name).collect();@@ -8623,6 +8633,16 @@ async fn search_impl(resolved.push(name.clone());}}+ } else if let Some(targets) = state.engine.aliases.get(*pattern) {+ // Expand to the concrete members, exactly as+ // `resolve_index_selector` already does for every other+ // endpoint. An alias naming one index resolves to that one, so+ // the single-member case is unchanged.+ for name in targets.value() {+ if !resolved.contains(name) {+ resolved.push(name.clone());+ }+ }} else {let s = pattern.to_string();if !resolved.contains(&s) {@@ -17951,7 +17971,24 @@ pub async fn count_docs(body: OptionalJson<Value>,) -> impl IntoResponse {// Multi-index / all selector: sum counts from every participating index.- if index == "_all" || index == "*" || index.contains(',') || index.contains('*') {+ //+ // An ALIAS belongs in this branch too. It used not to, so a multi-index+ // alias fell through to the single-index path and answered one member's+ // count — three indices of 40 documents behind one alias reported 40, with+ // no error (#433). This is a THIRD copy of the same resolution, alongside+ // `resolve_index_selector` and the one inside `search_impl`; they disagreed+ // about aliases in different ways, which is why `_search` and `_count`+ // could give different answers for the same selector.+ let alias_selector = index+ .split(',')+ .map(str::trim)+ .any(|n| state.engine.aliases.contains_key(n));+ if index == "_all"+ || index == "*"+ || index.contains(',')+ || index.contains('*')+ || alias_selector+ {let all = state.engine.list_indices().await;let all_names: Vec<String> = all.into_iter().map(|i| i.name).collect();let wanted: Vec<String> = if index == "_all" || index == "*" {@@ -17966,6 +18003,12 @@ pub async fn count_docs(out.push(n.clone());}}+ } else if let Some(targets) = state.engine.aliases.get(pat) {+ for n in targets.value() {+ if !out.contains(n) {+ out.push(n.clone());+ }+ }} else if !out.iter().any(|e| e == pat) {out.push(pat.to_string());}diff --git a/engine/crates/xerj-api/tests/alias_multi_index_reads.rs b/engine/crates/xerj-api/tests/alias_multi_index_reads.rsnew file mode 100644index 000000000..bf4195e1d--- /dev/null+++ b/engine/crates/xerj-api/tests/alias_multi_index_reads.rs@@ -0,0 +1,203 @@+//! Issue #433: a multi-index alias answered as an alias over ONE member on+//! every read path.+//!+//! Three indices behind one alias, 40 documents each. `GET /_alias/kb` listed+//! all three, and `_count`, `_search` and a scroll through the alias all+//! answered 40 — from the first member, with no error and a count that agreed+//! with the short result. Not a keyed consumer silently keeping a fraction:+//! every consumer got a fraction.+//!+//! The cause was three independent copies of index-selector resolution.+//! `resolve_index_selector` expanded aliases; `search_impl`'s own resolver only+//! looked for `_all` and `*`; `count_docs` gated its whole multi-index branch on+//! the same two plus a comma or a star. A plain alias fell through both.+//!+//! These go through the real HTTP routes because that is the only place the+//! three resolvers meet.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (status, serde_json::from_slice(&bytes).unwrap_or(Value::Null))+}++const PER_INDEX: usize = 40;++async fn seed(app: &axum::Router) {+ for member in ["kb-a", "kb-b", "kb-c"] {+ let (st, _) = call(+ app,+ "PUT",+ &format!("/{member}"),+ json!({"mappings": {"properties": {"v": {"type": "long"}, "home": {"type": "keyword"}}}}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create {member}");++ let mut bulk = String::new();+ for i in 0..PER_INDEX {+ bulk.push_str(&format!("{}\n", json!({"index": {"_id": i.to_string()}})));+ bulk.push_str(&format!("{}\n", json!({"v": i, "home": member})));+ }+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method("POST")+ .uri(format!("/{member}/_bulk"))+ .header("content-type", "application/x-ndjson")+ .body(Body::from(bulk))+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(response.status(), StatusCode::OK, "bulk {member}");+ let (st, _) = call(app, "POST", &format!("/{member}/_refresh"), Value::Null).await;+ assert_eq!(st, StatusCode::OK);+ }+ let (st, _) = call(+ app,+ "POST",+ "/_aliases",+ json!({"actions": [+ {"add": {"index": "kb-a", "alias": "kb"}},+ {"add": {"index": "kb-b", "alias": "kb"}},+ {"add": {"index": "kb-c", "alias": "kb"}},+ {"add": {"index": "kb-a", "alias": "solo"}}+ ]}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "create aliases");+}++/// `_count` and `_search` must see every member, and a one-member alias must+/// keep answering for exactly that member.+#[tokio::test]+async fn every_read_path_through_a_multi_index_alias_sees_every_member() {+ let (app, _dir) = app().await;+ seed(&app).await;+ let all = (PER_INDEX * 3) as u64;++ let (_, count) = call(&app, "GET", "/kb/_count", Value::Null).await;+ assert_eq!(+ count["count"].as_u64(),+ Some(all),+ "_count through the alias must cover all three members: {count}"+ );++ let (_, search) = call(+ &app,+ "POST",+ "/kb/_search",+ json!({"query": {"match_all": {}}, "size": 0}),+ )+ .await;+ assert_eq!(+ search["hits"]["total"]["value"].as_u64(),+ Some(all),+ "_search through the alias must cover all three members: {search}"+ );++ // The single-member case is the one a narrowing fix would break.+ let (_, solo) = call(&app, "GET", "/solo/_count", Value::Null).await;+ assert_eq!(+ solo["count"].as_u64(),+ Some(PER_INDEX as u64),+ "an alias over one index still answers for that index: {solo}"+ );+ let (_, concrete) = call(&app, "GET", "/kb-a/_count", Value::Null).await;+ assert_eq!(concrete["count"].as_u64(), Some(PER_INDEX as u64));+}++/// A scroll through the alias must walk every document AND report the index+/// each hit truly came from — `(_index, _id)` is the key reindex, migration,+/// backup and CDC consumers use, and ids collide across the members.+#[tokio::test]+async fn a_scroll_through_an_alias_walks_every_member_and_labels_each_hit() {+ let (app, _dir) = app().await;+ seed(&app).await;++ let (st, mut page) = call(+ &app,+ "POST",+ "/kb/_search?scroll=2m",+ json!({"query": {"match_all": {}}, "size": 50}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "scroll start: {page}");+ let mut sid = page["_scroll_id"].as_str().map(str::to_string);+ let mut hits: Vec<(String, String, String)> = Vec::new();++ for _ in 0..16 {+ let page_hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();+ if page_hits.is_empty() {+ break;+ }+ for h in &page_hits {+ hits.push((+ h["_index"].as_str().unwrap_or_default().to_string(),+ h["_id"].as_str().unwrap_or_default().to_string(),+ h["_source"]["home"].as_str().unwrap_or_default().to_string(),+ ));+ }+ let Some(id) = sid.clone() else { break };+ let (st, next) = call(+ &app,+ "POST",+ "/_search/scroll",+ json!({"scroll": "2m", "scroll_id": id}),+ )+ .await;+ assert_eq!(st, StatusCode::OK, "continuation: {next}");+ sid = next["_scroll_id"].as_str().map(str::to_string).or(sid);+ page = next;+ }++ assert_eq!(hits.len(), PER_INDEX * 3, "every member must be walked");++ let mislabelled: Vec<_> = hits+ .iter()+ .filter(|(reported, _, home)| reported != home)+ .take(5)+ .collect();+ assert!(+ mislabelled.is_empty(),+ "hits must report the index they came from, not the alias: {mislabelled:?}"+ );++ let distinct: std::collections::HashSet<(&str, &str)> = hits+ .iter()+ .map(|(i, id, _)| (i.as_str(), id.as_str()))+ .collect();+ assert_eq!(+ distinct.len(),+ PER_INDEX * 3,+ "(_index, _id) must be distinct — ids collide across the members"+ );+}