Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(es-compat): a multi-index alias answered as an alias over one member

xerj-orgaf5ff14cd60c
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 5410cfc45..d25a97e48 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -8605,7 +8605,17 @@ async fn search_impl(
.map(|n| resolve_date_math_index(n.trim()))
.collect();
let raw_names: Vec<&str> = resolved_names_owned.iter().map(|s| s.as_str()).collect();
- let needs_resolve = raw_names.iter().any(|n| *n == "_all" || n.contains('*'));
+ // An ALIAS also needs resolving, and used not to be counted here — which is
+ // why a multi-index alias behaved as an alias over one member on every read
+ // path (#433). `_count` and plain `_search` truncated the same way: three
+ // indices of 40 documents behind one alias answered 40, from the first
+ // member, with no error and a `_count` that agreed with the short result.
+ // The `_index`-stamping problem reported alongside it is downstream of this
+ // — the hits could not carry the right index because the other indices were
+ // never searched.
+ let needs_resolve = raw_names
+ .iter()
+ .any(|n| *n == "_all" || n.contains('*') || state.engine.aliases.contains_key(*n));
let index_names: Vec<String> = if needs_resolve {
let all = state.engine.list_indices().await;
let all_names: Vec<String> = all.into_iter().map(|i| i.name).collect();
@@ -8623,6 +8633,16 @@ async fn search_impl(
resolved.push(name.clone());
}
}
+ } else if let Some(targets) = state.engine.aliases.get(*pattern) {
+ // Expand to the concrete members, exactly as
+ // `resolve_index_selector` already does for every other
+ // endpoint. An alias naming one index resolves to that one, so
+ // the single-member case is unchanged.
+ for name in targets.value() {
+ if !resolved.contains(name) {
+ resolved.push(name.clone());
+ }
+ }
} else {
let s = pattern.to_string();
if !resolved.contains(&s) {
@@ -17951,7 +17971,24 @@ pub async fn count_docs(
body: OptionalJson<Value>,
) -> impl IntoResponse {
// Multi-index / all selector: sum counts from every participating index.
- if index == "_all" || index == "*" || index.contains(',') || index.contains('*') {
+ //
+ // An ALIAS belongs in this branch too. It used not to, so a multi-index
+ // alias fell through to the single-index path and answered one member's
+ // count — three indices of 40 documents behind one alias reported 40, with
+ // no error (#433). This is a THIRD copy of the same resolution, alongside
+ // `resolve_index_selector` and the one inside `search_impl`; they disagreed
+ // about aliases in different ways, which is why `_search` and `_count`
+ // could give different answers for the same selector.
+ let alias_selector = index
+ .split(',')
+ .map(str::trim)
+ .any(|n| state.engine.aliases.contains_key(n));
+ if index == "_all"
+ || index == "*"
+ || index.contains(',')
+ || index.contains('*')
+ || alias_selector
+ {
let all = state.engine.list_indices().await;
let all_names: Vec<String> = all.into_iter().map(|i| i.name).collect();
let wanted: Vec<String> = if index == "_all" || index == "*" {
@@ -17966,6 +18003,12 @@ pub async fn count_docs(
out.push(n.clone());
}
}
+ } else if let Some(targets) = state.engine.aliases.get(pat) {
+ for n in targets.value() {
+ if !out.contains(n) {
+ out.push(n.clone());
+ }
+ }
} else if !out.iter().any(|e| e == pat) {
out.push(pat.to_string());
}
diff --git a/engine/crates/xerj-api/tests/alias_multi_index_reads.rs b/engine/crates/xerj-api/tests/alias_multi_index_reads.rs
new file mode 100644
index 000000000..bf4195e1d
--- /dev/null
+++ b/engine/crates/xerj-api/tests/alias_multi_index_reads.rs
@@ -0,0 +1,203 @@
+//! Issue #433: a multi-index alias answered as an alias over ONE member on
+//! every read path.
+//!
+//! Three indices behind one alias, 40 documents each. `GET /_alias/kb` listed
+//! all three, and `_count`, `_search` and a scroll through the alias all
+//! answered 40 — from the first member, with no error and a count that agreed
+//! with the short result. Not a keyed consumer silently keeping a fraction:
+//! every consumer got a fraction.
+//!
+//! The cause was three independent copies of index-selector resolution.
+//! `resolve_index_selector` expanded aliases; `search_impl`'s own resolver only
+//! looked for `_all` and `*`; `count_docs` gated its whole multi-index branch on
+//! the same two plus a comma or a star. A plain alias fell through both.
+//!
+//! These go through the real HTTP routes because that is the only place the
+//! three resolvers meet.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, serde_json::from_slice(&bytes).unwrap_or(Value::Null))
+}
+
+const PER_INDEX: usize = 40;
+
+async fn seed(app: &axum::Router) {
+ for member in ["kb-a", "kb-b", "kb-c"] {
+ let (st, _) = call(
+ app,
+ "PUT",
+ &format!("/{member}"),
+ json!({"mappings": {"properties": {"v": {"type": "long"}, "home": {"type": "keyword"}}}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create {member}");
+
+ let mut bulk = String::new();
+ for i in 0..PER_INDEX {
+ bulk.push_str(&format!("{}\n", json!({"index": {"_id": i.to_string()}})));
+ bulk.push_str(&format!("{}\n", json!({"v": i, "home": member})));
+ }
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method("POST")
+ .uri(format!("/{member}/_bulk"))
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(bulk))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::OK, "bulk {member}");
+ let (st, _) = call(app, "POST", &format!("/{member}/_refresh"), Value::Null).await;
+ assert_eq!(st, StatusCode::OK);
+ }
+ let (st, _) = call(
+ app,
+ "POST",
+ "/_aliases",
+ json!({"actions": [
+ {"add": {"index": "kb-a", "alias": "kb"}},
+ {"add": {"index": "kb-b", "alias": "kb"}},
+ {"add": {"index": "kb-c", "alias": "kb"}},
+ {"add": {"index": "kb-a", "alias": "solo"}}
+ ]}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "create aliases");
+}
+
+/// `_count` and `_search` must see every member, and a one-member alias must
+/// keep answering for exactly that member.
+#[tokio::test]
+async fn every_read_path_through_a_multi_index_alias_sees_every_member() {
+ let (app, _dir) = app().await;
+ seed(&app).await;
+ let all = (PER_INDEX * 3) as u64;
+
+ let (_, count) = call(&app, "GET", "/kb/_count", Value::Null).await;
+ assert_eq!(
+ count["count"].as_u64(),
+ Some(all),
+ "_count through the alias must cover all three members: {count}"
+ );
+
+ let (_, search) = call(
+ &app,
+ "POST",
+ "/kb/_search",
+ json!({"query": {"match_all": {}}, "size": 0}),
+ )
+ .await;
+ assert_eq!(
+ search["hits"]["total"]["value"].as_u64(),
+ Some(all),
+ "_search through the alias must cover all three members: {search}"
+ );
+
+ // The single-member case is the one a narrowing fix would break.
+ let (_, solo) = call(&app, "GET", "/solo/_count", Value::Null).await;
+ assert_eq!(
+ solo["count"].as_u64(),
+ Some(PER_INDEX as u64),
+ "an alias over one index still answers for that index: {solo}"
+ );
+ let (_, concrete) = call(&app, "GET", "/kb-a/_count", Value::Null).await;
+ assert_eq!(concrete["count"].as_u64(), Some(PER_INDEX as u64));
+}
+
+/// A scroll through the alias must walk every document AND report the index
+/// each hit truly came from — `(_index, _id)` is the key reindex, migration,
+/// backup and CDC consumers use, and ids collide across the members.
+#[tokio::test]
+async fn a_scroll_through_an_alias_walks_every_member_and_labels_each_hit() {
+ let (app, _dir) = app().await;
+ seed(&app).await;
+
+ let (st, mut page) = call(
+ &app,
+ "POST",
+ "/kb/_search?scroll=2m",
+ json!({"query": {"match_all": {}}, "size": 50}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "scroll start: {page}");
+ let mut sid = page["_scroll_id"].as_str().map(str::to_string);
+ let mut hits: Vec<(String, String, String)> = Vec::new();
+
+ for _ in 0..16 {
+ let page_hits = page["hits"]["hits"].as_array().cloned().unwrap_or_default();
+ if page_hits.is_empty() {
+ break;
+ }
+ for h in &page_hits {
+ hits.push((
+ h["_index"].as_str().unwrap_or_default().to_string(),
+ h["_id"].as_str().unwrap_or_default().to_string(),
+ h["_source"]["home"].as_str().unwrap_or_default().to_string(),
+ ));
+ }
+ let Some(id) = sid.clone() else { break };
+ let (st, next) = call(
+ &app,
+ "POST",
+ "/_search/scroll",
+ json!({"scroll": "2m", "scroll_id": id}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "continuation: {next}");
+ sid = next["_scroll_id"].as_str().map(str::to_string).or(sid);
+ page = next;
+ }
+
+ assert_eq!(hits.len(), PER_INDEX * 3, "every member must be walked");
+
+ let mislabelled: Vec<_> = hits
+ .iter()
+ .filter(|(reported, _, home)| reported != home)
+ .take(5)
+ .collect();
+ assert!(
+ mislabelled.is_empty(),
+ "hits must report the index they came from, not the alias: {mislabelled:?}"
+ );
+
+ let distinct: std::collections::HashSet<(&str, &str)> = hits
+ .iter()
+ .map(|(i, id, _)| (i.as_str(), id.as_str()))
+ .collect();
+ assert_eq!(
+ distinct.len(),
+ PER_INDEX * 3,
+ "(_index, _id) must be distinct — ids collide across the members"
+ );
+}