Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(api,engine): writes through multi-index aliases follow is_write_index (#2326)

ordinary beingd6ef7fbb3c21
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 37765b300..e2374ea3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -669,6 +669,14 @@ samuelmbabhazi, elewashy, WYSTAF, kamwoh, mgtf, mariomarcolongo and Jun516.
nothing (including 0/N), and when a phase with no total has been silent
for 10 minutes.
+- Writing through a multi-index alias follows ES's write-index rule: one
+ backing index, or exactly one member carrying `is_write_index: true`; an
+ ambiguous alias is refused with `illegal_argument_exception` instead of
+ silently writing to the first backing index. The same resolution covers
+ `_bulk` items and the id-addressed GET/HEAD/DELETE, so a document written
+ through the alias is readable through it, and the response `_index` now
+ names the concrete index.
+
- Semantic `xerj autoindex` against `--embed-mode proxy` resumes after a
mid-run failure and does not re-embed documents already indexed, when the
backend, configured model, endpoint and confirmed vector width are
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index e0a960872..f0baf2c3a 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -3973,12 +3973,32 @@ pub(crate) fn document_body(doc: &Value) -> bytes::Bytes {
/// JSON extractor's extra validation pass and parse directly with simd_json /
/// serde_json. No schema validation is performed here; the engine layer handles
/// all type coercion. This shaves ~15–30 µs of extractor overhead per doc.
+/// Resolve an alias to the concrete index a document write must land in.
+/// A concrete name (or a name that will be auto-created) is returned
+/// unchanged; an ambiguous alias becomes ES's 400
+/// `illegal_argument_exception` instead of silently using the first backing
+/// index (`GET /{index}` does resolve aliases that way, but a write may not).
+fn write_target_or_refusal(
+ state: &AppState,
+ index: &str,
+) -> std::result::Result<String, axum::response::Response> {
+ match state.engine.write_target(index) {
+ Ok(Some(target)) => Ok(target),
+ Ok(None) => Ok(index.to_string()),
+ Err(reason) => Err(es_bad_request("illegal_argument_exception", &reason, None)),
+ }
+}
+
pub async fn index_doc_auto(
State(state): State<AppState>,
Path(index): Path<String>,
Query(params): Query<IndexDocAutoParams>,
body: bytes::Bytes,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
let doc = match parse_indexed_document_body(&body) {
Ok(v) => v,
Err(e) => return ApiError::new(e).into_response(),
@@ -4132,6 +4152,10 @@ pub async fn index_doc(
Query(params): Query<IndexDocParams>,
body: bytes::Bytes,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
// Size check before the DOM exists. See [`parse_indexed_document_body`].
let doc = match parse_indexed_document_body(&body) {
Ok(v) => v,
@@ -4343,6 +4367,10 @@ pub async fn create_doc(
Query(params): Query<CreateDocParams>,
body: bytes::Bytes,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
let doc = match parse_indexed_document_body(&body) {
Ok(v) => v,
Err(e) => return ApiError::new(e).into_response(),
@@ -4643,6 +4671,10 @@ pub async fn get_doc(
Path((index, id)): Path<(String, String)>,
Query(params): Query<GetDocParams>,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
// A closed index rejects read/search ops with ES index_closed_exception.
// Strict membership only — frozen/open indices are unaffected.
if state.engine.closed_indices.contains_key(&index) {
@@ -4802,6 +4834,10 @@ pub async fn delete_doc(
Path((index, id)): Path<(String, String)>,
Query(params): Query<DeleteDocParams>,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
// Wire → internal seq translation, same convention as the index path.
let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));
@@ -20706,6 +20742,7 @@ fn bulk_item_error_type(e: &str) -> &'static str {
|| e.contains("missing document body")
|| e.contains("unknown action type")
|| e.contains("no write index is defined")
+ || e.contains("has more than one write index")
|| e.starts_with("pipeline with id")
{
"illegal_argument_exception"
@@ -23192,6 +23229,10 @@ pub async fn update_doc(
Query(query_params): Query<UpdateDocParams>,
body: bytes::Bytes,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
// Before `get_or_create_index`, so a refused body does not create an
// index and does not read or write the document.
let body = match parse_update_body(&body) {
@@ -24094,6 +24135,10 @@ pub async fn head_doc(
State(state): State<AppState>,
Path((index, id)): Path<(String, String)>,
) -> impl IntoResponse {
+ let index = match write_target_or_refusal(&state, &index) {
+ Ok(index) => index,
+ Err(resp) => return resp,
+ };
let idx = match state.engine.get_index(&index) {
Ok(i) => i,
Err(_) => return StatusCode::NOT_FOUND.into_response(),
diff --git a/engine/crates/xerj-api/tests/alias_write_routing.rs b/engine/crates/xerj-api/tests/alias_write_routing.rs
new file mode 100644
index 000000000..59be72501
--- /dev/null
+++ b/engine/crates/xerj-api/tests/alias_write_routing.rs
@@ -0,0 +1,229 @@
+//! Writes through a multi-index alias must follow ES's write-index rule.
+//!
+//! `Engine::get_index` resolves an alias to its FIRST backing index (fine for
+//! reads by name), and every single-document write used to reach the store
+//! through it. A two-index alias therefore accepted `PUT /alias/_doc/1` with
+//! 201 and wrote the document to whichever member happened to be listed
+//! first, even when the other member carried `is_write_index: true`. ES
+//! refuses the ambiguous shape and routes the designated one.
+//!
+//! All document writes now resolve through `Engine::write_target`: one backing
+//! index, or exactly one `is_write_index: true`; otherwise ES's 400. The bulk
+//! path uses the same helper and rewrites the item's target so its result and
+//! storage agree.
+//!
+//! Elasticsearch is referenced for wire semantics only; no ES code is
+//! reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+async fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let mut req = Request::builder().method(method).uri(path);
+ let body = if body.is_null() {
+ Body::empty()
+ } else {
+ req = req.header("content-type", "application/json");
+ Body::from(body.to_string())
+ };
+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn ndjson(app: &axum::Router, path: &str, body: String) -> (StatusCode, Value) {
+ let response = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method("POST")
+ .uri(path)
+ .header("content-type", "application/x-ndjson")
+ .body(Body::from(body))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+}
+
+async fn docs_in(app: &axum::Router, index: &str) -> u64 {
+ let (status, body) = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ body["count"].as_u64().unwrap_or(0)
+}
+
+fn refusal(body: &Value) -> &str {
+ body.pointer("/error/root_cause/0/reason")
+ .and_then(Value::as_str)
+ .or_else(|| body.get("error").and_then(Value::as_str))
+ .unwrap_or_default()
+}
+
+#[tokio::test]
+async fn writes_follow_the_alias_write_index_and_refuse_ambiguity() {
+ let (app, _dir) = app().await;
+ for ix in ["i1", "i2"] {
+ let (status, body) = call(&app, "PUT", &format!("/{ix}"), json!({})).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ }
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/_aliases",
+ json!({"actions": [
+ {"add": {"index": "i1", "alias": "multi"}},
+ {"add": {"index": "i2", "alias": "multi"}},
+ ]}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+
+ // Ambiguous: more than one backing index, none designated. ES refuses and
+ // nothing is written.
+ let (status, body) = call(&app, "PUT", "/multi/_doc/1", json!({"t": "one"})).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert_eq!(
+ body.pointer("/error/type").and_then(Value::as_str),
+ Some("illegal_argument_exception"),
+ "{body}"
+ );
+ assert!(
+ refusal(&body).contains("no write index is defined for alias [multi]"),
+ "{body}"
+ );
+ assert_eq!(
+ docs_in(&app, "i1").await,
+ 0,
+ "no index received the refused write"
+ );
+ assert_eq!(docs_in(&app, "i2").await, 0);
+
+ // Designate i2: the write must land there, not on the first member.
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/i2/_alias/multi",
+ json!({"is_write_index": true}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(&app, "PUT", "/multi/_doc/2", json!({"t": "two"})).await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ assert_eq!(
+ body["_index"], "i2",
+ "response names the concrete index: {body}"
+ );
+ let (status, body) = call(&app, "GET", "/i2/_doc/2", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(&app, "GET", "/multi/_doc/2", Value::Null).await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "get via the alias uses the write index: {body}"
+ );
+ assert_eq!(body["_index"], "i2", "{body}");
+ let (status, _) = call(&app, "HEAD", "/multi/_doc/2", Value::Null).await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "head via the alias uses the write index"
+ );
+ let (status, _) = call(&app, "GET", "/i1/_doc/2", Value::Null).await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+ assert_eq!(docs_in(&app, "i1").await, 0);
+ assert_eq!(docs_in(&app, "i2").await, 1);
+
+ // Every document-write entry point takes the same route.
+ let (status, body) = call(&app, "POST", "/multi/_doc", json!({"t": "auto"})).await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ assert_eq!(body["_index"], "i2", "{body}");
+
+ let (status, body) = call(&app, "PUT", "/multi/_create/3", json!({"t": "created"})).await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ assert_eq!(body["_index"], "i2", "{body}");
+
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/multi/_update/2",
+ json!({"doc": {"t": "updated"}}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["_index"], "i2", "{body}");
+ let (_, doc) = call(&app, "GET", "/i2/_doc/2", Value::Null).await;
+ assert_eq!(doc["_source"]["t"], "updated", "{doc}");
+
+ let (status, body) = call(&app, "DELETE", "/multi/_doc/2", Value::Null).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["_index"], "i2", "{body}");
+
+ // Bulk resolves and rewrites the target too, including the item result.
+ let (status, body) = ndjson(
+ &app,
+ "/_bulk",
+ "{\"index\":{\"_index\":\"multi\",\"_id\":\"9\"}}\n{\"t\":\"bulk\"}\n".to_string(),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["items"][0]["index"]["_index"], "i2", "{body}");
+ assert_eq!(body["items"][0]["index"]["status"], 201, "{body}");
+
+ // A one-index alias needs no designation.
+ let (status, body) = call(
+ &app,
+ "POST",
+ "/_aliases",
+ json!({"actions": [{"add": {"index": "i1", "alias": "solo"}}]}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(&app, "PUT", "/solo/_doc/7", json!({"t": "solo"})).await;
+ assert_eq!(status, StatusCode::CREATED, "{body}");
+ assert_eq!(body["_index"], "i1", "{body}");
+
+ // Two designated write indices are refused like the ambiguous shape.
+ let (status, body) = call(
+ &app,
+ "PUT",
+ "/i1/_alias/multi",
+ json!({"is_write_index": true}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ let (status, body) = call(&app, "PUT", "/multi/_doc/10", json!({"t": "ten"})).await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert!(
+ refusal(&body).contains("has more than one write index"),
+ "{body}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index e808f8f4c..222363914 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -1202,10 +1202,10 @@ pub async fn process_bulk_with_opts(
// the write index via `is_write_index:true` in the alias metadata.
let url_require_alias = opts.require_alias.unwrap_or(false);
let default_source_req = opts.default_source_req.clone();
- parsed.retain(|action| {
+ parsed.retain_mut(|action| {
let eff_require_alias = action.require_alias.unwrap_or(url_require_alias);
- let target = &action.target_index;
- let is_alias = engine.aliases.contains_key(target);
+ let target = action.target_index.clone();
+ let is_alias = engine.aliases.contains_key(&target);
if eff_require_alias && !is_alias {
items[action.item_index] = Some(BulkItemResult {
action: action.action_type.clone(),
@@ -1216,43 +1216,32 @@ pub async fn process_bulk_with_opts(
error: Some(format!(
"no such index [{target}] and [require_alias] request flag is [true] and [{target}] is not an alias"
)),
- get_source: None,
+ get_source: None,
});
errors = true;
return false;
}
if is_alias {
- if let Some(backings) = engine.aliases.get(target) {
- if backings.len() > 1 {
- // Look up per-backing is_write_index in alias metadata.
- // The metadata map is keyed by backing index name and
- // stores `{alias: {is_write_index: true, ...}, ...}`.
- let has_write_target = backings.iter().any(|backing| {
- engine
- .index_alias_metadata
- .get(backing)
- .and_then(|v| {
- v.get(target).and_then(|meta| {
- meta.get("is_write_index").and_then(Value::as_bool)
- })
- })
- .unwrap_or(false)
+ // One write-target definition for every write path: a single
+ // backing index or exactly one `is_write_index:true`. The old code
+ // only checked that SOME write index existed and then left the
+ // target as the alias, so `get_or_create_index` wrote to the FIRST
+ // backing index and ignored the designation.
+ match engine.write_target(&target) {
+ Ok(Some(concrete)) => action.target_index = concrete,
+ Ok(None) => {}
+ Err(reason) => {
+ items[action.item_index] = Some(BulkItemResult {
+ action: action.action_type.clone(),
+ index: target.clone(),
+ id: action.doc_id.clone().unwrap_or_default(),
+ status: 400,
+ result: None,
+ error: Some(reason),
+ get_source: None,
});
- if !has_write_target {
- items[action.item_index] = Some(BulkItemResult {
- action: action.action_type.clone(),
- index: target.clone(),
- id: action.doc_id.clone().unwrap_or_default(),
- status: 400,
- result: None,
- error: Some(format!(
- "no write index is defined for alias [{target}]. The write index may be explicitly disabled using is_write_index=false or the alias points to multiple indices without one being designated as a write index"
- )),
- get_source: None,
- });
- errors = true;
- return false;
- }
+ errors = true;
+ return false;
}
}
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 1be4b60b1..43b006afa 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -3318,6 +3318,46 @@ impl Engine {
vec![name.to_string()]
}
+ /// The concrete index a write to `name` must target when `name` is an
+ /// alias: a single backing index, or exactly one marked
+ /// `is_write_index: true`. `Ok(None)` means `name` is not an alias (a
+ /// concrete index or a name the caller may auto-create). More than one
+ /// candidate is ambiguous and refused with ES's message.
+ ///
+ /// `get_index` deliberately resolves an alias to its FIRST backing index,
+ /// which is fine for reads by name but wrong for a write: a two-index
+ /// alias silently sent the document to whichever member was listed first,
+ /// ignoring `is_write_index`. Write paths resolve through this function
+ /// before they touch a mapping, pipeline, or store.
+ pub fn write_target(&self, name: &str) -> std::result::Result<Option<String>, String> {
+ let Some(backings) = self.aliases.get(name) else {
+ return Ok(None);
+ };
+ if backings.len() == 1 {
+ return Ok(Some(backings[0].clone()));
+ }
+ let mut writes = backings.iter().filter(|backing| {
+ let Some(meta) = self.index_alias_metadata.get(backing.as_str()) else {
+ return false;
+ };
+ meta.get(name)
+ .and_then(|alias_meta| alias_meta.get("is_write_index"))
+ .and_then(serde_json::Value::as_bool)
+ .unwrap_or(false)
+ });
+ let first = writes.next().cloned();
+ let second = writes.next();
+ match (first, second) {
+ (Some(target), None) => Ok(Some(target)),
+ (None, _) => Err(format!(
+ "no write index is defined for alias [{name}]. The write index may be \
+ explicitly disabled using is_write_index=false or the alias points to \
+ multiple indices without one being designated as a write index"
+ )),
+ (Some(_), Some(_)) => Err(format!("alias [{name}] has more than one write index")),
+ }
+ }
+
/// Delete an index and all its data.
///
/// Also drops any aliases that pointed only at this index (matching ES