AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(api,engine): writes through multi-index aliases follow is_write_index (#2326)
ordinary beingd6ef7fbb3c21
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 37765b300..e2374ea3b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -669,6 +669,14 @@ samuelmbabhazi, elewashy, WYSTAF, kamwoh, mgtf, mariomarcolongo and Jun516.nothing (including 0/N), and when a phase with no total has been silentfor 10 minutes.+- Writing through a multi-index alias follows ES's write-index rule: one+ backing index, or exactly one member carrying `is_write_index: true`; an+ ambiguous alias is refused with `illegal_argument_exception` instead of+ silently writing to the first backing index. The same resolution covers+ `_bulk` items and the id-addressed GET/HEAD/DELETE, so a document written+ through the alias is readable through it, and the response `_index` now+ names the concrete index.+- Semantic `xerj autoindex` against `--embed-mode proxy` resumes after amid-run failure and does not re-embed documents already indexed, when thebackend, configured model, endpoint and confirmed vector width arediff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex e0a960872..f0baf2c3a 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -3973,12 +3973,32 @@ pub(crate) fn document_body(doc: &Value) -> bytes::Bytes {/// JSON extractor's extra validation pass and parse directly with simd_json //// serde_json. No schema validation is performed here; the engine layer handles/// all type coercion. This shaves ~15–30 µs of extractor overhead per doc.+/// Resolve an alias to the concrete index a document write must land in.+/// A concrete name (or a name that will be auto-created) is returned+/// unchanged; an ambiguous alias becomes ES's 400+/// `illegal_argument_exception` instead of silently using the first backing+/// index (`GET /{index}` does resolve aliases that way, but a write may not).+fn write_target_or_refusal(+ state: &AppState,+ index: &str,+) -> std::result::Result<String, axum::response::Response> {+ match state.engine.write_target(index) {+ Ok(Some(target)) => Ok(target),+ Ok(None) => Ok(index.to_string()),+ Err(reason) => Err(es_bad_request("illegal_argument_exception", &reason, None)),+ }+}+pub async fn index_doc_auto(State(state): State<AppState>,Path(index): Path<String>,Query(params): Query<IndexDocAutoParams>,body: bytes::Bytes,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };let doc = match parse_indexed_document_body(&body) {Ok(v) => v,Err(e) => return ApiError::new(e).into_response(),@@ -4132,6 +4152,10 @@ pub async fn index_doc(Query(params): Query<IndexDocParams>,body: bytes::Bytes,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };// Size check before the DOM exists. See [`parse_indexed_document_body`].let doc = match parse_indexed_document_body(&body) {Ok(v) => v,@@ -4343,6 +4367,10 @@ pub async fn create_doc(Query(params): Query<CreateDocParams>,body: bytes::Bytes,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };let doc = match parse_indexed_document_body(&body) {Ok(v) => v,Err(e) => return ApiError::new(e).into_response(),@@ -4643,6 +4671,10 @@ pub async fn get_doc(Path((index, id)): Path<(String, String)>,Query(params): Query<GetDocParams>,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };// A closed index rejects read/search ops with ES index_closed_exception.// Strict membership only — frozen/open indices are unaffected.if state.engine.closed_indices.contains_key(&index) {@@ -4802,6 +4834,10 @@ pub async fn delete_doc(Path((index, id)): Path<(String, String)>,Query(params): Query<DeleteDocParams>,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };// Wire → internal seq translation, same convention as the index path.let if_seq_no_internal = params.if_seq_no.map(|s| s.saturating_add(1));@@ -20706,6 +20742,7 @@ fn bulk_item_error_type(e: &str) -> &'static str {|| e.contains("missing document body")|| e.contains("unknown action type")|| e.contains("no write index is defined")+ || e.contains("has more than one write index")|| e.starts_with("pipeline with id"){"illegal_argument_exception"@@ -23192,6 +23229,10 @@ pub async fn update_doc(Query(query_params): Query<UpdateDocParams>,body: bytes::Bytes,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };// Before `get_or_create_index`, so a refused body does not create an// index and does not read or write the document.let body = match parse_update_body(&body) {@@ -24094,6 +24135,10 @@ pub async fn head_doc(State(state): State<AppState>,Path((index, id)): Path<(String, String)>,) -> impl IntoResponse {+ let index = match write_target_or_refusal(&state, &index) {+ Ok(index) => index,+ Err(resp) => return resp,+ };let idx = match state.engine.get_index(&index) {Ok(i) => i,Err(_) => return StatusCode::NOT_FOUND.into_response(),diff --git a/engine/crates/xerj-api/tests/alias_write_routing.rs b/engine/crates/xerj-api/tests/alias_write_routing.rsnew file mode 100644index 000000000..59be72501--- /dev/null+++ b/engine/crates/xerj-api/tests/alias_write_routing.rs@@ -0,0 +1,229 @@+//! Writes through a multi-index alias must follow ES's write-index rule.+//!+//! `Engine::get_index` resolves an alias to its FIRST backing index (fine for+//! reads by name), and every single-document write used to reach the store+//! through it. A two-index alias therefore accepted `PUT /alias/_doc/1` with+//! 201 and wrote the document to whichever member happened to be listed+//! first, even when the other member carried `is_write_index: true`. ES+//! refuses the ambiguous shape and routes the designated one.+//!+//! All document writes now resolve through `Engine::write_target`: one backing+//! index, or exactly one `is_write_index: true`; otherwise ES's 400. The bulk+//! path uses the same helper and rewrites the item's target so its result and+//! storage agree.+//!+//! Elasticsearch is referenced for wire semantics only; no ES code is+//! reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++async fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {+ let mut req = Request::builder().method(method).uri(path);+ let body = if body.is_null() {+ Body::empty()+ } else {+ req = req.header("content-type", "application/json");+ Body::from(body.to_string())+ };+ let response = app.clone().oneshot(req.body(body).unwrap()).await.unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn ndjson(app: &axum::Router, path: &str, body: String) -> (StatusCode, Value) {+ let response = app+ .clone()+ .oneshot(+ Request::builder()+ .method("POST")+ .uri(path)+ .header("content-type", "application/x-ndjson")+ .body(Body::from(body))+ .unwrap(),+ )+ .await+ .unwrap();+ let status = response.status();+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+}++async fn docs_in(app: &axum::Router, index: &str) -> u64 {+ let (status, body) = call(app, "POST", &format!("/{index}/_refresh"), Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(app, "GET", &format!("/{index}/_count"), Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ body["count"].as_u64().unwrap_or(0)+}++fn refusal(body: &Value) -> &str {+ body.pointer("/error/root_cause/0/reason")+ .and_then(Value::as_str)+ .or_else(|| body.get("error").and_then(Value::as_str))+ .unwrap_or_default()+}++#[tokio::test]+async fn writes_follow_the_alias_write_index_and_refuse_ambiguity() {+ let (app, _dir) = app().await;+ for ix in ["i1", "i2"] {+ let (status, body) = call(&app, "PUT", &format!("/{ix}"), json!({})).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ }+ let (status, body) = call(+ &app,+ "POST",+ "/_aliases",+ json!({"actions": [+ {"add": {"index": "i1", "alias": "multi"}},+ {"add": {"index": "i2", "alias": "multi"}},+ ]}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");++ // Ambiguous: more than one backing index, none designated. ES refuses and+ // nothing is written.+ let (status, body) = call(&app, "PUT", "/multi/_doc/1", json!({"t": "one"})).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert_eq!(+ body.pointer("/error/type").and_then(Value::as_str),+ Some("illegal_argument_exception"),+ "{body}"+ );+ assert!(+ refusal(&body).contains("no write index is defined for alias [multi]"),+ "{body}"+ );+ assert_eq!(+ docs_in(&app, "i1").await,+ 0,+ "no index received the refused write"+ );+ assert_eq!(docs_in(&app, "i2").await, 0);++ // Designate i2: the write must land there, not on the first member.+ let (status, body) = call(+ &app,+ "PUT",+ "/i2/_alias/multi",+ json!({"is_write_index": true}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(&app, "PUT", "/multi/_doc/2", json!({"t": "two"})).await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ assert_eq!(+ body["_index"], "i2",+ "response names the concrete index: {body}"+ );+ let (status, body) = call(&app, "GET", "/i2/_doc/2", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(&app, "GET", "/multi/_doc/2", Value::Null).await;+ assert_eq!(+ status,+ StatusCode::OK,+ "get via the alias uses the write index: {body}"+ );+ assert_eq!(body["_index"], "i2", "{body}");+ let (status, _) = call(&app, "HEAD", "/multi/_doc/2", Value::Null).await;+ assert_eq!(+ status,+ StatusCode::OK,+ "head via the alias uses the write index"+ );+ let (status, _) = call(&app, "GET", "/i1/_doc/2", Value::Null).await;+ assert_eq!(status, StatusCode::NOT_FOUND);+ assert_eq!(docs_in(&app, "i1").await, 0);+ assert_eq!(docs_in(&app, "i2").await, 1);++ // Every document-write entry point takes the same route.+ let (status, body) = call(&app, "POST", "/multi/_doc", json!({"t": "auto"})).await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ assert_eq!(body["_index"], "i2", "{body}");++ let (status, body) = call(&app, "PUT", "/multi/_create/3", json!({"t": "created"})).await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ assert_eq!(body["_index"], "i2", "{body}");++ let (status, body) = call(+ &app,+ "POST",+ "/multi/_update/2",+ json!({"doc": {"t": "updated"}}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["_index"], "i2", "{body}");+ let (_, doc) = call(&app, "GET", "/i2/_doc/2", Value::Null).await;+ assert_eq!(doc["_source"]["t"], "updated", "{doc}");++ let (status, body) = call(&app, "DELETE", "/multi/_doc/2", Value::Null).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["_index"], "i2", "{body}");++ // Bulk resolves and rewrites the target too, including the item result.+ let (status, body) = ndjson(+ &app,+ "/_bulk",+ "{\"index\":{\"_index\":\"multi\",\"_id\":\"9\"}}\n{\"t\":\"bulk\"}\n".to_string(),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["items"][0]["index"]["_index"], "i2", "{body}");+ assert_eq!(body["items"][0]["index"]["status"], 201, "{body}");++ // A one-index alias needs no designation.+ let (status, body) = call(+ &app,+ "POST",+ "/_aliases",+ json!({"actions": [{"add": {"index": "i1", "alias": "solo"}}]}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(&app, "PUT", "/solo/_doc/7", json!({"t": "solo"})).await;+ assert_eq!(status, StatusCode::CREATED, "{body}");+ assert_eq!(body["_index"], "i1", "{body}");++ // Two designated write indices are refused like the ambiguous shape.+ let (status, body) = call(+ &app,+ "PUT",+ "/i1/_alias/multi",+ json!({"is_write_index": true}),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ let (status, body) = call(&app, "PUT", "/multi/_doc/10", json!({"t": "ten"})).await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert!(+ refusal(&body).contains("has more than one write index"),+ "{body}"+ );+}diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex e808f8f4c..222363914 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -1202,10 +1202,10 @@ pub async fn process_bulk_with_opts(// the write index via `is_write_index:true` in the alias metadata.let url_require_alias = opts.require_alias.unwrap_or(false);let default_source_req = opts.default_source_req.clone();- parsed.retain(|action| {+ parsed.retain_mut(|action| {let eff_require_alias = action.require_alias.unwrap_or(url_require_alias);- let target = &action.target_index;- let is_alias = engine.aliases.contains_key(target);+ let target = action.target_index.clone();+ let is_alias = engine.aliases.contains_key(&target);if eff_require_alias && !is_alias {items[action.item_index] = Some(BulkItemResult {action: action.action_type.clone(),@@ -1216,43 +1216,32 @@ pub async fn process_bulk_with_opts(error: Some(format!("no such index [{target}] and [require_alias] request flag is [true] and [{target}] is not an alias")),- get_source: None,+ get_source: None,});errors = true;return false;}if is_alias {- if let Some(backings) = engine.aliases.get(target) {- if backings.len() > 1 {- // Look up per-backing is_write_index in alias metadata.- // The metadata map is keyed by backing index name and- // stores `{alias: {is_write_index: true, ...}, ...}`.- let has_write_target = backings.iter().any(|backing| {- engine- .index_alias_metadata- .get(backing)- .and_then(|v| {- v.get(target).and_then(|meta| {- meta.get("is_write_index").and_then(Value::as_bool)- })- })- .unwrap_or(false)+ // One write-target definition for every write path: a single+ // backing index or exactly one `is_write_index:true`. The old code+ // only checked that SOME write index existed and then left the+ // target as the alias, so `get_or_create_index` wrote to the FIRST+ // backing index and ignored the designation.+ match engine.write_target(&target) {+ Ok(Some(concrete)) => action.target_index = concrete,+ Ok(None) => {}+ Err(reason) => {+ items[action.item_index] = Some(BulkItemResult {+ action: action.action_type.clone(),+ index: target.clone(),+ id: action.doc_id.clone().unwrap_or_default(),+ status: 400,+ result: None,+ error: Some(reason),+ get_source: None,});- if !has_write_target {- items[action.item_index] = Some(BulkItemResult {- action: action.action_type.clone(),- index: target.clone(),- id: action.doc_id.clone().unwrap_or_default(),- status: 400,- result: None,- error: Some(format!(- "no write index is defined for alias [{target}]. The write index may be explicitly disabled using is_write_index=false or the alias points to multiple indices without one being designated as a write index"- )),- get_source: None,- });- errors = true;- return false;- }+ errors = true;+ return false;}}}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 1be4b60b1..43b006afa 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -3318,6 +3318,46 @@ impl Engine {vec![name.to_string()]}+ /// The concrete index a write to `name` must target when `name` is an+ /// alias: a single backing index, or exactly one marked+ /// `is_write_index: true`. `Ok(None)` means `name` is not an alias (a+ /// concrete index or a name the caller may auto-create). More than one+ /// candidate is ambiguous and refused with ES's message.+ ///+ /// `get_index` deliberately resolves an alias to its FIRST backing index,+ /// which is fine for reads by name but wrong for a write: a two-index+ /// alias silently sent the document to whichever member was listed first,+ /// ignoring `is_write_index`. Write paths resolve through this function+ /// before they touch a mapping, pipeline, or store.+ pub fn write_target(&self, name: &str) -> std::result::Result<Option<String>, String> {+ let Some(backings) = self.aliases.get(name) else {+ return Ok(None);+ };+ if backings.len() == 1 {+ return Ok(Some(backings[0].clone()));+ }+ let mut writes = backings.iter().filter(|backing| {+ let Some(meta) = self.index_alias_metadata.get(backing.as_str()) else {+ return false;+ };+ meta.get(name)+ .and_then(|alias_meta| alias_meta.get("is_write_index"))+ .and_then(serde_json::Value::as_bool)+ .unwrap_or(false)+ });+ let first = writes.next().cloned();+ let second = writes.next();+ match (first, second) {+ (Some(target), None) => Ok(Some(target)),+ (None, _) => Err(format!(+ "no write index is defined for alias [{name}]. The write index may be \+ explicitly disabled using is_write_index=false or the alias points to \+ multiple indices without one being designated as a write index"+ )),+ (Some(_), Some(_)) => Err(format!("alias [{name}] has more than one write index")),+ }+ }+/// Delete an index and all its data.////// Also drops any aliases that pointed only at this index (matching ES