AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(bulk): apply items that share an _id in request order (#2650)
Amartya Vikram Singhbd9f0510f56d
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex fcaf4d17a..504989695 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -1480,6 +1480,34 @@ pub async fn process_bulk_with_opts(props};+ // A bulk body is an ordered log: ES applies its items in request order,+ // so `delete 1` followed by `index 1` leaves the doc present, and+ // `index 1 {v:1}`, `delete 1`, `index 1 {v:2}` leaves `{v:2}`. The+ // partition below runs its groups as phases (semantic, then turbo+ // index, then everything else), which reorders items across groups.+ // That is harmless while each `(index, _id)` lives in one group, but a+ // key that a non-turbo action (delete / update / create / CAS / routed+ // index) also touches would see its plain `index` items jump ahead and+ // the later delete win — the doc is gone and every item said success.+ // Collect those keys up front and keep all of their items on the+ // sequential path, which executes in request order.+ let sequential_keys: std::collections::HashSet<(String, String)> = parsed+ .iter()+ .filter(|a| {+ a.action_type != "index"+ || a.if_seq_no.is_some()+ || a.if_primary_term.is_some()+ || a.routing.is_some()+ || a.dynamic_templates.is_some()+ || a.pipeline.is_some()+ })+ .filter_map(|a| {+ a.doc_id+ .as_ref()+ .map(|id| (a.target_index.clone(), id.clone()))+ })+ .collect();+for mut action in parsed {// ── Numeric / boolean type enforcement (issue #781) ──────────────//@@ -1560,7 +1588,11 @@ pub async fn process_bulk_with_opts(&& action.dynamic_templates.is_none()&& action.pipeline.is_none()&& !has_strict_date- && !has_dynamic_copy;+ && !has_dynamic_copy+ && (sequential_keys.is_empty()+ || !action.doc_id.as_ref().is_some_and(|id| {+ sequential_keys.contains(&(action.target_index.clone(), id.clone()))+ }));let is_plain_index = has_plain_metadata && !has_embedding;if is_plain_index {match action.doc_id {diff --git a/engine/crates/xerj-engine/tests/bulk_same_id_order.rs b/engine/crates/xerj-engine/tests/bulk_same_id_order.rsnew file mode 100644index 000000000..4a3996847--- /dev/null+++ b/engine/crates/xerj-engine/tests/bulk_same_id_order.rs@@ -0,0 +1,175 @@+//! `_bulk` must apply items that share an `(index, _id)` in request order.+//!+//! The bulk executor partitions items into phases — plain `index` items run+//! as one turbo group, `delete` / `update` / `create` / CAS / routed items run+//! afterwards — which reorders items across groups. When one `_id` was touched+//! by both kinds in one request, its plain `index` items jumped ahead of an+//! earlier `delete` or `update`, so the final document did not match the+//! request: an `index`, `delete`, `index` sequence ended with the document+//! deleted, while every item reported success.++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::bulk::process_bulk;+use xerj_engine::Engine;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++fn seed(engine: &Engine, name: &str) {+ let mut schema = Schema::empty();+ schema.fields.push(FieldConfig::new("v", FieldType::Long));+ engine.create_index(name, schema).expect("create index");+}++fn ndjson(lines: &[Value]) -> String {+ let mut out = String::new();+ for l in lines {+ out.push_str(&l.to_string());+ out.push('\n');+ }+ out+}++async fn source(engine: &Engine, index: &str, id: &str) -> Option<Value> {+ engine+ .get_index(index)+ .expect("get index")+ .get_document(id)+ .await+ .expect("lookup")+}++/// FAIL-BEFORE: both `index` items ran in the turbo phase, then the `delete`+/// ran last and removed the document that the request's final item wrote.+#[tokio::test]+async fn index_delete_index_keeps_the_last_write() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ let body = ndjson(&[+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 1}),+ json!({"delete": {"_index": "t", "_id": "1"}}),+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 2}),+ ]);+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "bulk must not error: {:?}", result.items);+ let statuses: Vec<u16> = result.items.iter().map(|i| i.status).collect();+ assert_eq!(statuses, vec![201, 200, 201], "items: {:?}", result.items);++ let doc = source(&engine, "t", "1")+ .await+ .expect("the final `index` item must leave the document present");+ assert_eq!(doc.get("v"), Some(&json!(2)));+}++/// FAIL-BEFORE: the `index` ran first and the earlier `delete` removed it.+#[tokio::test]+async fn delete_then_index_leaves_the_document() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");+ process_bulk(+ &engine,+ None,+ &ndjson(&[+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 1}),+ ]),+ )+ .await;++ let body = ndjson(&[+ json!({"delete": {"_index": "t", "_id": "1"}}),+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 7}),+ ]);+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "bulk must not error: {:?}", result.items);+ assert_eq!(result.items[1].status, 201, "re-index after delete creates");++ let doc = source(&engine, "t", "1")+ .await+ .expect("document indexed after the delete must exist");+ assert_eq!(doc.get("v"), Some(&json!(7)));+}++/// FAIL-BEFORE: the `index` ran first, so `create` saw an existing document+/// and failed with 409 although it came first in the request.+#[tokio::test]+async fn create_then_index_runs_create_first() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ let body = ndjson(&[+ json!({"create": {"_index": "t", "_id": "1"}}),+ json!({"v": 1}),+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 2}),+ ]);+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "bulk must not error: {:?}", result.items);+ let statuses: Vec<u16> = result.items.iter().map(|i| i.status).collect();+ assert_eq!(statuses, vec![201, 200], "items: {:?}", result.items);+ assert_eq!(result.items[1].result.as_deref(), Some("updated"));++ let doc = source(&engine, "t", "1").await.expect("document");+ assert_eq!(doc.get("v"), Some(&json!(2)));+}++/// FAIL-BEFORE: the `index` ran first and the earlier upsert merged its+/// field into the replacement document, which a full `index` must not keep.+#[tokio::test]+async fn upsert_then_index_replaces_the_document() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ let body = ndjson(&[+ json!({"update": {"_index": "t", "_id": "1"}}),+ json!({"doc": {"u": 1}, "doc_as_upsert": true}),+ json!({"index": {"_index": "t", "_id": "1"}}),+ json!({"v": 3}),+ ]);+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "bulk must not error: {:?}", result.items);++ let doc = source(&engine, "t", "1").await.expect("document");+ assert_eq!(doc.get("v"), Some(&json!(3)));+ assert_eq!(+ doc.get("u"),+ None,+ "`index` replaces the whole source: {doc}"+ );+}++/// Plain `index` items on distinct ids keep using the turbo group and stay+/// unaffected by a `delete` of an unrelated id in the same request.+#[tokio::test]+async fn unrelated_ids_are_unaffected() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ seed(&engine, "t");++ let body = ndjson(&[+ json!({"index": {"_index": "t", "_id": "a"}}),+ json!({"v": 1}),+ json!({"delete": {"_index": "t", "_id": "zzz"}}),+ json!({"index": {"_index": "t", "_id": "b"}}),+ json!({"v": 2}),+ ]);+ let result = process_bulk(&engine, None, &body).await;+ assert_eq!(result.items[0].status, 201);+ assert_eq!(result.items[2].status, 201);+ assert!(source(&engine, "t", "a").await.is_some());+ assert!(source(&engine, "t", "b").await.is_some());+}