Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(bulk): apply items that share an _id in request order (#2650)

Amartya Vikram Singhbd9f0510f56d
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index fcaf4d17a..504989695 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -1480,6 +1480,34 @@ pub async fn process_bulk_with_opts(
props
};
+ // A bulk body is an ordered log: ES applies its items in request order,
+ // so `delete 1` followed by `index 1` leaves the doc present, and
+ // `index 1 {v:1}`, `delete 1`, `index 1 {v:2}` leaves `{v:2}`. The
+ // partition below runs its groups as phases (semantic, then turbo
+ // index, then everything else), which reorders items across groups.
+ // That is harmless while each `(index, _id)` lives in one group, but a
+ // key that a non-turbo action (delete / update / create / CAS / routed
+ // index) also touches would see its plain `index` items jump ahead and
+ // the later delete win — the doc is gone and every item said success.
+ // Collect those keys up front and keep all of their items on the
+ // sequential path, which executes in request order.
+ let sequential_keys: std::collections::HashSet<(String, String)> = parsed
+ .iter()
+ .filter(|a| {
+ a.action_type != "index"
+ || a.if_seq_no.is_some()
+ || a.if_primary_term.is_some()
+ || a.routing.is_some()
+ || a.dynamic_templates.is_some()
+ || a.pipeline.is_some()
+ })
+ .filter_map(|a| {
+ a.doc_id
+ .as_ref()
+ .map(|id| (a.target_index.clone(), id.clone()))
+ })
+ .collect();
+
for mut action in parsed {
// ── Numeric / boolean type enforcement (issue #781) ──────────────
//
@@ -1560,7 +1588,11 @@ pub async fn process_bulk_with_opts(
&& action.dynamic_templates.is_none()
&& action.pipeline.is_none()
&& !has_strict_date
- && !has_dynamic_copy;
+ && !has_dynamic_copy
+ && (sequential_keys.is_empty()
+ || !action.doc_id.as_ref().is_some_and(|id| {
+ sequential_keys.contains(&(action.target_index.clone(), id.clone()))
+ }));
let is_plain_index = has_plain_metadata && !has_embedding;
if is_plain_index {
match action.doc_id {
diff --git a/engine/crates/xerj-engine/tests/bulk_same_id_order.rs b/engine/crates/xerj-engine/tests/bulk_same_id_order.rs
new file mode 100644
index 000000000..4a3996847
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/bulk_same_id_order.rs
@@ -0,0 +1,175 @@
+//! `_bulk` must apply items that share an `(index, _id)` in request order.
+//!
+//! The bulk executor partitions items into phases — plain `index` items run
+//! as one turbo group, `delete` / `update` / `create` / CAS / routed items run
+//! afterwards — which reorders items across groups. When one `_id` was touched
+//! by both kinds in one request, its plain `index` items jumped ahead of an
+//! earlier `delete` or `update`, so the final document did not match the
+//! request: an `index`, `delete`, `index` sequence ended with the document
+//! deleted, while every item reported success.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::bulk::process_bulk;
+use xerj_engine::Engine;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+fn seed(engine: &Engine, name: &str) {
+ let mut schema = Schema::empty();
+ schema.fields.push(FieldConfig::new("v", FieldType::Long));
+ engine.create_index(name, schema).expect("create index");
+}
+
+fn ndjson(lines: &[Value]) -> String {
+ let mut out = String::new();
+ for l in lines {
+ out.push_str(&l.to_string());
+ out.push('\n');
+ }
+ out
+}
+
+async fn source(engine: &Engine, index: &str, id: &str) -> Option<Value> {
+ engine
+ .get_index(index)
+ .expect("get index")
+ .get_document(id)
+ .await
+ .expect("lookup")
+}
+
+/// FAIL-BEFORE: both `index` items ran in the turbo phase, then the `delete`
+/// ran last and removed the document that the request's final item wrote.
+#[tokio::test]
+async fn index_delete_index_keeps_the_last_write() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "t");
+
+ let body = ndjson(&[
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 1}),
+ json!({"delete": {"_index": "t", "_id": "1"}}),
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 2}),
+ ]);
+ let result = process_bulk(&engine, None, &body).await;
+ assert!(!result.errors, "bulk must not error: {:?}", result.items);
+ let statuses: Vec<u16> = result.items.iter().map(|i| i.status).collect();
+ assert_eq!(statuses, vec![201, 200, 201], "items: {:?}", result.items);
+
+ let doc = source(&engine, "t", "1")
+ .await
+ .expect("the final `index` item must leave the document present");
+ assert_eq!(doc.get("v"), Some(&json!(2)));
+}
+
+/// FAIL-BEFORE: the `index` ran first and the earlier `delete` removed it.
+#[tokio::test]
+async fn delete_then_index_leaves_the_document() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "t");
+ process_bulk(
+ &engine,
+ None,
+ &ndjson(&[
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 1}),
+ ]),
+ )
+ .await;
+
+ let body = ndjson(&[
+ json!({"delete": {"_index": "t", "_id": "1"}}),
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 7}),
+ ]);
+ let result = process_bulk(&engine, None, &body).await;
+ assert!(!result.errors, "bulk must not error: {:?}", result.items);
+ assert_eq!(result.items[1].status, 201, "re-index after delete creates");
+
+ let doc = source(&engine, "t", "1")
+ .await
+ .expect("document indexed after the delete must exist");
+ assert_eq!(doc.get("v"), Some(&json!(7)));
+}
+
+/// FAIL-BEFORE: the `index` ran first, so `create` saw an existing document
+/// and failed with 409 although it came first in the request.
+#[tokio::test]
+async fn create_then_index_runs_create_first() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "t");
+
+ let body = ndjson(&[
+ json!({"create": {"_index": "t", "_id": "1"}}),
+ json!({"v": 1}),
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 2}),
+ ]);
+ let result = process_bulk(&engine, None, &body).await;
+ assert!(!result.errors, "bulk must not error: {:?}", result.items);
+ let statuses: Vec<u16> = result.items.iter().map(|i| i.status).collect();
+ assert_eq!(statuses, vec![201, 200], "items: {:?}", result.items);
+ assert_eq!(result.items[1].result.as_deref(), Some("updated"));
+
+ let doc = source(&engine, "t", "1").await.expect("document");
+ assert_eq!(doc.get("v"), Some(&json!(2)));
+}
+
+/// FAIL-BEFORE: the `index` ran first and the earlier upsert merged its
+/// field into the replacement document, which a full `index` must not keep.
+#[tokio::test]
+async fn upsert_then_index_replaces_the_document() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "t");
+
+ let body = ndjson(&[
+ json!({"update": {"_index": "t", "_id": "1"}}),
+ json!({"doc": {"u": 1}, "doc_as_upsert": true}),
+ json!({"index": {"_index": "t", "_id": "1"}}),
+ json!({"v": 3}),
+ ]);
+ let result = process_bulk(&engine, None, &body).await;
+ assert!(!result.errors, "bulk must not error: {:?}", result.items);
+
+ let doc = source(&engine, "t", "1").await.expect("document");
+ assert_eq!(doc.get("v"), Some(&json!(3)));
+ assert_eq!(
+ doc.get("u"),
+ None,
+ "`index` replaces the whole source: {doc}"
+ );
+}
+
+/// Plain `index` items on distinct ids keep using the turbo group and stay
+/// unaffected by a `delete` of an unrelated id in the same request.
+#[tokio::test]
+async fn unrelated_ids_are_unaffected() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ seed(&engine, "t");
+
+ let body = ndjson(&[
+ json!({"index": {"_index": "t", "_id": "a"}}),
+ json!({"v": 1}),
+ json!({"delete": {"_index": "t", "_id": "zzz"}}),
+ json!({"index": {"_index": "t", "_id": "b"}}),
+ json!({"v": 2}),
+ ]);
+ let result = process_bulk(&engine, None, &body).await;
+ assert_eq!(result.items[0].status, 201);
+ assert_eq!(result.items[2].status, 201);
+ assert!(source(&engine, "t", "a").await.is_some());
+ assert!(source(&engine, "t", "b").await.is_some());
+}