AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
perf(bulk): publish explicit-id index actions as one batch (Refs #2427) (#2717)
xerj-org18e459cfef7c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex fd70bcb14..99b1593e0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -47,6 +47,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- `_bulk` throughput with explicit `_id`s: `index` actions that carry an `_id`+ (and prepared `semantic_text` documents) are now published as one batch per+ index instead of one `index_document` call each, which took the WAL lock,+ copied the source into the storage memtable and opened a publication bracket+ per document. Per-item results are unchanged: `created`/`updated`, `_version`,+ and per-item date and size errors that keep their `_id`. (Refs #2427)- `xerj autoindex --fresh` over an already-indexed folder no longer pays oneedges-index search, bulk and refresh per replaced file (and again persuperseded anchor) to soft-invalidate prior relationship edges. The graphdiff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 504989695..95b1710db 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -112,6 +112,139 @@ pub(crate) async fn with_semantic_lifecycle_hook<F: std::future::Future>(.await}+#[cfg(test)]+fn semantic_lifecycle_hook_installed(target: &str) -> bool {+ SEMANTIC_LIFECYCLE_HOOK+ .try_with(|(hook_target, _)| hook_target == "*" || hook_target == target)+ .unwrap_or(false)+}++#[cfg(not(test))]+fn semantic_lifecycle_hook_installed(_target: &str) -> bool {+ false+}++#[cfg(test)]+thread_local! {+ /// Test-only count of explicit-id batch publications (#2427).+ static EXPLICIT_ID_BATCHES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };+}++/// Publish semantically prepared sources as one batch (#2427). A failed+/// preparation fails only its own item; an empty id gets a generated one.+async fn publish_prepared_batch(+ idx: &crate::index::Index,+ index_name: &str,+ prepared: Vec<(usize, String, Result<Value>)>,+ items: &mut [Option<BulkItemResult>],+) -> bool {+ let mut errors = false;+ let mut docs = Vec::with_capacity(prepared.len());+ for (item_idx, id, prepared) in prepared {+ let failure = match prepared {+ Ok(source) => {+ let id = if id.is_empty() {+ uuid::Uuid::new_v4().to_string()+ } else {+ id+ };+ match serde_json::to_vec(&source) {+ Ok(bytes) => {+ docs.push((item_idx, id, std::sync::Arc::<[u8]>::from(bytes)));+ continue;+ }+ Err(e) => (id, 500, e.to_string()),+ }+ }+ Err(e) => (id, engine_error_http_status(&e), e.to_string()),+ };+ let (id, status, error) = failure;+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.to_string(),+ id,+ status,+ result: None,+ error: Some(error),+ get_source: None,+ });+ errors = true;+ }+ errors | publish_explicit_id_batch(idx, index_name, docs, items).await+}++/// Publish `index` actions that carry an `_id` as one batch and fill their+/// item slots in request order. Returns whether any item failed (#2427).+async fn publish_explicit_id_batch(+ idx: &crate::index::Index,+ index_name: &str,+ docs: Vec<(usize, String, std::sync::Arc<[u8]>)>,+ items: &mut [Option<BulkItemResult>],+) -> bool {+ if docs.is_empty() {+ return false;+ }+ #[cfg(test)]+ EXPLICIT_ID_BATCHES.with(|count| count.set(count.get() + 1));+ let mut errors = false;+ let mut slots = Vec::with_capacity(docs.len());+ let batch = docs+ .into_iter()+ .map(|(item_idx, id, bytes)| {+ slots.push((item_idx, id.clone()));+ (id, bytes)+ })+ .collect();+ match idx.index_batch_explicit_ids_raw(batch).await {+ Ok(results) => {+ for ((item_idx, id), result) in slots.into_iter().zip(results) {+ items[item_idx] = Some(match result {+ Ok(resp) => BulkItemResult {+ action: "index".into(),+ index: index_name.to_string(),+ id: resp.id,+ // 201 for a new doc, 200 for an overwrite (ES).+ status: if resp.result == "updated" { 200 } else { 201 },+ result: Some(resp.result),+ error: None,+ get_source: None,+ },+ Err(e) => {+ errors = true;+ BulkItemResult {+ action: "index".into(),+ index: index_name.to_string(),+ id,+ status: engine_error_http_status(&e),+ result: None,+ error: Some(e.to_string()),+ get_source: None,+ }+ }+ });+ }+ }+ Err(e) => {+ // Whole-batch failure (write block, breaker, back-pressure):+ // every item gets the same status, as on the auto-id batch.+ let status = engine_error_http_status(&e);+ for (item_idx, id) in slots {+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.to_string(),+ id,+ status,+ result: None,+ error: Some(e.to_string()),+ get_source: None,+ });+ }+ errors = true;+ }+ }+ errors+}+// ── Types ─────────────────────────────────────────────────────────────────────#[derive(Debug, Clone, Serialize, Deserialize)]@@ -1755,6 +1888,20 @@ pub async fn process_bulk_with_opts(let prepared = idx.apply_semantic_embeddings_batch(valid_sources).await;#[cfg(test)]let _ = semantic_lifecycle_hook(&index_name, SemanticLifecyclePoint::Prepared);+ // #2427: without ingest-memory tracing nothing is accounted per+ // document, so publish the prepared group as one batch instead of+ // one `index_document_prepared` call per document. The tracing mode+ // (and its lifecycle tests) keeps the sequential path so each+ // prepared document's ownership is still released as it publishes.+ if valid_memory.is_none() && !semantic_lifecycle_hook_installed(&index_name) {+ let docs = valid_meta+ .into_iter()+ .zip(prepared)+ .map(|((item_idx, id), prepared)| (item_idx, id.unwrap_or_default(), prepared))+ .collect();+ errors |= publish_prepared_batch(&idx, &index_name, docs, &mut items).await;+ continue;+ }let mut valid_memory = valid_memory.map(Vec::into_iter);for ((item_idx, id), prepared_source) in valid_meta.into_iter().zip(prepared) {let memory = valid_memory.as_mut().and_then(Iterator::next);@@ -1931,70 +2078,61 @@ pub async fn process_bulk_with_opts(}};- // Index each doc via the per-doc path. index_document() handles- // WAL write, storage memtable (_source), AND FTS memtable- // (search + aggregations) in one call — no separate raw insert.- for (item_idx, doc_id, doc_bytes) in batch {- // The parse stage already validated these bytes (syntax +- // object-ness), so this parse cannot fail in practice — but- // never fall back to indexing an EMPTY doc: a `{}` here would- // be silent data loss reported as success. Fail the item loud.- let source = match serde_json::from_slice::<Value>(&doc_bytes) {- Ok(v) => v,- Err(e) => {- items[item_idx] = Some(BulkItemResult {- action: "index".into(),- index: index_name.clone(),- id: doc_id,- status: 400,- result: None,- error: Some(format!("failed to parse: invalid document JSON: {e}")),- get_source: None,- });- errors = true;- continue;- }- };- tracing::debug!(- doc_id = doc_id.as_str(),- source = %source,- bytes_len = doc_bytes.len(),- "bulk: indexing doc via per-doc path"- );- let id_opt = if doc_id.is_empty() {- None- } else {- Some(doc_id)- };- match idx.index_document(id_opt, source).await {- Ok(resp) => {- // 201 for new doc, 200 for overwrite (ES semantics).- let status: u16 = if resp.result == "updated" { 200 } else { 201 };- items[item_idx] = Some(BulkItemResult {- action: "index".into(),- index: index_name.clone(),- id: resp.id,- status,- result: Some(resp.result),- error: None,- get_source: None,- });- }- Err(e) => {- let status = engine_error_http_status(&e);- items[item_idx] = Some(BulkItemResult {- action: "index".into(),- index: index_name.clone(),- id: String::new(),- status,- result: None,- error: Some(e.to_string()),- get_source: None,- });- errors = true;+ // #2427: publish the whole explicit-id group as one batch. The+ // per-document `index_document` loop this replaces held the WAL lock,+ // copied the source into the storage memtable, and opened a+ // publication bracket once per document; eight concurrent bulks+ // used about two cores. `index_batch_explicit_ids_raw` keeps the+ // per-doc answers (`created`/`updated`, `_version`, date errors).+ //+ // An index created from a native schema can carry embedding fields+ // without a `semantic_text` mapping, so the partition above calls it+ // plain. `index_document` embedded those documents itself; prepare+ // them here in one shared pass before the batch publication.+ if !idx.semantic_vector_fields().await.is_empty() {+ let mut meta = Vec::with_capacity(batch.len());+ let mut sources = Vec::with_capacity(batch.len());+ for (item_idx, doc_id, doc_bytes) in batch {+ match serde_json::from_slice::<Value>(&doc_bytes) {+ Ok(source) => {+ meta.push((item_idx, doc_id));+ sources.push(source);+ }+ Err(e) => {+ items[item_idx] = Some(BulkItemResult {+ action: "index".into(),+ index: index_name.clone(),+ id: doc_id,+ status: 400,+ result: None,+ error: Some(format!("failed to parse: invalid document JSON: {e}")),+ get_source: None,+ });+ errors = true;+ }}}+ let prepared = idx.apply_semantic_embeddings_batch(sources).await;+ let docs = meta+ .into_iter()+ .zip(prepared)+ .map(|((item_idx, id), prepared)| (item_idx, id, prepared))+ .collect();+ errors |= publish_prepared_batch(&idx, &index_name, docs, &mut items).await;+ continue;}+ let docs = batch+ .into_iter()+ .map(|(item_idx, doc_id, doc_bytes)| {+ let doc_id = if doc_id.is_empty() {+ uuid::Uuid::new_v4().to_string()+ } else {+ doc_id+ };+ (item_idx, doc_id, doc_bytes)+ })+ .collect();+ errors |= publish_explicit_id_batch(&idx, &index_name, docs, &mut items).await;}// Execute non-index ops individually (create, update, delete).@@ -3903,3 +4041,159 @@ mod parse_admission_tests {}}}++#[cfg(test)]+mod explicit_id_batch_tests {+ use super::*;+ use serde_json::json;+ use tempfile::TempDir;+ use xerj_common::config::Config;+ use xerj_common::types::{EmbeddingConfig, FieldConfig, FieldType, Schema};++ fn engine_with(name: &str, schema: Schema) -> (TempDir, Engine) {+ let dir = TempDir::new().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = Engine::new(config).unwrap();+ engine.create_index(name, schema).unwrap();+ (dir, engine)+ }++ fn statuses(result: &BulkResult) -> Vec<(u16, Option<String>)> {+ result+ .items+ .iter()+ .map(|item| (item.status, item.result.clone()))+ .collect()+ }++ fn created() -> (u16, Option<String>) {+ (201, Some("created".into()))+ }++ fn updated() -> (u16, Option<String>) {+ (200, Some("updated".into()))+ }++ /// #2427: explicit-id `index` actions were published one+ /// `index_document` call at a time, each also copying the source into+ /// the storage memtable. They now go through one turbo batch, which+ /// keeps only the WAL + FTS representation, while `created`/`updated`+ /// and `_version` stay exactly what the per-doc path reported.+ #[tokio::test]+ async fn explicit_id_index_actions_publish_as_one_batch() {+ let name = "explicit-id-batch";+ let (_dir, engine) = engine_with(name, Schema::empty());+ let body = format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"a\"}}}}\n{{\"n\":1,\"t\":\"alpha\"}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"b\"}}}}\n{{\"n\":2,\"t\":\"beta\"}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"a\"}}}}\n{{\"n\":3,\"t\":\"alpha again\"}}\n"+ );+ EXPLICIT_ID_BATCHES.with(|count| count.set(0));+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "{:?}", result.items);+ assert_eq!(statuses(&result), vec![created(), created(), updated()]);+ assert_eq!(EXPLICIT_ID_BATCHES.with(|count| count.get()), 1);++ let idx = engine.get_or_create_index(name).unwrap();+ assert_eq!(+ idx.resident_memtable_entries(),+ 0,+ "explicit-id bulk documents were copied into the storage memtable one by one"+ );+ assert_eq!(idx.lookup_version("a"), Some(2));+ assert_eq!(idx.lookup_version("b"), Some(1));+ let a = idx.get_document("a").await.unwrap().unwrap();+ assert_eq!(a["n"], json!(3));+ assert_eq!(a["t"], json!("alpha again"));++ // A tombstone keeps counting; a re-index after delete is "created".+ let body = format!(+ "{{\"delete\":{{\"_index\":\"{name}\",\"_id\":\"b\"}}}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"a\"}}}}\n{{\"n\":4}}\n"+ );+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "{:?}", result.items);+ assert_eq!(idx.lookup_version("a"), Some(3));+ assert_eq!(idx.lookup_version("b"), None);+ let body = format!("{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"b\"}}}}\n{{\"n\":5}}\n");+ let result = process_bulk(&engine, None, &body).await;+ assert_eq!(statuses(&result), vec![created()]);+ assert_eq!(idx.lookup_version("b"), Some(3));+ let b = idx.get_document("b").await.unwrap().unwrap();+ assert_eq!(b["n"], json!(5));+ }++ /// The batch must keep per-document default-format date refusal: one+ /// bad date fails only its own item.+ #[tokio::test]+ async fn explicit_id_batch_keeps_per_item_date_errors() {+ let name = "explicit-id-batch-dates";+ let mut schema = Schema::empty();+ schema.fields.push(FieldConfig::new("ts", FieldType::Date));+ let (_dir, engine) = engine_with(name, schema);+ let body = format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"ok1\"}}}}\n{{\"ts\":\"2024-01-02\"}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"bad\"}}}}\n{{\"ts\":\"not a date\"}}\n\+ {{\"index\":{{\"_index\":\"{name}\",\"_id\":\"ok2\"}}}}\n{{\"ts\":\"2024-01-03T04:05:06Z\"}}\n"+ );+ let result = process_bulk(&engine, None, &body).await;+ assert!(result.errors);+ assert_eq!(result.items[0].status, 201, "{:?}", result.items);+ assert_eq!(result.items[1].status, 400, "{:?}", result.items);+ assert_eq!(result.items[1].id, "bad");+ assert_eq!(result.items[2].status, 201, "{:?}", result.items);+ let idx = engine.get_or_create_index(name).unwrap();+ assert!(idx.get_document("bad").await.unwrap().is_none());+ assert!(idx.get_document("ok2").await.unwrap().is_some());+ }++ /// Semantic documents (ingest-memory tracing off) publish their prepared+ /// sources as one batch too, vectors included.+ #[tokio::test]+ async fn semantic_index_actions_publish_as_one_batch() {+ let name = "explicit-id-batch-semantic";+ let mut schema = Schema::empty();+ let mut field = FieldConfig::new("content", FieldType::Text);+ field.options.dimensions = Some(16);+ field.options.similarity = Some("cosine".to_string());+ field.embedding = Some(EmbeddingConfig {+ endpoint: None,+ model: None,+ target_field: Some("content_vector".into()),+ });+ schema.fields.push(field);+ let (_dir, engine) = engine_with(name, schema);+ engine+ .put_index_mapping(+ name,+ json!({"properties": {"content": {"type": "semantic_text"}}}),+ )+ .unwrap();+ let body = (0..3)+ .map(|i| {+ format!(+ "{{\"index\":{{\"_index\":\"{name}\",\"_id\":\"{}\"}}}}\n\+ {{\"content\":\"semantic document number {i}\"}}\n",+ i % 2+ )+ })+ .collect::<String>();+ EXPLICIT_ID_BATCHES.with(|count| count.set(0));+ let result = process_bulk(&engine, None, &body).await;+ assert!(!result.errors, "{:?}", result.items);+ assert_eq!(statuses(&result), vec![created(), created(), updated()]);+ assert_eq!(EXPLICIT_ID_BATCHES.with(|count| count.get()), 1);+ let idx = engine.get_or_create_index(name).unwrap();+ assert_eq!(idx.resident_memtable_entries(), 0);+ assert_eq!(idx.lookup_version("0"), Some(2));+ let doc = idx.get_document("0").await.unwrap().unwrap();+ assert_eq!(doc["content"], json!("semantic document number 2"));+ assert!(+ idx.semantic_vector_fields()+ .await+ .contains("content_vector"),+ "semantic target field missing"+ );+ }+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 576b2d33b..f1e3f1315 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -12585,6 +12585,93 @@ impl Index {Ok(responses)}+ /// Batch-index documents that carry caller-chosen `_id`s (#2427).+ ///+ /// `_bulk` used to publish every explicit-id `index` action through its+ /// own [`Index::index_document`] call: one WAL lock and append, one deep+ /// copy into the storage memtable, and one publication bracket per+ /// document. At eight concurrent bulks the node used about two cores and+ /// sat on those per-document locks. This runs the same group through the+ /// [`Index::index_batch_turbo_raw`] batch instead and keeps the per-doc+ /// answers the per-doc path gave:+ ///+ /// * every id is held in the write-publication registry (sorted, so two+ /// batches cannot deadlock) for the whole batch, so no other writer of+ /// these ids can change the version map in between;+ /// * `created`/`updated` and the per-document `_version` are derived from+ /// the version map read under those locks, counting repeats of one id+ /// inside the batch in request order — the same values+ /// `VersionMap::set` assigns as the batch appends;+ /// * default-format date validation still rejects a document on its own.+ ///+ /// The outer `Err` is a whole-batch failure (write block, breaker,+ /// back-pressure); an inner `Err` belongs to that one document.+ pub(crate) async fn index_batch_explicit_ids_raw(+ &self,+ docs: Vec<(String, Arc<[u8]>)>,+ ) -> Result<Vec<Result<IndexResponse>>> {+ let mut results: Vec<Option<Result<IndexResponse>>> =+ (0..docs.len()).map(|_| None).collect();+ let mut positions = Vec::with_capacity(docs.len());+ let mut batch = Vec::with_capacity(docs.len());+ let check_dates = self.has_date_fields.load(Ordering::Relaxed);+ for (position, (id, bytes)) in docs.into_iter().enumerate() {+ // One oversized document would fail the whole turbo batch; the+ // per-doc path refused only that document, so refuse it here.+ if xerj_common::document_limit::indexed_document_too_large(bytes.len()) {+ results[position] = Some(Err(EngineError::Common(+ xerj_common::XerjError::invalid_document_json(+ xerj_common::document_limit::indexed_document_refusal(bytes.len()),+ ),+ )));+ continue;+ }+ if check_dates {+ if let Ok(source) = serde_json::from_slice::<Value>(&bytes) {+ if let Err(e) = self.validate_default_format_dates(&source).await {+ results[position] = Some(Err(e));+ continue;+ }+ }+ }+ positions.push(position);+ batch.push((id, bytes));+ }+ if !batch.is_empty() {+ let _publication_guards = self+ .write_publication+ .lock_many(batch.iter().map(|(id, _)| id.clone()))+ .await;+ // (live, version) per id before this batch, read under the locks.+ let mut prior: HashMap<String, (bool, u64)> = HashMap::with_capacity(batch.len());+ for (id, _) in &batch {+ prior.entry(id.clone()).or_insert_with(|| {+ self.store+ .version_map+ .get(id)+ .map(|entry| (!entry.deleted, entry.version))+ .unwrap_or((false, 0))+ });+ }+ self.dataset_version.fetch_add(1, Ordering::Release);+ let responses = self.index_batch_turbo_raw(batch).await?;+ for (position, mut response) in positions.into_iter().zip(responses) {+ let (live, version) = prior+ .get_mut(&response.id)+ .expect("turbo responses keep the request ids");+ response.result = if *live { "updated" } else { "created" }.to_string();+ *version += 1;+ *live = true;+ response.version = *version;+ results[position] = Some(Ok(response));+ }+ }+ Ok(results+ .into_iter()+ .map(|result| result.expect("every document has a result"))+ .collect())+ }+/// M5.11 — ULTRA-turbo raw-bytes ingest.////// Takes already-serialized NDJSON document lines, validates and parses