AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(bulk): run per-item ingest pipelines on the _bulk API
xerj-org1152882457d0
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex f09a2d9d5..52526f491 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -833,6 +833,7 @@ pub async fn process_bulk_with_opts(&& action.if_primary_term.is_none()&& action.routing.is_none()&& action.dynamic_templates.is_none()+ && action.pipeline.is_none()&& !index_has_strict_date(&action.target_index, &mut index_needs_date_validation)&& !index_has_dynamic_copy(&action.target_index, &mut index_needs_dynamic_copy);if is_plain_index {@@ -1027,7 +1028,7 @@ pub async fn process_bulk_with_opts(doc_bytes,item_index: item_idx,require_alias: _,- pipeline: _,+ pipeline,if_seq_no,if_primary_term,routing,@@ -1053,6 +1054,57 @@ pub async fn process_bulk_with_opts(None}});+ // ── Ingest pipeline execution ───────────────────────────────────+ //+ // ES runs the item's ingest pipeline BEFORE field mapping,+ // `copy_to`, and date / dynamic-template validation, so we apply+ // it here — right after the doc body is parsed. Only `index` /+ // `create` carry a source body through a pipeline (`update` /+ // `delete` never do). Unknown pipeline names were already rejected+ // as per-item 400s in the pre-partition validation pass above, so a+ // lookup miss here is defensive only — mirror the same error.+ if let Some(pid) = pipeline.as_deref() {+ if matches!(action_type.as_str(), "index" | "create") {+ let input = doc_body+ .clone()+ .unwrap_or_else(|| Value::Object(serde_json::Map::new()));+ match engine.process_through_pipeline(pid, vec![input]) {+ Ok(mut results) if !results.is_empty() => {+ let (p_action, transformed) = results.remove(0);+ if matches!(p_action, xerj_wasm::pipeline::ProcessAction::Drop) {+ // Pipeline dropped the doc: ES reports a+ // successful no-op and does not index it.+ items[item_idx] = Some(BulkItemResult {+ action: action_type.to_string(),+ index: target_index.clone(),+ id: error_id.clone(),+ status: 200,+ result: Some("noop".into()),+ error: None,+ get_source: None,+ });+ continue;+ }+ doc_body = Some(transformed);+ }+ // Empty result set — index the original body unchanged.+ Ok(_) => {}+ Err(_) => {+ items[item_idx] = Some(BulkItemResult {+ action: action_type.to_string(),+ index: target_index.clone(),+ id: error_id.clone(),+ status: 400,+ result: None,+ error: Some(format!("pipeline with id [{pid}] does not exist")),+ get_source: None,+ });+ errors = true;+ continue;+ }+ }+ }+ }// Apply dynamic-template `copy_to` at ingest. When a doc// field name matches a dynamic template's `match` pattern// (or equals the template name) and the template's mappingdiff --git a/engine/crates/xerj-engine/tests/es_compat_tests.rs b/engine/crates/xerj-engine/tests/es_compat_tests.rsindex d3d944e7c..f273e3cb1 100644--- a/engine/crates/xerj-engine/tests/es_compat_tests.rs+++ b/engine/crates/xerj-engine/tests/es_compat_tests.rs@@ -853,6 +853,87 @@ async fn test_es_bulk_mixed_operations() {);}+/// ES YAML test: bulk/90_pipeline.yml — per-item ingest pipeline execution.+///+/// A bulk `index` item carrying `pipeline: <name>` must be run through the+/// named ingest pipeline before it is stored; an item without a pipeline is+/// stored verbatim; and an item referencing an unknown pipeline fails with a+/// per-item 400 `illegal_argument_exception`.+#[tokio::test]+async fn test_es_bulk_per_item_pipeline() {+ let (engine, _dir) = test_engine();+ engine.create_index("bulk_pipe", Schema::empty()).unwrap();++ // Register a pipeline that stamps `ingested = "yes"` on every doc.+ engine+ .create_pipeline(+ "set_pipe",+ json!({+ "description": "stamp ingested marker",+ "stages": [+ { "type": "set", "config": { "field": "ingested", "value": "yes", "override": true } }+ ]+ }),+ )+ .expect("create_pipeline");++ let ndjson = concat!(+ // Item 0: no pipeline → stored verbatim.+ r#"{"index":{"_index":"bulk_pipe","_id":"plain"}}"#,+ "\n",+ r#"{"f1":"v1"}"#,+ "\n",+ // Item 1: pipeline set_pipe → should gain ingested=yes.+ r#"{"index":{"_index":"bulk_pipe","_id":"piped","pipeline":"set_pipe"}}"#,+ "\n",+ r#"{"f1":"v2"}"#,+ "\n",+ // Item 2: unknown pipeline → per-item 400.+ r#"{"index":{"_index":"bulk_pipe","_id":"bad","pipeline":"missing_pipe"}}"#,+ "\n",+ r#"{"f1":"v3"}"#,+ "\n",+ );++ let result = process_bulk(&engine, Some("bulk_pipe"), ndjson).await;+ assert_eq!(result.items.len(), 3);++ // Item 0 stored without the pipeline field.+ assert_eq!(result.items[0].status, 201);+ // Item 1 succeeded via the pipeline.+ assert_eq!(result.items[1].status, 201);+ // Item 2 failed because the pipeline does not exist.+ assert!(result.errors, "unknown pipeline should mark errors=true");+ assert_eq!(result.items[2].status, 400);+ assert_eq!(+ result.items[2].error.as_deref(),+ Some("pipeline with id [missing_pipe] does not exist")+ );++ let idx = engine.get_index("bulk_pipe").unwrap();++ // The piped doc carries the pipeline-injected field.+ let piped = idx.get_document("piped").await.unwrap().expect("piped doc");+ assert_eq!(+ piped.get("ingested").and_then(Value::as_str),+ Some("yes"),+ "pipeline must run on the bulk item: {piped:?}"+ );++ // The plain doc was NOT touched by the pipeline.+ let plain = idx.get_document("plain").await.unwrap().expect("plain doc");+ assert!(+ plain.get("ingested").is_none(),+ "doc without a pipeline must be stored verbatim: {plain:?}"+ );++ // The bad-pipeline doc was never indexed.+ assert!(+ idx.get_document("bad").await.unwrap().is_none(),+ "item referencing an unknown pipeline must not be stored"+ );+}+// ═════════════════════════════════════════════════════════════════════════════// Count API// ═════════════════════════════════════════════════════════════════════════════diff --git a/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml b/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.ymlindex 7e541d5d1..4b61bada8 100644--- a/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml+++ b/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml@@ -23,3 +23,42 @@index: test_index- match: {count: 1}++---+"Bulk item runs its ingest pipeline":+ - do:+ ingest.put_pipeline:+ id: "set_pipe"+ body:+ processors:+ - set:+ field: "ingested"+ value: "yes"++ - do:+ bulk:+ refresh: true+ body:+ - '{"index": {"_index": "pipe_index", "_id": "no_pipe"}}'+ - '{"f1": "plain"}'+ - '{"index": {"_index": "pipe_index", "_id": "with_pipe", "pipeline": "set_pipe"}}'+ - '{"f1": "piped"}'++ - match: { errors: false }+ - match: { items.0.index.result: created }+ - match: { items.1.index.result: created }++ # The pipelined doc gained the injected field.+ - do:+ get:+ index: pipe_index+ id: with_pipe+ - match: { _source.ingested: "yes" }++ # The non-pipelined doc was stored verbatim (no injected field).+ - do:+ get:+ index: pipe_index+ id: no_pipe+ - match: { _source.f1: "plain" }+ - is_false: _source.ingested