Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(bulk): run per-item ingest pipelines on the _bulk API

xerj-org1152882457d0
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index f09a2d9d5..52526f491 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -833,6 +833,7 @@ pub async fn process_bulk_with_opts(
&& action.if_primary_term.is_none()
&& action.routing.is_none()
&& action.dynamic_templates.is_none()
+ && action.pipeline.is_none()
&& !index_has_strict_date(&action.target_index, &mut index_needs_date_validation)
&& !index_has_dynamic_copy(&action.target_index, &mut index_needs_dynamic_copy);
if is_plain_index {
@@ -1027,7 +1028,7 @@ pub async fn process_bulk_with_opts(
doc_bytes,
item_index: item_idx,
require_alias: _,
- pipeline: _,
+ pipeline,
if_seq_no,
if_primary_term,
routing,
@@ -1053,6 +1054,57 @@ pub async fn process_bulk_with_opts(
None
}
});
+ // ── Ingest pipeline execution ───────────────────────────────────
+ //
+ // ES runs the item's ingest pipeline BEFORE field mapping,
+ // `copy_to`, and date / dynamic-template validation, so we apply
+ // it here — right after the doc body is parsed. Only `index` /
+ // `create` carry a source body through a pipeline (`update` /
+ // `delete` never do). Unknown pipeline names were already rejected
+ // as per-item 400s in the pre-partition validation pass above, so a
+ // lookup miss here is defensive only — mirror the same error.
+ if let Some(pid) = pipeline.as_deref() {
+ if matches!(action_type.as_str(), "index" | "create") {
+ let input = doc_body
+ .clone()
+ .unwrap_or_else(|| Value::Object(serde_json::Map::new()));
+ match engine.process_through_pipeline(pid, vec![input]) {
+ Ok(mut results) if !results.is_empty() => {
+ let (p_action, transformed) = results.remove(0);
+ if matches!(p_action, xerj_wasm::pipeline::ProcessAction::Drop) {
+ // Pipeline dropped the doc: ES reports a
+ // successful no-op and does not index it.
+ items[item_idx] = Some(BulkItemResult {
+ action: action_type.to_string(),
+ index: target_index.clone(),
+ id: error_id.clone(),
+ status: 200,
+ result: Some("noop".into()),
+ error: None,
+ get_source: None,
+ });
+ continue;
+ }
+ doc_body = Some(transformed);
+ }
+ // Empty result set — index the original body unchanged.
+ Ok(_) => {}
+ Err(_) => {
+ items[item_idx] = Some(BulkItemResult {
+ action: action_type.to_string(),
+ index: target_index.clone(),
+ id: error_id.clone(),
+ status: 400,
+ result: None,
+ error: Some(format!("pipeline with id [{pid}] does not exist")),
+ get_source: None,
+ });
+ errors = true;
+ continue;
+ }
+ }
+ }
+ }
// Apply dynamic-template `copy_to` at ingest. When a doc
// field name matches a dynamic template's `match` pattern
// (or equals the template name) and the template's mapping
diff --git a/engine/crates/xerj-engine/tests/es_compat_tests.rs b/engine/crates/xerj-engine/tests/es_compat_tests.rs
index d3d944e7c..f273e3cb1 100644
--- a/engine/crates/xerj-engine/tests/es_compat_tests.rs
+++ b/engine/crates/xerj-engine/tests/es_compat_tests.rs
@@ -853,6 +853,87 @@ async fn test_es_bulk_mixed_operations() {
);
}
+/// ES YAML test: bulk/90_pipeline.yml — per-item ingest pipeline execution.
+///
+/// A bulk `index` item carrying `pipeline: <name>` must be run through the
+/// named ingest pipeline before it is stored; an item without a pipeline is
+/// stored verbatim; and an item referencing an unknown pipeline fails with a
+/// per-item 400 `illegal_argument_exception`.
+#[tokio::test]
+async fn test_es_bulk_per_item_pipeline() {
+ let (engine, _dir) = test_engine();
+ engine.create_index("bulk_pipe", Schema::empty()).unwrap();
+
+ // Register a pipeline that stamps `ingested = "yes"` on every doc.
+ engine
+ .create_pipeline(
+ "set_pipe",
+ json!({
+ "description": "stamp ingested marker",
+ "stages": [
+ { "type": "set", "config": { "field": "ingested", "value": "yes", "override": true } }
+ ]
+ }),
+ )
+ .expect("create_pipeline");
+
+ let ndjson = concat!(
+ // Item 0: no pipeline → stored verbatim.
+ r#"{"index":{"_index":"bulk_pipe","_id":"plain"}}"#,
+ "\n",
+ r#"{"f1":"v1"}"#,
+ "\n",
+ // Item 1: pipeline set_pipe → should gain ingested=yes.
+ r#"{"index":{"_index":"bulk_pipe","_id":"piped","pipeline":"set_pipe"}}"#,
+ "\n",
+ r#"{"f1":"v2"}"#,
+ "\n",
+ // Item 2: unknown pipeline → per-item 400.
+ r#"{"index":{"_index":"bulk_pipe","_id":"bad","pipeline":"missing_pipe"}}"#,
+ "\n",
+ r#"{"f1":"v3"}"#,
+ "\n",
+ );
+
+ let result = process_bulk(&engine, Some("bulk_pipe"), ndjson).await;
+ assert_eq!(result.items.len(), 3);
+
+ // Item 0 stored without the pipeline field.
+ assert_eq!(result.items[0].status, 201);
+ // Item 1 succeeded via the pipeline.
+ assert_eq!(result.items[1].status, 201);
+ // Item 2 failed because the pipeline does not exist.
+ assert!(result.errors, "unknown pipeline should mark errors=true");
+ assert_eq!(result.items[2].status, 400);
+ assert_eq!(
+ result.items[2].error.as_deref(),
+ Some("pipeline with id [missing_pipe] does not exist")
+ );
+
+ let idx = engine.get_index("bulk_pipe").unwrap();
+
+ // The piped doc carries the pipeline-injected field.
+ let piped = idx.get_document("piped").await.unwrap().expect("piped doc");
+ assert_eq!(
+ piped.get("ingested").and_then(Value::as_str),
+ Some("yes"),
+ "pipeline must run on the bulk item: {piped:?}"
+ );
+
+ // The plain doc was NOT touched by the pipeline.
+ let plain = idx.get_document("plain").await.unwrap().expect("plain doc");
+ assert!(
+ plain.get("ingested").is_none(),
+ "doc without a pipeline must be stored verbatim: {plain:?}"
+ );
+
+ // The bad-pipeline doc was never indexed.
+ assert!(
+ idx.get_document("bad").await.unwrap().is_none(),
+ "item referencing an unknown pipeline must not be stored"
+ );
+}
+
// ═════════════════════════════════════════════════════════════════════════════
// Count API
// ═════════════════════════════════════════════════════════════════════════════
diff --git a/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml b/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml
index 7e541d5d1..4b61bada8 100644
--- a/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml
+++ b/engine/tests/es-compat-yaml/yaml/bulk/90_pipeline.yml
@@ -23,3 +23,42 @@
index: test_index
- match: {count: 1}
+
+---
+"Bulk item runs its ingest pipeline":
+ - do:
+ ingest.put_pipeline:
+ id: "set_pipe"
+ body:
+ processors:
+ - set:
+ field: "ingested"
+ value: "yes"
+
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{"index": {"_index": "pipe_index", "_id": "no_pipe"}}'
+ - '{"f1": "plain"}'
+ - '{"index": {"_index": "pipe_index", "_id": "with_pipe", "pipeline": "set_pipe"}}'
+ - '{"f1": "piped"}'
+
+ - match: { errors: false }
+ - match: { items.0.index.result: created }
+ - match: { items.1.index.result: created }
+
+ # The pipelined doc gained the injected field.
+ - do:
+ get:
+ index: pipe_index
+ id: with_pipe
+ - match: { _source.ingested: "yes" }
+
+ # The non-pipelined doc was stored verbatim (no injected field).
+ - do:
+ get:
+ index: pipe_index
+ id: no_pipe
+ - match: { _source.f1: "plain" }
+ - is_false: _source.ingested