Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(fts,wasm,api): apply the #204 sweep to the sweep's own new surface

xerj-org272bc7fd2927
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 678c328c2..8a55caf30 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -16540,11 +16540,17 @@ pub async fn index_stats(
Ok(idx) => idx.hnsw_stats().await,
Err(_) => json!({ "present": false }),
};
+ let schema_persist_failures = state
+ .engine
+ .get_index(name)
+ .map(|idx| idx.schema_persist_failures())
+ .unwrap_or(0);
let primaries = json!({
"docs": { "count": doc_count, "deleted": 0 },
"store": { "size_in_bytes": store_size_bytes },
"dense_vector": dv,
"hnsw": hnsw,
+ "mappings": { "schema_persist_failures": schema_persist_failures },
"merges": {
"current": if merging { 1 } else { 0 },
"current_docs": 0,
@@ -16681,11 +16687,24 @@ pub async fn index_stats(
)
})
.unwrap_or((0, false));
+ // XERJ extension (issue #204): dynamic-mapping evolution that could not be
+ // written to `schema.json`. Non-zero means the on-disk mapping is behind
+ // the live one and a restart will lose those fields. It is counted rather
+ // than raised because schema evolution runs after the document has been
+ // admitted — there is no caller left to fail — so a health check needs to
+ // be able to READ it. This is that read: without it the counter existed and
+ // nothing in the workspace looked at it.
+ let schema_persist_failures = state
+ .engine
+ .get_index(&index)
+ .map(|idx| idx.schema_persist_failures())
+ .unwrap_or(0);
let primaries = json!({
"docs": { "count": doc_count, "deleted": 0 },
"store": { "size_in_bytes": store_size_bytes },
"dense_vector": dense_vector_stats,
"hnsw": hnsw_stats,
+ "mappings": { "schema_persist_failures": schema_persist_failures },
"indexing": {
// Real cumulative indexing counters (RC4 W4 item 1): plumbed from
// the per-index `metric_index_count` / `metric_index_total_ms`
@@ -21652,6 +21671,27 @@ pub async fn put_settings(
// Normalize body into the `{ "index": { ... } }` shape we store.
let inner = body.get("index").cloned().unwrap_or(body.clone());
+ // An `analysis` block here is accepted-and-ignored, and it is the one hole
+ // that would walk straight around the create-time gate: this handler merges
+ // the body into `engine.index_settings` (the display copy read back by
+ // `GET /{index}/_settings`) and nothing else. The index's
+ // `AnalyzerRegistry` is built once, at create/open (index.rs:5428, :5617),
+ // and there is no rebuild path — so a settings PUT that "adds an analyzer"
+ // changes what the API echoes back and not one token of what is indexed.
+ // Refuse it. Elasticsearch also refuses analysis updates on an open index
+ // (they are non-dynamic settings), so this narrows behaviour towards ES
+ // rather than away from it (issue #204).
+ if inner.pointer("/analysis").is_some() || body.pointer("/analysis").is_some() {
+ let e = xerj_common::XerjError::config(format!(
+ "index [{index}]: `analysis` settings cannot be updated after the index is \
+ created — the analyzer registry is built at index creation and this request \
+ would change only what `GET /{index}/_settings` echoes back, not how \
+ documents are analysed. Create a new index with the analysis block and \
+ reindex."
+ ));
+ return ApiError::new(e).into_response();
+ }
+
for idx in &targets {
let mut existing = state
.engine
@@ -21745,11 +21785,33 @@ pub async fn put_ingest_pipeline(
"date" => "timestamp_parse".to_string(),
"json" => "json_parse".to_string(),
"convert" => "convert".to_string(),
- "append" | "set" => "set".to_string(),
+ // `append` used to be folded into `set` here, which REPLACES the
+ // field instead of extending it — a different document, under a
+ // 200, and in disagreement with the `_simulate` interpreter below,
+ // which has always implemented a real append (issue #204). There is
+ // now an `append` stage; the two agree.
+ "set" => "set".to_string(),
"copy" => "copy_field".to_string(),
other => other.to_string(),
}
}
+
+ /// Elasticsearch `convert` target types that name exactly the same
+ /// conversion as one of xerj's, under a different word.
+ ///
+ /// Issue #204 permits an *equivalent* fallback; `ConvertTypePlugin`'s
+ /// `integer` parses to `i64` and its `float` to `f64` (`builtins.rs`), so
+ /// ES `long` and `double` are the same conversion and 400ing them rejected
+ /// previously-accepted ES pipelines for no gain. `auto`, `ip`, `short` and
+ /// `byte` have no equivalent here and stay refused rather than being
+ /// silently widened.
+ fn map_convert_type(es_type: &str) -> Option<&'static str> {
+ match es_type {
+ "long" => Some("integer"),
+ "double" => Some("float"),
+ _ => None,
+ }
+ }
let xerj_config = if let Some(processors) = body.get("processors").and_then(Value::as_array) {
let mut stages: Vec<Value> = Vec::with_capacity(processors.len());
for (i, proc) in processors.iter().enumerate() {
@@ -21798,6 +21860,19 @@ pub async fn put_ingest_pipeline(
_ => proc_config.clone(),
}
}
+ "convert" => {
+ // ES vocabulary → xerj vocabulary for the two targets that
+ // are the same conversion under a different name.
+ let mut cfg = proc_config.clone();
+ if let Some(mapped) = proc_config
+ .get("type")
+ .and_then(Value::as_str)
+ .and_then(map_convert_type)
+ {
+ cfg["type"] = json!(mapped);
+ }
+ cfg
+ }
_ => proc_config.clone(),
};
stages.push(json!({"type": xerj_type, "config": adapted_config}));
@@ -21844,6 +21919,25 @@ pub async fn put_ingest_pipeline(
);
Json(json!({ "acknowledged": true })).into_response()
}
+ // Same shape one level down: the processor exists, an option on it does
+ // not. ES accepts the definition, so we do too — and record why it
+ // cannot run, so no document is written as if the option had been
+ // honoured (issue #204 applied to this endpoint's own new contract).
+ Err(xerj_wasm::WasmError::UnsupportedOption {
+ stage,
+ option,
+ reason,
+ }) => {
+ state.engine.register_unrunnable_pipeline(
+ &id,
+ body,
+ format!(
+ "processor [{stage}] option [{option}] is not supported by this xerj \
+ build ({reason}); no document can be ingested through pipeline [{id}]"
+ ),
+ );
+ Json(json!({ "acknowledged": true })).into_response()
+ }
Err(e) => {
tracing::warn!(pipeline = %id, error = %e, "rejected ingest pipeline definition");
let err = xerj_common::XerjError::invalid_mapping(format!(
@@ -25042,22 +25136,69 @@ async fn clone_index_to(state: &AppState, source: &str, target: &str) -> Result<
.get_index(target)
.map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
- // Copy all documents.
- let search_req = xerj_query::parse_request(&json!({
- "query": { "match_all": {} },
- "size": 10000,
- "from": 0,
- }))
- .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;
+ // Copy EVERY document, and fail loudly if any copy fails.
+ //
+ // Issue #204, the "a partial export instead of a full one" case, verbatim.
+ // This used to be a single `size: 10000` search whose per-document writes
+ // were swallowed with `let _ =`, and then answered `{"acknowledged": true,
+ // "shards_acknowledged": true}` — so cloning an index with 10,001
+ // documents silently produced a 10,000-document copy, and a clone in which
+ // every single write failed was reported as a success. `_clone`, `_shrink`
+ // and `_split` all land here, on user data.
+ //
+ // Same keyset (`search_after` on `_id`) paging the `_reindex` handler uses,
+ // for the same reason: `from`-offset paging truncates at
+ // `max_result_window`. The source is flushed first because the in-memory
+ // memtable's sort path does not order by `_id` reliably, so un-flushed
+ // writes would be skipped or duplicated across pages.
+ let _ = src_idx.flush().await;
+
+ const CLONE_PAGE_SIZE: usize = 1_000;
+ let mut search_after: Option<Value> = None;
+ let mut copied = 0usize;
- let results = src_idx
- .search(&search_req)
- .await
- .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
+ loop {
+ let mut search_body = json!({
+ "query": { "match_all": {} },
+ "size": CLONE_PAGE_SIZE,
+ "sort": [{ "_id": "asc" }],
+ });
+ if let Some(sa) = &search_after {
+ search_body["search_after"] = sa.clone();
+ }
+ let search_req = xerj_query::parse_request(&search_body)
+ .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;
- for hit in results.hits {
- if !hit.source.is_null() {
- let _ = dest_idx.index_document(Some(hit.id), hit.source).await;
+ let results = src_idx
+ .search(&search_req)
+ .await
+ .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;
+
+ let batch = results.hits.len();
+ if batch == 0 {
+ break;
+ }
+ search_after = results.hits.last().map(|h| json!([h.id.clone()]));
+
+ for hit in results.hits {
+ if hit.source.is_null() {
+ continue;
+ }
+ let id = hit.id.clone();
+ dest_idx
+ .index_document(Some(hit.id), hit.source)
+ .await
+ .map_err(|e| {
+ ApiError::new(xerj_common::XerjError::internal(format!(
+ "clone of [{source}] into [{target}] failed after {copied} \
+ document(s): document [{id}] could not be written: {e}"
+ )))
+ })?;
+ copied += 1;
+ }
+
+ if batch < CLONE_PAGE_SIZE {
+ break;
}
}
Ok(())
diff --git a/engine/crates/xerj-api/tests/fail_closed_config_http.rs b/engine/crates/xerj-api/tests/fail_closed_config_http.rs
new file mode 100644
index 000000000..5a3914641
--- /dev/null
+++ b/engine/crates/xerj-api/tests/fail_closed_config_http.rs
@@ -0,0 +1,396 @@
+//! Issue #204, from the outside: configuration xerj cannot honour must be
+//! refused at the door, or refused at the point of use — never accepted with a
+//! 200 and then quietly not done.
+//!
+//! Every case here was found by measurement against the first cut of this
+//! branch, and each one is the HTTP-level statement of a defect that had only
+//! engine-level (or no) coverage:
+//!
+//! 1. the create-time analysis gate had no HTTP test at all — neither its
+//! status code nor its ES error shape;
+//! 2. the gate judged a declared filter by its `type` while `apply_settings`
+//! resolves it by NAME, so the canonical Elasticsearch-docs
+//! `english_stop` block started 400ing although xerj serves it correctly;
+//! 3. `PUT /{index}/_settings` walked straight around the gate — it merged an
+//! `analysis` block into the display copy and rebuilt nothing;
+//! 4. `PUT /_ingest/pipeline` was redefined by this branch to mean "compiled ⇒
+//! xerj honours this", and then accepted an ES `grok` `patterns` array, a
+//! processor-level `if` guard, and an `append` it executed as `set`;
+//! 5. `_clone` copied at most 10,000 documents and swallowed every write
+//! error, then answered `{"acknowledged": true}`.
+//!
+//! Elasticsearch is referenced for semantics only. It is AGPL-3.0/SSPL-1.0/
+//! Elastic-2.0 licensed and no code from it is reproduced here.
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{json, Value};
+use tower::ServiceExt;
+
+fn app() -> (axum::Router, tempfile::TempDir) {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+ (xerj_api::router::build_es_compat_router(state), dir)
+}
+
+async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {
+ let res = app.clone().oneshot(req).await.expect("response");
+ let status = res.status();
+ let bytes = axum::body::to_bytes(res.into_body(), 4 * 1024 * 1024)
+ .await
+ .expect("body");
+ let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, body)
+}
+
+fn put(path: &str, body: Value) -> Request<Body> {
+ Request::builder()
+ .method("PUT")
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request")
+}
+
+fn post(path: &str, body: Value) -> Request<Body> {
+ Request::builder()
+ .method("POST")
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request")
+}
+
+fn get(path: &str) -> Request<Body> {
+ Request::builder()
+ .method("GET")
+ .uri(path)
+ .body(Body::empty())
+ .expect("request")
+}
+
+// ── The create-time analysis gate ────────────────────────────────────────────
+
+#[tokio::test]
+async fn an_analysis_block_we_cannot_honour_is_a_400_in_the_es_error_shape() {
+ let (app, _dir) = app();
+ let (status, body) = send(
+ &app,
+ put(
+ "/autocomplete",
+ json!({
+ "settings": { "analysis": { "analyzer": {
+ "ac": { "type": "custom", "tokenizer": "edge_ngram_tok" }
+ } } }
+ }),
+ ),
+ )
+ .await;
+
+ // Pre-fix this was `200 {"acknowledged": true}` and the index tokenised
+ // with `standard` — an autocomplete index that matches nothing.
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert_eq!(body["status"], 400, "{body}");
+ assert!(body["error"]["type"].is_string(), "{body}");
+ assert!(body["error"]["root_cause"].is_array(), "{body}");
+ assert!(
+ body["error"]["reason"]
+ .as_str()
+ .expect("reason")
+ .contains("edge_ngram_tok"),
+ "the reason must name the construct we cannot honour: {body}"
+ );
+
+ // And nothing was left behind.
+ let (status, _) = send(&app, get("/autocomplete")).await;
+ assert_eq!(status, StatusCode::NOT_FOUND);
+}
+
+#[tokio::test]
+async fn the_canonical_elasticsearch_analysis_block_is_still_accepted() {
+ let (app, _dir) = app();
+ // Straight from the Elasticsearch docs' `rebuilt_english`, cut to the parts
+ // xerj implements. `english_stop` declares `type: stop`, which
+ // `apply_settings` cannot BUILD — and then resolves by NAME to the built-in
+ // English stopword filter, which is exactly what it declares. Judging it on
+ // `type` alone made this a 400: a straight regression on a settings block
+ // xerj had always accepted and analysed correctly.
+ let (status, body) = send(
+ &app,
+ put(
+ "/blog",
+ json!({
+ "settings": { "analysis": {
+ "filter": { "english_stop": { "type": "stop", "stopwords": "_english_" } },
+ "analyzer": { "rebuilt_english": {
+ "type": "custom",
+ "tokenizer": "standard",
+ "filter": ["lowercase", "english_stop"]
+ } }
+ } },
+ "mappings": { "properties": {
+ "body": { "type": "text", "analyzer": "rebuilt_english" }
+ } }
+ }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["acknowledged"], true, "{body}");
+}
+
+#[tokio::test]
+async fn analysis_cannot_be_smuggled_past_the_gate_through_put_settings() {
+ let (app, _dir) = app();
+ let (status, _) = send(&app, put("/blog", json!({}))).await;
+ assert_eq!(status, StatusCode::OK);
+
+ // The registry is built at create/open and there is no rebuild path, so
+ // this could only ever have changed what `GET /_settings` echoes back.
+ let (status, body) = send(
+ &app,
+ put(
+ "/blog/_settings",
+ json!({ "analysis": { "analyzer": {
+ "late": { "type": "custom", "tokenizer": "whitespace" }
+ } } }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert!(
+ body["error"]["reason"]
+ .as_str()
+ .expect("reason")
+ .contains("analysis"),
+ "{body}"
+ );
+
+ // …and the display copy was not quietly updated either.
+ let (_, settings) = send(&app, get("/blog/_settings")).await;
+ assert!(
+ settings.pointer("/blog/settings/index/analysis").is_none(),
+ "a refused settings PUT must not leave the analysis block behind: {settings}"
+ );
+
+ // A settings PUT that xerj *can* honour is unaffected.
+ let (status, _) = send(
+ &app,
+ put(
+ "/blog/_settings",
+ json!({ "index": { "number_of_replicas": 0 } }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+}
+
+// ── PUT /_ingest/pipeline: compiled ⇒ honoured ───────────────────────────────
+
+/// Register `pipeline`, then write one document through it. Returns the write's
+/// status and body.
+async fn write_through(
+ app: &axum::Router,
+ id: &str,
+ processors: Value,
+ doc: Value,
+) -> (StatusCode, Value) {
+ let (status, body) = send(
+ app,
+ put(
+ &format!("/_ingest/pipeline/{id}"),
+ json!({ "processors": processors }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "pipeline definition: {body}");
+ assert_eq!(body["acknowledged"], true, "{body}");
+ send(
+ app,
+ put(&format!("/docs/_doc/1?pipeline={id}&refresh=true"), doc),
+ )
+ .await
+}
+
+#[tokio::test]
+async fn an_es_grok_processor_is_accepted_and_then_refuses_the_write() {
+ let (app, _dir) = app();
+ // Measured pre-fix: this compiled to the SYSLOG default, answered 200, and
+ // indexed `{"message": "10.0.0.1 GET"}` with no `client` field anywhere.
+ let (status, body) = write_through(
+ &app,
+ "grok_pipe",
+ json!([{ "grok": { "field": "message", "patterns": ["%{IP:client} %{WORD:method}"] } }]),
+ json!({ "message": "10.0.0.1 GET" }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ let reason = body["error"]["reason"].as_str().expect("reason");
+ assert!(reason.contains("patterns"), "{body}");
+
+ // The document was NOT indexed untransformed.
+ let (status, _) = send(&app, get("/docs/_doc/1")).await;
+ assert_ne!(status, StatusCode::OK, "the write must not have happened");
+}
+
+#[tokio::test]
+async fn a_processor_level_if_guard_is_accepted_and_then_refuses_the_write() {
+ let (app, _dir) = app();
+ // Measured pre-fix: `{"foo": "something-else"}` was indexed as
+ // `{"foo": "something-else", "env": "prod"}` — the guard excluded it.
+ let (status, body) = write_through(
+ &app,
+ "guarded",
+ json!([{ "set": { "field": "env", "value": "prod", "if": "ctx.foo == 'never'" } }]),
+ json!({ "foo": "something-else" }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");
+ assert!(
+ body["error"]["reason"]
+ .as_str()
+ .expect("reason")
+ .contains("if"),
+ "{body}"
+ );
+}
+
+#[tokio::test]
+async fn set_overrides_an_existing_value_like_elasticsearch() {
+ let (app, _dir) = app();
+ let (status, body) = write_through(
+ &app,
+ "setter",
+ json!([{ "set": { "field": "env", "value": "prod" } }]),
+ json!({ "env": "dev" }),
+ )
+ .await;
+ assert!(status.is_success(), "{body}");
+
+ let (_, doc) = send(&app, get("/docs/_doc/1")).await;
+ // Pre-fix: `override` defaulted to false and this stayed "dev".
+ assert_eq!(doc["_source"]["env"], "prod", "{doc}");
+}
+
+#[tokio::test]
+async fn append_appends_rather_than_replacing() {
+ let (app, _dir) = app();
+ let (status, body) = write_through(
+ &app,
+ "appender",
+ json!([{ "append": { "field": "tags", "value": "b" } }]),
+ json!({ "tags": ["a"] }),
+ )
+ .await;
+ assert!(status.is_success(), "{body}");
+
+ let (_, doc) = send(&app, get("/docs/_doc/1")).await;
+ // Pre-fix: `append` was mapped onto `set`, so this was `"b"` — while the
+ // `_simulate` interpreter for the same pipeline produced `["a","b"]`.
+ assert_eq!(doc["_source"]["tags"], json!(["a", "b"]), "{doc}");
+}
+
+#[tokio::test]
+async fn convert_accepts_the_elasticsearch_spellings_of_the_same_conversion() {
+ let (app, _dir) = app();
+ // `long` is xerj's `integer` (i64) and `double` is its `float` (f64) — an
+ // equivalent fallback, which issue #204 explicitly permits. Refusing them
+ // 400ed previously-working ES pipelines for no gain.
+ let (status, body) = write_through(
+ &app,
+ "converter",
+ json!([
+ { "convert": { "field": "count", "type": "long" } },
+ { "convert": { "field": "ratio", "type": "double" } }
+ ]),
+ json!({ "count": "404", "ratio": "1.5" }),
+ )
+ .await;
+ assert!(status.is_success(), "{body}");
+
+ let (_, doc) = send(&app, get("/docs/_doc/1")).await;
+ assert_eq!(doc["_source"]["count"], 404, "{doc}");
+ assert_eq!(doc["_source"]["ratio"], 1.5, "{doc}");
+
+ // `auto` has no equivalent here and stays refused.
+ let (status, _) = send(
+ &app,
+ put(
+ "/_ingest/pipeline/auto_conv",
+ json!({ "processors": [{ "convert": { "field": "c", "type": "auto" } }] }),
+ ),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST);
+}
+
+// ── _clone copies everything, or says it did not ─────────────────────────────
+
+/// More documents than one clone page (1,000) and more than the old hardcoded
+/// `size: 10000` ceiling, so the test fails on both the truncation and the
+/// paging bug rather than only the second.
+#[tokio::test]
+async fn clone_copies_every_document_not_the_first_ten_thousand() {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = xerj_api::state::AppState::new(config, engine, metrics);
+
+ const DOCS: usize = 10_001;
+ state
+ .engine
+ .create_index("src", xerj_common::types::Schema::empty())
+ .expect("create_index");
+ let idx = state.engine.get_index("src").expect("get_index");
+ for i in 0..DOCS {
+ idx.index_document(Some(format!("{i:08}")), json!({ "n": i }))
+ .await
+ .expect("index_document");
+ }
+ idx.refresh().await.expect("refresh");
+
+ let app = xerj_api::router::build_es_compat_router(state);
+
+ // The source really does hold more than the old ceiling, and the single
+ // `size: 10000` search the pre-fix clone used really does stop at it — so
+ // the assertion below fails on the old code rather than passing vacuously.
+ let (_, src_count) = send(&app, get("/src/_count")).await;
+ assert_eq!(
+ src_count["count"].as_u64(),
+ Some(DOCS as u64),
+ "{src_count}"
+ );
+ let (_, capped) = send(
+ &app,
+ post(
+ "/src/_search",
+ json!({ "query": { "match_all": {} }, "size": 10000 }),
+ ),
+ )
+ .await;
+ assert_eq!(
+ capped["hits"]["hits"].as_array().map(Vec::len),
+ Some(10_000),
+ "the pre-fix clone's one-shot search tops out here"
+ );
+
+ let (status, body) = send(&app, post("/src/_clone/dst", json!({}))).await;
+ assert_eq!(status, StatusCode::OK, "{body}");
+ assert_eq!(body["acknowledged"], true, "{body}");
+
+ let (_, count) = send(&app, get("/dst/_count")).await;
+ // Pre-fix: exactly 10000 — a partial copy reported as a completed one.
+ assert_eq!(
+ count["count"].as_u64(),
+ Some(DOCS as u64),
+ "clone must copy every document: {count}"
+ );
+}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index ea5b82820..2e4b7e8db 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1694,9 +1694,11 @@ impl Engine {
/// Store a pipeline definition that could not be compiled, together with
/// the reason, and make sure nothing stale is left behind.
///
- /// Issue #204. This exists for exactly one case: a definition xerj accepts
- /// for wire compatibility (Elasticsearch does) but cannot execute, because
- /// it names a processor this build does not implement. Storing it keeps
+ /// Issue #204. This exists for one case: a definition xerj accepts for wire
+ /// compatibility (Elasticsearch does) but cannot execute — because it names
+ /// a processor this build does not implement, or sets an option on one that
+ /// this build cannot honour (a processor-level `if` guard, an `on_failure`
+ /// chain, a `grok` `patterns` array). Storing it keeps
/// `GET`/`_simulate` truthful about what was submitted; recording the
/// reason is what stops [`Self::process_through_pipeline`] from silently
/// passing documents through untransformed. A previously-compiled pipeline
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 71cd398a0..93689208c 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -17775,6 +17775,10 @@ impl Index {
/// Number of times [`Self::persist_evolved_schema`] could not write
/// `schema.json`. Non-zero means the on-disk mapping is behind the
/// in-memory one and a restart will lose dynamically-mapped fields.
+ ///
+ /// Read by `GET /{index}/_stats` and `GET /_all/_stats`, which report it as
+ /// `primaries.mappings.schema_persist_failures` — a health check can see it
+ /// without reading logs.
pub fn schema_persist_failures(&self) -> u64 {
self.schema_persist_failures.load(Ordering::Relaxed)
}
diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rs
index 031250d70..490c749df 100644
--- a/engine/crates/xerj-fts/src/analyzer.rs
+++ b/engine/crates/xerj-fts/src/analyzer.rs
@@ -1312,19 +1312,43 @@ impl AnalyzerRegistry {
/// existing index. That leniency is right at open time and wrong at create
/// time, where every one of these constructs used to be accepted with a
/// `200 {"acknowledged": true}` and then quietly replaced by something
- /// weaker:
+ /// weaker.
///
- /// - an unresolvable `tokenizer` became `standard` — an `edge_ngram`
- /// autocomplete index that matches nothing;
- /// - an unresolvable `filter` was dropped — a missing `lowercase` turns
- /// every match case-sensitive;
- /// - an unsupported filter/tokenizer `type` was skipped entirely;
- /// - an unknown non-custom analyzer `type` left the analyzer unregistered,
- /// so fields naming it silently got `standard`.
+ /// # What is checked
+ ///
+ /// 1. a declared `filter` / `tokenizer` whose `type` this build cannot
+ /// construct **and** whose *name* does not resolve to an equivalent
+ /// built-in (see [`Self::builtin_filter_honours`] — the by-name fallback
+ /// `apply_settings` actually takes);
+ /// 2. a `pattern` tokenizer whose regex does not compile;
+ /// 3. an analyzer naming a `tokenizer` / `filter` that is neither declared
+ /// nor built in — the unresolvable tokenizer used to become `standard`
+ /// (an `edge_ngram` autocomplete index that matches nothing) and the
+ /// unresolvable filter used to be dropped (a missing `lowercase` turns
+ /// every match case-sensitive);
+ /// 4. an unknown non-custom analyzer `type`, which left the analyzer
+ /// unregistered so fields naming it silently got `standard`;
+ /// 5. `char_filter` — declared at either level and never built by
+ /// `apply_settings`, so the stripping/mapping never happens;
+ /// 6. `normalizer` — accepted by ES for `keyword` fields, never built here;
+ /// 7. an analyzer whose `filter` is not an array or whose `tokenizer` is
+ /// not a string: `apply_settings` drops the value shape-first and falls
+ /// back to "no filters" / `standard`.
+ ///
+ /// # What is NOT checked
+ ///
+ /// Option-level divergence *inside* a type this build does construct — e.g.
+ /// `synonym.synonyms_path` (file-backed rule lists are not read),
+ /// `length.min` (defaults to 2 here, 0 in ES), `asciifolding
+ /// .preserve_original`. Those are pre-existing gaps, tracked separately;
+ /// this function does not claim to cover them, and the by-name fallback in
+ /// (1) is deliberately conservative about them: a name that resolves to a
+ /// built-in is only accepted when the declared options ask for exactly what
+ /// that built-in does.
///
/// Returns one human-readable message per problem, empty when the block can
- /// be honoured exactly as written. `Index::create_with_settings` turns a
- /// non-empty result into a 400 so the caller learns at the door.
+ /// be honoured as written. `Index::create_with_settings` turns a non-empty
+ /// result into a 400 so the caller learns at the door.
///
/// `settings` may be either the full envelope (`{"settings": {"analysis":
/// …}}`) or the inner settings object — the same two shapes
@@ -1348,13 +1372,39 @@ impl AnalyzerRegistry {
.get("type")
.and_then(|v| v.as_str())
.unwrap_or("");
- if !SUPPORTED_FILTER_TYPES.contains(&filter_type) {
+ if SUPPORTED_FILTER_TYPES.contains(&filter_type) {
+ continue;
+ }
+ // The `type` is one `apply_settings` cannot BUILD — but that is
+ // not the whole story, and reading only the type is what made
+ // this check reject settings blocks xerj serves correctly.
+ // `apply_settings` looks the name up in `custom_filters` first
+ // and falls through to `resolve_builtin_filter(name)`
+ // (analyzer.rs:1284-1286), so `{"english_stop": {"type":
+ // "stop", "stopwords": "_english_"}}` — the canonical
+ // Elasticsearch-docs `rebuilt_english` block — really does
+ // strip English stopwords. Complaining about its type 400s a
+ // `PUT /{index}` that used to work AND analysed as asked.
+ if Self::builtin_filter_honours(filter_name, filter_def) {
+ continue;
+ }
+ if probe.resolve_builtin_filter(filter_name).is_some() {
+ // The name resolves, but to something that is NOT what the
+ // declaration asks for (a custom `stopwords` list, another
+ // `language`). Saying "unsupported type" here would be
+ // misleading — the type is fine, the options are not.
problems.push(format!(
- "token filter [{filter_name}]: unsupported type [{filter_type}] \
- (supported: {})",
- SUPPORTED_FILTER_TYPES.join(", ")
+ "token filter [{filter_name}]: [{filter_type}] with these options is \
+ not supported — the built-in [{filter_name}] filter would be used \
+ instead, which is not what this declares"
));
+ continue;
}
+ problems.push(format!(
+ "token filter [{filter_name}]: unsupported type [{filter_type}] \
+ (supported: {})",
+ SUPPORTED_FILTER_TYPES.join(", ")
+ ));
}
}
@@ -1364,6 +1414,20 @@ impl AnalyzerRegistry {
declared_tokenizers.insert(tok_name.as_str());
let tok_type = tok_def.get("type").and_then(|v| v.as_str()).unwrap_or("");
if !SUPPORTED_TOKENIZER_TYPES.contains(&tok_type) {
+ // Same by-name fallback as filters: `apply_settings` reaches
+ // `resolve_builtin_tokenizer(name)` (analyzer.rs:1454-1473)
+ // when the declared type built nothing.
+ if Self::builtin_tokenizer_honours(tok_name, tok_def) {
+ continue;
+ }
+ if probe.resolve_builtin_tokenizer(tok_name).is_some() {
+ problems.push(format!(
+ "tokenizer [{tok_name}]: [{tok_type}] with these options is not \
+ supported — the built-in [{tok_name}] tokenizer would be used \
+ instead, which is not what this declares"
+ ));
+ continue;
+ }
problems.push(format!(
"tokenizer [{tok_name}]: unsupported type [{tok_type}] \
(supported: {})",
@@ -1385,6 +1449,34 @@ impl AnalyzerRegistry {
}
}
+ // `char_filter` never reaches the built pipeline: every analyzer is
+ // registered as `AnalyzerPipeline::new(vec![], tokenizer, filters)` —
+ // the char-filter slot is hard-coded empty. A declared `html_strip` or
+ // `mapping` therefore strips/maps nothing, with no signal anywhere.
+ if analysis
+ .pointer("/char_filter")
+ .and_then(|v| v.as_object())
+ .is_some_and(|m| !m.is_empty())
+ {
+ problems.push(
+ "char filters are not supported (analysis.char_filter is declared but no \
+ char filter is ever applied)"
+ .to_string(),
+ );
+ }
+ // `normalizer` is the keyword-field analogue and is equally unbuilt.
+ if analysis
+ .pointer("/normalizer")
+ .and_then(|v| v.as_object())
+ .is_some_and(|m| !m.is_empty())
+ {
+ problems.push(
+ "normalizers are not supported (analysis.normalizer is declared but never \
+ built, so keyword fields naming one are indexed unchanged)"
+ .to_string(),
+ );
+ }
+
if let Some(analyzer_map) = analysis.pointer("/analyzer").and_then(|v| v.as_object()) {
for (analyzer_name, analyzer_def) in analyzer_map {
let analyzer_type = analyzer_def
@@ -1401,6 +1493,49 @@ impl AnalyzerRegistry {
continue;
}
+ // Per-analyzer char filters: same hard-coded-empty slot.
+ if analyzer_def
+ .get("char_filter")
+ .is_some_and(|v| !matches!(v, serde_json::Value::Array(a) if a.is_empty()))
+ {
+ problems.push(format!(
+ "analyzer [{analyzer_name}]: `char_filter` is not supported — the \
+ declared char filters are never applied"
+ ));
+ }
+
+ // Shape errors are silent losses, not type errors:
+ // `apply_settings` reads `tokenizer` with `as_str()` and
+ // `filter` with `as_array()`, so a wrong-shaped value is
+ // dropped and the analyzer is built with `standard` / no
+ // filters — accepted, and not what was written.
+ match analyzer_def.get("tokenizer") {
+ None | Some(serde_json::Value::String(_)) => {}
+ Some(other) => problems.push(format!(
+ "analyzer [{analyzer_name}]: `tokenizer` must be a tokenizer name \
+ (string), got {other} — it would be ignored and `standard` used"
+ )),
+ }
+ let mut filter_names: Vec<&str> = Vec::new();
+ match analyzer_def.get("filter") {
+ None => {}
+ Some(serde_json::Value::Array(arr)) => {
+ for v in arr {
+ match v.as_str() {
+ Some(s) => filter_names.push(s),
+ None => problems.push(format!(
+ "analyzer [{analyzer_name}]: `filter` entries must be \
+ filter names (strings), got {v} — it would be ignored"
+ )),
+ }
+ }
+ }
+ Some(other) => problems.push(format!(
+ "analyzer [{analyzer_name}]: `filter` must be an array of filter \
+ names, got {other} — it would be ignored and no filters applied"
+ )),
+ }
+
let tokenizer_name = analyzer_def
.get("tokenizer")
.and_then(|v| v.as_str())
@@ -1413,11 +1548,6 @@ impl AnalyzerRegistry {
));
}
- let filter_names: Vec<&str> = analyzer_def
- .get("filter")
- .and_then(|v| v.as_array())
- .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
- .unwrap_or_default();
for fname in filter_names {
if !declared_filters.contains(fname)
&& probe.resolve_builtin_filter(fname).is_none()
@@ -1431,9 +1561,72 @@ impl AnalyzerRegistry {
}
problems.sort();
+ problems.dedup();
problems
}
+ /// Does a declared token filter resolve, by NAME, to a built-in that does
+ /// exactly what the declaration asks for?
+ ///
+ /// This is the fallback `apply_settings` genuinely takes: a `type` it
+ /// cannot build leaves `custom_filters` without an entry, and the analyzer
+ /// loop then calls [`Self::resolve_builtin_filter`] with the *declared
+ /// name*. Judging such a declaration on its `type` alone reports a filter
+ /// that is in fact honoured — and, through the create-time gate, 400s a
+ /// `PUT /{index}` that previously worked.
+ ///
+ /// Deliberately narrow: only the exact option sets the built-in reproduces
+ /// return `true`. A custom `stopwords` list under the name `english_stop`
+ /// is still a problem, because the English list is what would actually run.
+ fn builtin_filter_honours(name: &str, def: &serde_json::Value) -> bool {
+ let ty = def.get("type").and_then(|v| v.as_str()).unwrap_or("");
+ match (name, ty) {
+ // `LowercaseFilter`. ES's `lowercase` filter takes an optional
+ // `language` (greek/irish/turkish); only the default is this.
+ ("lowercase", "lowercase") => def.get("language").is_none(),
+ // `StopwordsFilter::english()`. ES's `stop` defaults to
+ // `_english_`, which is exactly the list built here.
+ ("stop" | "english_stop", "stop") => {
+ def.get("stopwords_path").is_none()
+ && match def.get("stopwords") {
+ None => true,
+ Some(serde_json::Value::String(s)) => s == "_english_",
+ _ => false,
+ }
+ && def.get("ignore_case").is_none()
+ && def.get("remove_trailing").is_none()
+ }
+ // `StemmerFilter::english()` — rust-stemmers' Snowball English.
+ ("stemmer" | "english_stemmer", "stemmer") => {
+ match def.get("language").or_else(|| def.get("name")) {
+ None => true,
+ Some(serde_json::Value::String(s)) => {
+ matches!(s.as_str(), "english" | "porter2")
+ }
+ _ => false,
+ }
+ }
+ _ => false,
+ }
+ }
+
+ /// Tokenizer counterpart of [`Self::builtin_filter_honours`].
+ ///
+ /// `apply_settings` resolves an unbuilt tokenizer by declared name through
+ /// [`Self::resolve_builtin_tokenizer`], so `{"standard": {"type":
+ /// "standard"}}` is honoured exactly as written.
+ fn builtin_tokenizer_honours(name: &str, def: &serde_json::Value) -> bool {
+ let ty = def.get("type").and_then(|v| v.as_str()).unwrap_or("");
+ match (name, ty) {
+ // `max_token_length` is not implemented by `StandardTokenizer`.
+ ("standard", "standard") => def.get("max_token_length").is_none(),
+ ("whitespace", "whitespace") => def.get("max_token_length").is_none(),
+ // `buffer_size` is a performance knob with no semantic effect.
+ ("keyword", "keyword") => true,
+ _ => false,
+ }
+ }
+
/// Locate the `analysis` block inside an index-settings object.
///
/// ES accepts both the shorthand (`settings.analysis.*`) and the canonical
@@ -2373,4 +2566,150 @@ mod unsupported_analysis_tests {
);
}
}
+
+ /// The canonical Elasticsearch-docs `rebuilt_english` shape, cut down to the
+ /// parts xerj can serve. Judging `english_stop` on its `type` alone made
+ /// `unsupported_analysis` report it as unsupported — and, through the
+ /// create-time gate, 400 a `PUT /{index}` that xerj had always accepted AND
+ /// analysed correctly. Both halves are asserted here: no complaint, and the
+ /// registry built from the same block really does strip English stopwords.
+ #[test]
+ fn declared_filter_resolved_by_name_is_honoured_not_reported() {
+ let settings = json!({
+ "analysis": {
+ "filter": {
+ "english_stop": { "type": "stop", "stopwords": "_english_" }
+ },
+ "analyzer": {
+ "rebuilt_english": {
+ "type": "custom",
+ "tokenizer": "standard",
+ "filter": ["lowercase", "english_stop"]
+ }
+ }
+ }
+ });
+ assert_eq!(
+ AnalyzerRegistry::unsupported_analysis(&settings),
+ Vec::<String>::new()
+ );
+
+ let mut registry = AnalyzerRegistry::with_defaults();
+ registry.apply_settings(&settings);
+ let terms = registry
+ .get_analyzer("rebuilt_english")
+ .expect("analyzer must be registered")
+ .analyze_to_terms("The Quick Brown Fox");
+ assert_eq!(
+ terms,
+ vec!["quick".to_string(), "brown".to_string(), "fox".to_string()],
+ "the built registry must apply the stopword filter the block declares"
+ );
+ }
+
+ /// …but only when the built-in is what the declaration actually asks for.
+ /// A custom stopword list under the same name is NOT honoured — the English
+ /// list would run instead — so it must still be reported, and with a message
+ /// about the options rather than a misleading "unsupported type".
+ #[test]
+ fn declared_filter_with_options_the_builtin_cannot_reproduce_is_reported() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "filter": {
+ "english_stop": { "type": "stop", "stopwords": ["pelican", "walrus"] }
+ }
+ }
+ }));
+ assert_eq!(problems.len(), 1, "{problems:?}");
+ assert!(
+ problems[0].contains("not supported") && problems[0].contains("english_stop"),
+ "{problems:?}"
+ );
+ }
+
+ #[test]
+ fn builtin_tokenizer_resolved_by_name_is_honoured_not_reported() {
+ assert_eq!(
+ AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "tokenizer": { "whitespace": { "type": "whitespace" } },
+ "analyzer": { "a": { "type": "custom", "tokenizer": "whitespace" } }
+ }
+ })),
+ Vec::<String>::new()
+ );
+ }
+
+ /// `AnalyzerPipeline::new(vec![], …)` — the char-filter slot is hard-coded
+ /// empty, so a declared `html_strip` strips nothing. Measured pre-fix:
+ /// `<b>hello</b>` tokenised to `["b", "hello", "b"]` with `problems == []`.
+ #[test]
+ fn char_filters_are_reported_because_they_are_never_built() {
+ let settings = json!({
+ "analysis": {
+ "char_filter": { "strip_html": { "type": "html_strip" } },
+ "analyzer": {
+ "a": {
+ "type": "custom",
+ "tokenizer": "standard",
+ "char_filter": ["strip_html"]
+ }
+ }
+ }
+ });
+ let problems = AnalyzerRegistry::unsupported_analysis(&settings);
+ assert_eq!(problems.len(), 2, "{problems:?}");
+ assert!(
+ problems.iter().all(|p| p.contains("char filter")),
+ "{problems:?}"
+ );
+
+ // The reason it must be reported: nothing strips the tags.
+ let mut registry = AnalyzerRegistry::with_defaults();
+ registry.apply_settings(&settings);
+ assert!(
+ registry
+ .get_analyzer("a")
+ .expect("registered")
+ .analyze_to_terms("<b>hello</b>")
+ .contains(&"b".to_string()),
+ "char filters really are not applied — that is why they are refused"
+ );
+ }
+
+ #[test]
+ fn normalizers_are_reported_because_they_are_never_built() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "normalizer": { "lower": { "type": "custom", "filter": ["lowercase"] } }
+ }
+ }));
+ assert_eq!(problems.len(), 1, "{problems:?}");
+ assert!(problems[0].contains("normalizer"), "{problems:?}");
+ }
+
+ /// `apply_settings` reads `filter` with `as_array()`, so a bare string is
+ /// dropped and the analyzer is built with no filters at all. Measured
+ /// pre-fix: `problems == []`, terms `["ABC"]` from `"ABC"` — the
+ /// `lowercase` the caller wrote never ran.
+ #[test]
+ fn wrong_shaped_analyzer_keys_are_reported() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "analyzer": {
+ "a": { "type": "custom", "tokenizer": "standard", "filter": "lowercase" },
+ "b": { "type": "custom", "tokenizer": ["standard"] }
+ }
+ }
+ }));
+ assert_eq!(problems.len(), 2, "{problems:?}");
+ assert!(
+ problems.iter().any(|p| p.contains("`filter` must be")),
+ "{problems:?}"
+ );
+ assert!(
+ problems.iter().any(|p| p.contains("`tokenizer` must be")),
+ "{problems:?}"
+ );
+ }
}
diff --git a/engine/crates/xerj-wasm/src/builtins.rs b/engine/crates/xerj-wasm/src/builtins.rs
index 5c40fdfc2..69e32d3ef 100644
--- a/engine/crates/xerj-wasm/src/builtins.rs
+++ b/engine/crates/xerj-wasm/src/builtins.rs
@@ -873,6 +873,68 @@ impl TransformPlugin for SetPlugin {
}
}
+// ─────────────────────────────────────────────────────────────────────────────
+// AppendPlugin
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Append a value to a field, creating or widening a list as needed.
+///
+/// Elasticsearch's `append` semantics, which is what this exists for: the field
+/// becomes a list if it is not one already, a list `value` extends rather than
+/// nests, and a missing field is created as a single-element list
+/// (`IngestDocument.appendFieldValue`, read for semantics only).
+///
+/// Issue #204: `append` used to be *mapped onto `set`* in the ES→xerj processor
+/// translation (`es_compat.rs`), so `{"append": {"field": "tags", "value":
+/// "b"}}` REPLACED `["a"]` with `"b"` instead of producing `["a", "b"]` —
+/// under a `200 {"acknowledged": true}`, and disagreeing with the `_simulate`
+/// interpreter, which has always implemented a real append. Two behaviours for
+/// one pipeline; this is the one that runs at ingest.
+pub struct AppendPlugin {
+ field: String,
+ value: Value,
+}
+
+impl AppendPlugin {
+ pub fn new(field: impl Into<String>, value: Value) -> Self {
+ Self {
+ field: field.into(),
+ value,
+ }
+ }
+}
+
+impl TransformPlugin for AppendPlugin {
+ fn name(&self) -> &str {
+ "append"
+ }
+
+ fn process(&self, doc: &mut Value) -> ProcessAction {
+ let Some(obj) = doc.as_object_mut() else {
+ return ProcessAction::Pass;
+ };
+ let slot = obj
+ .entry(self.field.clone())
+ .or_insert_with(|| Value::Array(Vec::new()));
+ match slot {
+ Value::Array(existing) => match self.value.clone() {
+ Value::Array(more) => existing.extend(more),
+ one => existing.push(one),
+ },
+ scalar => {
+ let previous = scalar.clone();
+ let mut out = vec![previous];
+ match self.value.clone() {
+ Value::Array(more) => out.extend(more),
+ one => out.push(one),
+ }
+ *scalar = Value::Array(out);
+ }
+ }
+ ProcessAction::Pass
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// RemoveNullPlugin
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-wasm/src/lib.rs b/engine/crates/xerj-wasm/src/lib.rs
index 9af56875d..4fa0a8de9 100644
--- a/engine/crates/xerj-wasm/src/lib.rs
+++ b/engine/crates/xerj-wasm/src/lib.rs
@@ -62,6 +62,24 @@ pub enum WasmError {
#[error("unsupported stage type '{stage}'")]
UnsupportedStage { stage: String },
+ /// The stage type IS implemented, but the caller set an option this build
+ /// cannot honour — an Elasticsearch processor-level `if` guard, an
+ /// `on_failure` recovery chain, a `grok` `patterns` array.
+ ///
+ /// Issue #204 again, one level down. Reporting these as `InvalidConfig`
+ /// would tell a caller their valid ES processor is malformed; ignoring them
+ /// is what the sweep exists to kill — a `set` that runs on documents its
+ /// `if` excludes is a silently wrong document, not a cosmetic gap. Handled
+ /// exactly like [`WasmError::UnsupportedStage`] at the API boundary: the
+ /// definition is accepted (ES accepts it) and recorded as unrunnable, so
+ /// every ingest through it refuses loudly.
+ #[error("stage '{stage}': option '{option}' is not supported by this build ({reason})")]
+ UnsupportedOption {
+ stage: String,
+ option: String,
+ reason: String,
+ },
+
/// The pipeline is registered but carries a stage this build cannot run,
/// so no document may pass through it.
#[error("pipeline '{pipeline}' is registered but cannot run: {reason}")]
diff --git a/engine/crates/xerj-wasm/src/pipeline.rs b/engine/crates/xerj-wasm/src/pipeline.rs
index 303b9d0b3..de8c583f0 100644
--- a/engine/crates/xerj-wasm/src/pipeline.rs
+++ b/engine/crates/xerj-wasm/src/pipeline.rs
@@ -11,10 +11,10 @@ use tracing::{debug, warn};
use crate::{
builtins::{
- AddFieldPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin, FieldRenamePlugin,
- GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin, RemoveNullPlugin,
- RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin, UppercasePlugin,
- UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,
+ AddFieldPlugin, AppendPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin,
+ FieldRenamePlugin, GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin,
+ RemoveNullPlugin, RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin,
+ UppercasePlugin, UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,
},
Result, TransformPlugin, WasmError,
};
@@ -99,15 +99,17 @@ pub struct Pipeline {
impl Pipeline {
/// Build a [`Pipeline`] from a [`PipelineConfig`].
///
- /// Returns [`WasmError::InvalidConfig`] when a plugin-specific config is
- /// malformed, and [`WasmError::UnsupportedStage`] when the stage type
- /// itself is one xerj does not implement.
+ /// Three outcomes, deliberately distinct (issue #204):
///
- /// Those two are deliberately distinct (issue #204). A malformed config is
- /// a caller error and the definition must be refused. An unimplemented
- /// stage type is a *xerj capability gap*: the caller wrote something
- /// legitimate that we cannot run, and the honest answer is to say so at the
- /// point of use rather than pretend either way.
+ /// - [`WasmError::InvalidConfig`] — the caller's mistake. The definition
+ /// must be refused.
+ /// - [`WasmError::UnsupportedStage`] — a *xerj capability gap*: the caller
+ /// wrote something legitimate that we cannot run. The honest answer is to
+ /// say so at the point of use rather than pretend either way.
+ /// - [`WasmError::UnsupportedOption`] — the same gap one level down: the
+ /// stage exists, an option on it does not (a processor-level `if` guard,
+ /// a `grok` `patterns` array). Silently dropping these is what makes a
+ /// compiled pipeline write the wrong document under a `201`.
pub fn from_config(name: impl Into<String>, config: &PipelineConfig) -> Result<Self> {
let name = name.into();
let mut stages: Vec<Arc<dyn TransformPlugin>> = Vec::new();
@@ -116,6 +118,13 @@ impl Pipeline {
let plugin =
build_plugin(&stage_cfg.stage_type, &stage_cfg.config).map_err(|e| match e {
StageBuildError::Unsupported(stage) => WasmError::UnsupportedStage { stage },
+ StageBuildError::UnsupportedOption { option, reason } => {
+ WasmError::UnsupportedOption {
+ stage: stage_cfg.stage_type.clone(),
+ option,
+ reason,
+ }
+ }
StageBuildError::Config(reason) => WasmError::InvalidConfig {
plugin: stage_cfg.stage_type.clone(),
reason,
@@ -191,14 +200,17 @@ impl std::fmt::Debug for Pipeline {
/// Why [`build_plugin`] refused a stage.
///
-/// Split in two on purpose (issue #204): `Config` is the caller's mistake and
-/// must be refused at definition time; `Unsupported` is xerj's gap and has to
-/// be reported as such so callers are not told their processor is invalid when
-/// it is merely unimplemented here.
+/// Split three ways on purpose (issue #204): `Config` is the caller's mistake
+/// and must be refused at definition time; `Unsupported` and
+/// `UnsupportedOption` are xerj's gaps and have to be reported as such, so
+/// callers are not told their processor is invalid when it is merely
+/// unimplemented here.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum StageBuildError {
/// The stage type is not implemented by this build.
Unsupported(String),
+ /// The stage type IS implemented but an option on it is not.
+ UnsupportedOption { option: String, reason: String },
/// The stage type exists but its config cannot be honoured.
Config(String),
}
@@ -209,11 +221,47 @@ impl From<String> for StageBuildError {
}
}
+/// Elasticsearch processor-level keys that decide **whether** a processor runs
+/// or **what happens when it fails**, none of which the compiled pipeline can
+/// act on.
+///
+/// Issue #204, applied to this PR's own new surface. `PUT /_ingest/pipeline`
+/// now means "compiled ⇒ xerj honours this"; a `{"set": {…, "if": "ctx.foo ==
+/// 'never'"}}` that compiles and then sets the field on every document makes
+/// that promise false, and writes the wrong document under a `201`. The
+/// pipeline is accepted (ES accepts it) and recorded as unrunnable, so the gap
+/// surfaces at ingest instead of being silently resolved in the caller's face.
+///
+/// `tag` and `description` are deliberately NOT here: they are identification
+/// metadata that changes no document and no decision on this path.
+const UNHONOURED_PROCESSOR_KEYS: &[(&str, &str)] = &[
+ (
+ "if",
+ "the guard is not evaluated, so the processor would run on documents it excludes",
+ ),
+ (
+ "on_failure",
+ "the recovery processors are not run; the pipeline's `on_error` policy applies instead",
+ ),
+ (
+ "ignore_failure",
+ "a failing processor is not ignored; the pipeline's `on_error` policy applies instead",
+ ),
+];
+
/// Instantiate a built-in plugin by name.
fn build_plugin(
stage_type: &str,
config: &Value,
) -> std::result::Result<Arc<dyn TransformPlugin>, StageBuildError> {
+ for (key, reason) in UNHONOURED_PROCESSOR_KEYS {
+ if config.get(key).is_some() {
+ return Err(StageBuildError::UnsupportedOption {
+ option: (*key).to_string(),
+ reason: (*reason).to_string(),
+ });
+ }
+ }
match stage_type {
"json_parse" => {
let field = str_field(config, "field")?;
@@ -280,6 +328,27 @@ fn build_plugin(
"grok" => {
let field = str_field(config, "field")?;
+ // Elasticsearch's `grok` processor is driven by `patterns` — an
+ // ARRAY of grok expressions (`["%{IP:client} %{WORD:method}"]`),
+ // optionally with `pattern_definitions`. xerj's `grok` is a
+ // different, narrower thing: one NAMED pattern out of
+ // `GROK_PATTERN_NAMES`. Reading `pattern` and defaulting to
+ // `SYSLOG` meant an ES grok processor compiled clean, answered
+ // `200 acknowledged` under this PR's "compiled ⇒ honoured"
+ // contract, and left the document untouched (issue #204 — measured:
+ // `{"message": "10.0.0.1 GET"}` came back with no `client` field).
+ for key in ["patterns", "pattern_definitions"] {
+ if config.get(key).is_some() {
+ return Err(StageBuildError::UnsupportedOption {
+ option: key.to_string(),
+ reason: format!(
+ "xerj's grok stage takes a named `pattern` from [{}], not \
+ arbitrary grok expressions",
+ GROK_PATTERN_NAMES.join(", ")
+ ),
+ });
+ }
+ }
let pattern_name = match config.get("pattern") {
None => "SYSLOG".to_string(),
Some(Value::String(s)) => {
@@ -371,13 +440,47 @@ fn build_plugin(
.get("value")
.cloned()
.ok_or_else(|| "missing 'value'".to_string())?;
- let override_existing = config
- .get("override")
- .and_then(Value::as_bool)
- .unwrap_or(false);
+ // Elasticsearch's `set` processor defaults `override` to TRUE
+ // (`SetProcessor.java:135` — `readBooleanProperty(…, "override",
+ // true)`; ES is read for semantics only, no code reproduced). This
+ // was `unwrap_or(false)`, so `{"set": {"field": "env", "value":
+ // "prod"}}` left an existing `{"env": "dev"}` untouched: a weaker
+ // fallback on user configuration, in the file the #204 sweep
+ // rewrote. A present-but-non-boolean `override` is a caller error
+ // rather than a silent fall-through to the default.
+ let override_existing = match config.get("override") {
+ None => true,
+ Some(v) => v
+ .as_bool()
+ .ok_or_else(|| format!("'override' must be a boolean, got {v}"))?,
+ };
Ok(Arc::new(SetPlugin::new(field, value, override_existing)))
}
+ "append" => {
+ let field = str_field(config, "field")?;
+ let value = config
+ .get("value")
+ .cloned()
+ .ok_or_else(|| "missing 'value'".to_string())?;
+ // ES's `allow_duplicates` defaults to true (`AppendProcessor
+ // .java:121`) and that is what `AppendPlugin` does; the
+ // de-duplicating variant is not implemented, so asking for it must
+ // not be silently ignored.
+ match config.get("allow_duplicates") {
+ None | Some(Value::Bool(true)) => {}
+ Some(_) => {
+ return Err(StageBuildError::UnsupportedOption {
+ option: "allow_duplicates".to_string(),
+ reason: "values are always appended; de-duplicating append is not \
+ implemented"
+ .to_string(),
+ })
+ }
+ }
+ Ok(Arc::new(AppendPlugin::new(field, value)))
+ }
+
"remove_null" => Ok(Arc::new(RemoveNullPlugin)),
"url_decode" => {
@@ -667,4 +770,150 @@ mod fallback_regression_tests {
// Absent `formats` is still the documented epoch/RFC-3339 default.
assert!(build("timestamp_parse", serde_json::json!({ "field": "ts" })).is_ok());
}
+
+ /// A processor-level `if` guard is not evaluated on this path. Compiling it
+ /// meant `PUT /_ingest/pipeline` answered 200 — which this change redefined
+ /// to mean "xerj honours this" — and then ran the processor on exactly the
+ /// documents the guard excludes. Measured pre-fix: `{"foo":
+ /// "something-else"}` came out `{"foo": "something-else", "env": "prod"}`.
+ #[test]
+ fn a_processor_level_if_guard_is_not_silently_dropped() {
+ let err = build(
+ "set",
+ serde_json::json!({
+ "field": "env", "value": "prod", "if": "ctx.foo == 'never'"
+ }),
+ )
+ .expect_err("an unhonourable `if` must not compile");
+ assert!(
+ matches!(&err, WasmError::UnsupportedOption { stage, option, .. }
+ if stage == "set" && option == "if"),
+ "expected UnsupportedOption(if), got: {err:?}"
+ );
+ }
+
+ #[test]
+ fn on_failure_and_ignore_failure_are_not_silently_dropped() {
+ for key in ["on_failure", "ignore_failure"] {
+ let mut cfg = serde_json::json!({ "field": "env", "value": "prod" });
+ cfg[key] = serde_json::json!(if key == "ignore_failure" {
+ serde_json::json!(true)
+ } else {
+ serde_json::json!([{ "set": { "field": "x", "value": 1 } }])
+ });
+ let err = build("set", cfg).expect_err("{key} must not be dropped");
+ assert!(
+ matches!(&err, WasmError::UnsupportedOption { option, .. } if option == key),
+ "expected UnsupportedOption({key}), got: {err:?}"
+ );
+ }
+ }
+
+ /// `tag` and `description` are identification metadata — they change no
+ /// document and no decision on this path — so they stay accepted.
+ #[test]
+ fn processor_metadata_keys_still_compile() {
+ assert!(build(
+ "set",
+ serde_json::json!({
+ "field": "env", "value": "prod",
+ "tag": "t1", "description": "sets env"
+ })
+ )
+ .is_ok());
+ }
+
+ /// ES's `grok` is driven by `patterns` (an array of grok expressions);
+ /// xerj's takes one NAMED pattern. Reading `pattern` and defaulting to
+ /// `SYSLOG` meant an ES grok processor compiled clean and left the document
+ /// untouched — measured pre-fix: `{"message": "10.0.0.1 GET"}` with the
+ /// `%{IP:client}` capture nowhere.
+ #[test]
+ fn es_grok_patterns_array_is_not_silently_replaced_by_syslog() {
+ let err = build(
+ "grok",
+ serde_json::json!({
+ "field": "message", "patterns": ["%{IP:client} %{WORD:method}"]
+ }),
+ )
+ .expect_err("`patterns` must not compile to the SYSLOG default");
+ assert!(
+ matches!(&err, WasmError::UnsupportedOption { stage, option, .. }
+ if stage == "grok" && option == "patterns"),
+ "expected UnsupportedOption(patterns), got: {err:?}"
+ );
+ }
+
+ /// Elasticsearch's `set` defaults `override` to true (SetProcessor.java:135
+ /// — read for semantics only). `unwrap_or(false)` made `{"set": {"field":
+ /// "env", "value": "prod"}}` a no-op against an existing `{"env": "dev"}`.
+ #[test]
+ fn set_overrides_by_default_like_elasticsearch() {
+ let pl = build(
+ "set",
+ serde_json::json!({ "field": "env", "value": "prod" }),
+ )
+ .expect("set builds");
+ let mut doc = serde_json::json!({ "env": "dev" });
+ pl.process(&mut doc);
+ assert_eq!(doc["env"], "prod");
+
+ // …and the opt-out still works.
+ let pl = build(
+ "set",
+ serde_json::json!({ "field": "env", "value": "prod", "override": false }),
+ )
+ .expect("set builds");
+ let mut doc = serde_json::json!({ "env": "dev" });
+ pl.process(&mut doc);
+ assert_eq!(doc["env"], "dev");
+
+ // A non-boolean `override` is refused rather than defaulted.
+ assert!(build(
+ "set",
+ serde_json::json!({ "field": "env", "value": "prod", "override": "yes" })
+ )
+ .is_err());
+ }
+
+ /// `append` used to be mapped onto `set` in the ES translation, so it
+ /// REPLACED the field. It is now a real append and agrees with the
+ /// `_simulate` interpreter.
+ #[test]
+ fn append_extends_rather_than_replacing() {
+ let pl = build(
+ "append",
+ serde_json::json!({ "field": "tags", "value": "b" }),
+ )
+ .expect("append builds");
+
+ let mut doc = serde_json::json!({ "tags": ["a"] });
+ pl.process(&mut doc);
+ assert_eq!(doc["tags"], serde_json::json!(["a", "b"]));
+
+ // A scalar widens into a list; a missing field becomes a 1-element one.
+ let mut doc = serde_json::json!({ "tags": "a" });
+ pl.process(&mut doc);
+ assert_eq!(doc["tags"], serde_json::json!(["a", "b"]));
+ let mut doc = serde_json::json!({});
+ pl.process(&mut doc);
+ assert_eq!(doc["tags"], serde_json::json!(["b"]));
+
+ // A list value extends rather than nesting.
+ let pl = build(
+ "append",
+ serde_json::json!({ "field": "tags", "value": ["b", "c"] }),
+ )
+ .expect("append builds");
+ let mut doc = serde_json::json!({ "tags": ["a"] });
+ pl.process(&mut doc);
+ assert_eq!(doc["tags"], serde_json::json!(["a", "b", "c"]));
+
+ // De-duplicating append is not implemented; asking for it is refused.
+ assert!(build(
+ "append",
+ serde_json::json!({ "field": "tags", "value": "b", "allow_duplicates": false })
+ )
+ .is_err());
+ }
}