AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(fts,wasm,api): apply the #204 sweep to the sweep's own new surface
xerj-org272bc7fd2927
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 678c328c2..8a55caf30 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -16540,11 +16540,17 @@ pub async fn index_stats(Ok(idx) => idx.hnsw_stats().await,Err(_) => json!({ "present": false }),};+ let schema_persist_failures = state+ .engine+ .get_index(name)+ .map(|idx| idx.schema_persist_failures())+ .unwrap_or(0);let primaries = json!({"docs": { "count": doc_count, "deleted": 0 },"store": { "size_in_bytes": store_size_bytes },"dense_vector": dv,"hnsw": hnsw,+ "mappings": { "schema_persist_failures": schema_persist_failures },"merges": {"current": if merging { 1 } else { 0 },"current_docs": 0,@@ -16681,11 +16687,24 @@ pub async fn index_stats()}).unwrap_or((0, false));+ // XERJ extension (issue #204): dynamic-mapping evolution that could not be+ // written to `schema.json`. Non-zero means the on-disk mapping is behind+ // the live one and a restart will lose those fields. It is counted rather+ // than raised because schema evolution runs after the document has been+ // admitted — there is no caller left to fail — so a health check needs to+ // be able to READ it. This is that read: without it the counter existed and+ // nothing in the workspace looked at it.+ let schema_persist_failures = state+ .engine+ .get_index(&index)+ .map(|idx| idx.schema_persist_failures())+ .unwrap_or(0);let primaries = json!({"docs": { "count": doc_count, "deleted": 0 },"store": { "size_in_bytes": store_size_bytes },"dense_vector": dense_vector_stats,"hnsw": hnsw_stats,+ "mappings": { "schema_persist_failures": schema_persist_failures },"indexing": {// Real cumulative indexing counters (RC4 W4 item 1): plumbed from// the per-index `metric_index_count` / `metric_index_total_ms`@@ -21652,6 +21671,27 @@ pub async fn put_settings(// Normalize body into the `{ "index": { ... } }` shape we store.let inner = body.get("index").cloned().unwrap_or(body.clone());+ // An `analysis` block here is accepted-and-ignored, and it is the one hole+ // that would walk straight around the create-time gate: this handler merges+ // the body into `engine.index_settings` (the display copy read back by+ // `GET /{index}/_settings`) and nothing else. The index's+ // `AnalyzerRegistry` is built once, at create/open (index.rs:5428, :5617),+ // and there is no rebuild path — so a settings PUT that "adds an analyzer"+ // changes what the API echoes back and not one token of what is indexed.+ // Refuse it. Elasticsearch also refuses analysis updates on an open index+ // (they are non-dynamic settings), so this narrows behaviour towards ES+ // rather than away from it (issue #204).+ if inner.pointer("/analysis").is_some() || body.pointer("/analysis").is_some() {+ let e = xerj_common::XerjError::config(format!(+ "index [{index}]: `analysis` settings cannot be updated after the index is \+ created — the analyzer registry is built at index creation and this request \+ would change only what `GET /{index}/_settings` echoes back, not how \+ documents are analysed. Create a new index with the analysis block and \+ reindex."+ ));+ return ApiError::new(e).into_response();+ }+for idx in &targets {let mut existing = state.engine@@ -21745,11 +21785,33 @@ pub async fn put_ingest_pipeline("date" => "timestamp_parse".to_string(),"json" => "json_parse".to_string(),"convert" => "convert".to_string(),- "append" | "set" => "set".to_string(),+ // `append` used to be folded into `set` here, which REPLACES the+ // field instead of extending it — a different document, under a+ // 200, and in disagreement with the `_simulate` interpreter below,+ // which has always implemented a real append (issue #204). There is+ // now an `append` stage; the two agree.+ "set" => "set".to_string(),"copy" => "copy_field".to_string(),other => other.to_string(),}}++ /// Elasticsearch `convert` target types that name exactly the same+ /// conversion as one of xerj's, under a different word.+ ///+ /// Issue #204 permits an *equivalent* fallback; `ConvertTypePlugin`'s+ /// `integer` parses to `i64` and its `float` to `f64` (`builtins.rs`), so+ /// ES `long` and `double` are the same conversion and 400ing them rejected+ /// previously-accepted ES pipelines for no gain. `auto`, `ip`, `short` and+ /// `byte` have no equivalent here and stay refused rather than being+ /// silently widened.+ fn map_convert_type(es_type: &str) -> Option<&'static str> {+ match es_type {+ "long" => Some("integer"),+ "double" => Some("float"),+ _ => None,+ }+ }let xerj_config = if let Some(processors) = body.get("processors").and_then(Value::as_array) {let mut stages: Vec<Value> = Vec::with_capacity(processors.len());for (i, proc) in processors.iter().enumerate() {@@ -21798,6 +21860,19 @@ pub async fn put_ingest_pipeline(_ => proc_config.clone(),}}+ "convert" => {+ // ES vocabulary → xerj vocabulary for the two targets that+ // are the same conversion under a different name.+ let mut cfg = proc_config.clone();+ if let Some(mapped) = proc_config+ .get("type")+ .and_then(Value::as_str)+ .and_then(map_convert_type)+ {+ cfg["type"] = json!(mapped);+ }+ cfg+ }_ => proc_config.clone(),};stages.push(json!({"type": xerj_type, "config": adapted_config}));@@ -21844,6 +21919,25 @@ pub async fn put_ingest_pipeline();Json(json!({ "acknowledged": true })).into_response()}+ // Same shape one level down: the processor exists, an option on it does+ // not. ES accepts the definition, so we do too — and record why it+ // cannot run, so no document is written as if the option had been+ // honoured (issue #204 applied to this endpoint's own new contract).+ Err(xerj_wasm::WasmError::UnsupportedOption {+ stage,+ option,+ reason,+ }) => {+ state.engine.register_unrunnable_pipeline(+ &id,+ body,+ format!(+ "processor [{stage}] option [{option}] is not supported by this xerj \+ build ({reason}); no document can be ingested through pipeline [{id}]"+ ),+ );+ Json(json!({ "acknowledged": true })).into_response()+ }Err(e) => {tracing::warn!(pipeline = %id, error = %e, "rejected ingest pipeline definition");let err = xerj_common::XerjError::invalid_mapping(format!(@@ -25042,22 +25136,69 @@ async fn clone_index_to(state: &AppState, source: &str, target: &str) -> Result<.get_index(target).map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;- // Copy all documents.- let search_req = xerj_query::parse_request(&json!({- "query": { "match_all": {} },- "size": 10000,- "from": 0,- }))- .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;+ // Copy EVERY document, and fail loudly if any copy fails.+ //+ // Issue #204, the "a partial export instead of a full one" case, verbatim.+ // This used to be a single `size: 10000` search whose per-document writes+ // were swallowed with `let _ =`, and then answered `{"acknowledged": true,+ // "shards_acknowledged": true}` — so cloning an index with 10,001+ // documents silently produced a 10,000-document copy, and a clone in which+ // every single write failed was reported as a success. `_clone`, `_shrink`+ // and `_split` all land here, on user data.+ //+ // Same keyset (`search_after` on `_id`) paging the `_reindex` handler uses,+ // for the same reason: `from`-offset paging truncates at+ // `max_result_window`. The source is flushed first because the in-memory+ // memtable's sort path does not order by `_id` reliably, so un-flushed+ // writes would be skipped or duplicated across pages.+ let _ = src_idx.flush().await;++ const CLONE_PAGE_SIZE: usize = 1_000;+ let mut search_after: Option<Value> = None;+ let mut copied = 0usize;- let results = src_idx- .search(&search_req)- .await- .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;+ loop {+ let mut search_body = json!({+ "query": { "match_all": {} },+ "size": CLONE_PAGE_SIZE,+ "sort": [{ "_id": "asc" }],+ });+ if let Some(sa) = &search_after {+ search_body["search_after"] = sa.clone();+ }+ let search_req = xerj_query::parse_request(&search_body)+ .map_err(|e| ApiError::new(xerj_common::XerjError::invalid_query(e.to_string())))?;- for hit in results.hits {- if !hit.source.is_null() {- let _ = dest_idx.index_document(Some(hit.id), hit.source).await;+ let results = src_idx+ .search(&search_req)+ .await+ .map_err(|e| ApiError::new(xerj_common::XerjError::from(e)))?;++ let batch = results.hits.len();+ if batch == 0 {+ break;+ }+ search_after = results.hits.last().map(|h| json!([h.id.clone()]));++ for hit in results.hits {+ if hit.source.is_null() {+ continue;+ }+ let id = hit.id.clone();+ dest_idx+ .index_document(Some(hit.id), hit.source)+ .await+ .map_err(|e| {+ ApiError::new(xerj_common::XerjError::internal(format!(+ "clone of [{source}] into [{target}] failed after {copied} \+ document(s): document [{id}] could not be written: {e}"+ )))+ })?;+ copied += 1;+ }++ if batch < CLONE_PAGE_SIZE {+ break;}}Ok(())diff --git a/engine/crates/xerj-api/tests/fail_closed_config_http.rs b/engine/crates/xerj-api/tests/fail_closed_config_http.rsnew file mode 100644index 000000000..5a3914641--- /dev/null+++ b/engine/crates/xerj-api/tests/fail_closed_config_http.rs@@ -0,0 +1,396 @@+//! Issue #204, from the outside: configuration xerj cannot honour must be+//! refused at the door, or refused at the point of use — never accepted with a+//! 200 and then quietly not done.+//!+//! Every case here was found by measurement against the first cut of this+//! branch, and each one is the HTTP-level statement of a defect that had only+//! engine-level (or no) coverage:+//!+//! 1. the create-time analysis gate had no HTTP test at all — neither its+//! status code nor its ES error shape;+//! 2. the gate judged a declared filter by its `type` while `apply_settings`+//! resolves it by NAME, so the canonical Elasticsearch-docs+//! `english_stop` block started 400ing although xerj serves it correctly;+//! 3. `PUT /{index}/_settings` walked straight around the gate — it merged an+//! `analysis` block into the display copy and rebuilt nothing;+//! 4. `PUT /_ingest/pipeline` was redefined by this branch to mean "compiled ⇒+//! xerj honours this", and then accepted an ES `grok` `patterns` array, a+//! processor-level `if` guard, and an `append` it executed as `set`;+//! 5. `_clone` copied at most 10,000 documents and swallowed every write+//! error, then answered `{"acknowledged": true}`.+//!+//! Elasticsearch is referenced for semantics only. It is AGPL-3.0/SSPL-1.0/+//! Elastic-2.0 licensed and no code from it is reproduced here.++use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::{json, Value};+use tower::ServiceExt;++fn app() -> (axum::Router, tempfile::TempDir) {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);+ (xerj_api::router::build_es_compat_router(state), dir)+}++async fn send(app: &axum::Router, req: Request<Body>) -> (StatusCode, Value) {+ let res = app.clone().oneshot(req).await.expect("response");+ let status = res.status();+ let bytes = axum::body::to_bytes(res.into_body(), 4 * 1024 * 1024)+ .await+ .expect("body");+ let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ (status, body)+}++fn put(path: &str, body: Value) -> Request<Body> {+ Request::builder()+ .method("PUT")+ .uri(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request")+}++fn post(path: &str, body: Value) -> Request<Body> {+ Request::builder()+ .method("POST")+ .uri(path)+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request")+}++fn get(path: &str) -> Request<Body> {+ Request::builder()+ .method("GET")+ .uri(path)+ .body(Body::empty())+ .expect("request")+}++// ── The create-time analysis gate ────────────────────────────────────────────++#[tokio::test]+async fn an_analysis_block_we_cannot_honour_is_a_400_in_the_es_error_shape() {+ let (app, _dir) = app();+ let (status, body) = send(+ &app,+ put(+ "/autocomplete",+ json!({+ "settings": { "analysis": { "analyzer": {+ "ac": { "type": "custom", "tokenizer": "edge_ngram_tok" }+ } } }+ }),+ ),+ )+ .await;++ // Pre-fix this was `200 {"acknowledged": true}` and the index tokenised+ // with `standard` — an autocomplete index that matches nothing.+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert_eq!(body["status"], 400, "{body}");+ assert!(body["error"]["type"].is_string(), "{body}");+ assert!(body["error"]["root_cause"].is_array(), "{body}");+ assert!(+ body["error"]["reason"]+ .as_str()+ .expect("reason")+ .contains("edge_ngram_tok"),+ "the reason must name the construct we cannot honour: {body}"+ );++ // And nothing was left behind.+ let (status, _) = send(&app, get("/autocomplete")).await;+ assert_eq!(status, StatusCode::NOT_FOUND);+}++#[tokio::test]+async fn the_canonical_elasticsearch_analysis_block_is_still_accepted() {+ let (app, _dir) = app();+ // Straight from the Elasticsearch docs' `rebuilt_english`, cut to the parts+ // xerj implements. `english_stop` declares `type: stop`, which+ // `apply_settings` cannot BUILD — and then resolves by NAME to the built-in+ // English stopword filter, which is exactly what it declares. Judging it on+ // `type` alone made this a 400: a straight regression on a settings block+ // xerj had always accepted and analysed correctly.+ let (status, body) = send(+ &app,+ put(+ "/blog",+ json!({+ "settings": { "analysis": {+ "filter": { "english_stop": { "type": "stop", "stopwords": "_english_" } },+ "analyzer": { "rebuilt_english": {+ "type": "custom",+ "tokenizer": "standard",+ "filter": ["lowercase", "english_stop"]+ } }+ } },+ "mappings": { "properties": {+ "body": { "type": "text", "analyzer": "rebuilt_english" }+ } }+ }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["acknowledged"], true, "{body}");+}++#[tokio::test]+async fn analysis_cannot_be_smuggled_past_the_gate_through_put_settings() {+ let (app, _dir) = app();+ let (status, _) = send(&app, put("/blog", json!({}))).await;+ assert_eq!(status, StatusCode::OK);++ // The registry is built at create/open and there is no rebuild path, so+ // this could only ever have changed what `GET /_settings` echoes back.+ let (status, body) = send(+ &app,+ put(+ "/blog/_settings",+ json!({ "analysis": { "analyzer": {+ "late": { "type": "custom", "tokenizer": "whitespace" }+ } } }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert!(+ body["error"]["reason"]+ .as_str()+ .expect("reason")+ .contains("analysis"),+ "{body}"+ );++ // …and the display copy was not quietly updated either.+ let (_, settings) = send(&app, get("/blog/_settings")).await;+ assert!(+ settings.pointer("/blog/settings/index/analysis").is_none(),+ "a refused settings PUT must not leave the analysis block behind: {settings}"+ );++ // A settings PUT that xerj *can* honour is unaffected.+ let (status, _) = send(+ &app,+ put(+ "/blog/_settings",+ json!({ "index": { "number_of_replicas": 0 } }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+}++// ── PUT /_ingest/pipeline: compiled ⇒ honoured ───────────────────────────────++/// Register `pipeline`, then write one document through it. Returns the write's+/// status and body.+async fn write_through(+ app: &axum::Router,+ id: &str,+ processors: Value,+ doc: Value,+) -> (StatusCode, Value) {+ let (status, body) = send(+ app,+ put(+ &format!("/_ingest/pipeline/{id}"),+ json!({ "processors": processors }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::OK, "pipeline definition: {body}");+ assert_eq!(body["acknowledged"], true, "{body}");+ send(+ app,+ put(&format!("/docs/_doc/1?pipeline={id}&refresh=true"), doc),+ )+ .await+}++#[tokio::test]+async fn an_es_grok_processor_is_accepted_and_then_refuses_the_write() {+ let (app, _dir) = app();+ // Measured pre-fix: this compiled to the SYSLOG default, answered 200, and+ // indexed `{"message": "10.0.0.1 GET"}` with no `client` field anywhere.+ let (status, body) = write_through(+ &app,+ "grok_pipe",+ json!([{ "grok": { "field": "message", "patterns": ["%{IP:client} %{WORD:method}"] } }]),+ json!({ "message": "10.0.0.1 GET" }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ let reason = body["error"]["reason"].as_str().expect("reason");+ assert!(reason.contains("patterns"), "{body}");++ // The document was NOT indexed untransformed.+ let (status, _) = send(&app, get("/docs/_doc/1")).await;+ assert_ne!(status, StatusCode::OK, "the write must not have happened");+}++#[tokio::test]+async fn a_processor_level_if_guard_is_accepted_and_then_refuses_the_write() {+ let (app, _dir) = app();+ // Measured pre-fix: `{"foo": "something-else"}` was indexed as+ // `{"foo": "something-else", "env": "prod"}` — the guard excluded it.+ let (status, body) = write_through(+ &app,+ "guarded",+ json!([{ "set": { "field": "env", "value": "prod", "if": "ctx.foo == 'never'" } }]),+ json!({ "foo": "something-else" }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "{body}");+ assert!(+ body["error"]["reason"]+ .as_str()+ .expect("reason")+ .contains("if"),+ "{body}"+ );+}++#[tokio::test]+async fn set_overrides_an_existing_value_like_elasticsearch() {+ let (app, _dir) = app();+ let (status, body) = write_through(+ &app,+ "setter",+ json!([{ "set": { "field": "env", "value": "prod" } }]),+ json!({ "env": "dev" }),+ )+ .await;+ assert!(status.is_success(), "{body}");++ let (_, doc) = send(&app, get("/docs/_doc/1")).await;+ // Pre-fix: `override` defaulted to false and this stayed "dev".+ assert_eq!(doc["_source"]["env"], "prod", "{doc}");+}++#[tokio::test]+async fn append_appends_rather_than_replacing() {+ let (app, _dir) = app();+ let (status, body) = write_through(+ &app,+ "appender",+ json!([{ "append": { "field": "tags", "value": "b" } }]),+ json!({ "tags": ["a"] }),+ )+ .await;+ assert!(status.is_success(), "{body}");++ let (_, doc) = send(&app, get("/docs/_doc/1")).await;+ // Pre-fix: `append` was mapped onto `set`, so this was `"b"` — while the+ // `_simulate` interpreter for the same pipeline produced `["a","b"]`.+ assert_eq!(doc["_source"]["tags"], json!(["a", "b"]), "{doc}");+}++#[tokio::test]+async fn convert_accepts_the_elasticsearch_spellings_of_the_same_conversion() {+ let (app, _dir) = app();+ // `long` is xerj's `integer` (i64) and `double` is its `float` (f64) — an+ // equivalent fallback, which issue #204 explicitly permits. Refusing them+ // 400ed previously-working ES pipelines for no gain.+ let (status, body) = write_through(+ &app,+ "converter",+ json!([+ { "convert": { "field": "count", "type": "long" } },+ { "convert": { "field": "ratio", "type": "double" } }+ ]),+ json!({ "count": "404", "ratio": "1.5" }),+ )+ .await;+ assert!(status.is_success(), "{body}");++ let (_, doc) = send(&app, get("/docs/_doc/1")).await;+ assert_eq!(doc["_source"]["count"], 404, "{doc}");+ assert_eq!(doc["_source"]["ratio"], 1.5, "{doc}");++ // `auto` has no equivalent here and stays refused.+ let (status, _) = send(+ &app,+ put(+ "/_ingest/pipeline/auto_conv",+ json!({ "processors": [{ "convert": { "field": "c", "type": "auto" } }] }),+ ),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST);+}++// ── _clone copies everything, or says it did not ─────────────────────────────++/// More documents than one clone page (1,000) and more than the old hardcoded+/// `size: 10000` ceiling, so the test fails on both the truncation and the+/// paging bug rather than only the second.+#[tokio::test]+async fn clone_copies_every_document_not_the_first_ten_thousand() {+ let dir = tempfile::tempdir().expect("tempdir");+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.storage.wal_sync = xerj_common::config::WalSync::Async;+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");+ let state = xerj_api::state::AppState::new(config, engine, metrics);++ const DOCS: usize = 10_001;+ state+ .engine+ .create_index("src", xerj_common::types::Schema::empty())+ .expect("create_index");+ let idx = state.engine.get_index("src").expect("get_index");+ for i in 0..DOCS {+ idx.index_document(Some(format!("{i:08}")), json!({ "n": i }))+ .await+ .expect("index_document");+ }+ idx.refresh().await.expect("refresh");++ let app = xerj_api::router::build_es_compat_router(state);++ // The source really does hold more than the old ceiling, and the single+ // `size: 10000` search the pre-fix clone used really does stop at it — so+ // the assertion below fails on the old code rather than passing vacuously.+ let (_, src_count) = send(&app, get("/src/_count")).await;+ assert_eq!(+ src_count["count"].as_u64(),+ Some(DOCS as u64),+ "{src_count}"+ );+ let (_, capped) = send(+ &app,+ post(+ "/src/_search",+ json!({ "query": { "match_all": {} }, "size": 10000 }),+ ),+ )+ .await;+ assert_eq!(+ capped["hits"]["hits"].as_array().map(Vec::len),+ Some(10_000),+ "the pre-fix clone's one-shot search tops out here"+ );++ let (status, body) = send(&app, post("/src/_clone/dst", json!({}))).await;+ assert_eq!(status, StatusCode::OK, "{body}");+ assert_eq!(body["acknowledged"], true, "{body}");++ let (_, count) = send(&app, get("/dst/_count")).await;+ // Pre-fix: exactly 10000 — a partial copy reported as a completed one.+ assert_eq!(+ count["count"].as_u64(),+ Some(DOCS as u64),+ "clone must copy every document: {count}"+ );+}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex ea5b82820..2e4b7e8db 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1694,9 +1694,11 @@ impl Engine {/// Store a pipeline definition that could not be compiled, together with/// the reason, and make sure nothing stale is left behind.///- /// Issue #204. This exists for exactly one case: a definition xerj accepts- /// for wire compatibility (Elasticsearch does) but cannot execute, because- /// it names a processor this build does not implement. Storing it keeps+ /// Issue #204. This exists for one case: a definition xerj accepts for wire+ /// compatibility (Elasticsearch does) but cannot execute — because it names+ /// a processor this build does not implement, or sets an option on one that+ /// this build cannot honour (a processor-level `if` guard, an `on_failure`+ /// chain, a `grok` `patterns` array). Storing it keeps/// `GET`/`_simulate` truthful about what was submitted; recording the/// reason is what stops [`Self::process_through_pipeline`] from silently/// passing documents through untransformed. A previously-compiled pipelinediff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 71cd398a0..93689208c 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -17775,6 +17775,10 @@ impl Index {/// Number of times [`Self::persist_evolved_schema`] could not write/// `schema.json`. Non-zero means the on-disk mapping is behind the/// in-memory one and a restart will lose dynamically-mapped fields.+ ///+ /// Read by `GET /{index}/_stats` and `GET /_all/_stats`, which report it as+ /// `primaries.mappings.schema_persist_failures` — a health check can see it+ /// without reading logs.pub fn schema_persist_failures(&self) -> u64 {self.schema_persist_failures.load(Ordering::Relaxed)}diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rsindex 031250d70..490c749df 100644--- a/engine/crates/xerj-fts/src/analyzer.rs+++ b/engine/crates/xerj-fts/src/analyzer.rs@@ -1312,19 +1312,43 @@ impl AnalyzerRegistry {/// existing index. That leniency is right at open time and wrong at create/// time, where every one of these constructs used to be accepted with a/// `200 {"acknowledged": true}` and then quietly replaced by something- /// weaker:+ /// weaker.///- /// - an unresolvable `tokenizer` became `standard` — an `edge_ngram`- /// autocomplete index that matches nothing;- /// - an unresolvable `filter` was dropped — a missing `lowercase` turns- /// every match case-sensitive;- /// - an unsupported filter/tokenizer `type` was skipped entirely;- /// - an unknown non-custom analyzer `type` left the analyzer unregistered,- /// so fields naming it silently got `standard`.+ /// # What is checked+ ///+ /// 1. a declared `filter` / `tokenizer` whose `type` this build cannot+ /// construct **and** whose *name* does not resolve to an equivalent+ /// built-in (see [`Self::builtin_filter_honours`] — the by-name fallback+ /// `apply_settings` actually takes);+ /// 2. a `pattern` tokenizer whose regex does not compile;+ /// 3. an analyzer naming a `tokenizer` / `filter` that is neither declared+ /// nor built in — the unresolvable tokenizer used to become `standard`+ /// (an `edge_ngram` autocomplete index that matches nothing) and the+ /// unresolvable filter used to be dropped (a missing `lowercase` turns+ /// every match case-sensitive);+ /// 4. an unknown non-custom analyzer `type`, which left the analyzer+ /// unregistered so fields naming it silently got `standard`;+ /// 5. `char_filter` — declared at either level and never built by+ /// `apply_settings`, so the stripping/mapping never happens;+ /// 6. `normalizer` — accepted by ES for `keyword` fields, never built here;+ /// 7. an analyzer whose `filter` is not an array or whose `tokenizer` is+ /// not a string: `apply_settings` drops the value shape-first and falls+ /// back to "no filters" / `standard`.+ ///+ /// # What is NOT checked+ ///+ /// Option-level divergence *inside* a type this build does construct — e.g.+ /// `synonym.synonyms_path` (file-backed rule lists are not read),+ /// `length.min` (defaults to 2 here, 0 in ES), `asciifolding+ /// .preserve_original`. Those are pre-existing gaps, tracked separately;+ /// this function does not claim to cover them, and the by-name fallback in+ /// (1) is deliberately conservative about them: a name that resolves to a+ /// built-in is only accepted when the declared options ask for exactly what+ /// that built-in does.////// Returns one human-readable message per problem, empty when the block can- /// be honoured exactly as written. `Index::create_with_settings` turns a- /// non-empty result into a 400 so the caller learns at the door.+ /// be honoured as written. `Index::create_with_settings` turns a non-empty+ /// result into a 400 so the caller learns at the door.////// `settings` may be either the full envelope (`{"settings": {"analysis":/// …}}`) or the inner settings object — the same two shapes@@ -1348,13 +1372,39 @@ impl AnalyzerRegistry {.get("type").and_then(|v| v.as_str()).unwrap_or("");- if !SUPPORTED_FILTER_TYPES.contains(&filter_type) {+ if SUPPORTED_FILTER_TYPES.contains(&filter_type) {+ continue;+ }+ // The `type` is one `apply_settings` cannot BUILD — but that is+ // not the whole story, and reading only the type is what made+ // this check reject settings blocks xerj serves correctly.+ // `apply_settings` looks the name up in `custom_filters` first+ // and falls through to `resolve_builtin_filter(name)`+ // (analyzer.rs:1284-1286), so `{"english_stop": {"type":+ // "stop", "stopwords": "_english_"}}` — the canonical+ // Elasticsearch-docs `rebuilt_english` block — really does+ // strip English stopwords. Complaining about its type 400s a+ // `PUT /{index}` that used to work AND analysed as asked.+ if Self::builtin_filter_honours(filter_name, filter_def) {+ continue;+ }+ if probe.resolve_builtin_filter(filter_name).is_some() {+ // The name resolves, but to something that is NOT what the+ // declaration asks for (a custom `stopwords` list, another+ // `language`). Saying "unsupported type" here would be+ // misleading — the type is fine, the options are not.problems.push(format!(- "token filter [{filter_name}]: unsupported type [{filter_type}] \- (supported: {})",- SUPPORTED_FILTER_TYPES.join(", ")+ "token filter [{filter_name}]: [{filter_type}] with these options is \+ not supported — the built-in [{filter_name}] filter would be used \+ instead, which is not what this declares"));+ continue;}+ problems.push(format!(+ "token filter [{filter_name}]: unsupported type [{filter_type}] \+ (supported: {})",+ SUPPORTED_FILTER_TYPES.join(", ")+ ));}}@@ -1364,6 +1414,20 @@ impl AnalyzerRegistry {declared_tokenizers.insert(tok_name.as_str());let tok_type = tok_def.get("type").and_then(|v| v.as_str()).unwrap_or("");if !SUPPORTED_TOKENIZER_TYPES.contains(&tok_type) {+ // Same by-name fallback as filters: `apply_settings` reaches+ // `resolve_builtin_tokenizer(name)` (analyzer.rs:1454-1473)+ // when the declared type built nothing.+ if Self::builtin_tokenizer_honours(tok_name, tok_def) {+ continue;+ }+ if probe.resolve_builtin_tokenizer(tok_name).is_some() {+ problems.push(format!(+ "tokenizer [{tok_name}]: [{tok_type}] with these options is not \+ supported — the built-in [{tok_name}] tokenizer would be used \+ instead, which is not what this declares"+ ));+ continue;+ }problems.push(format!("tokenizer [{tok_name}]: unsupported type [{tok_type}] \(supported: {})",@@ -1385,6 +1449,34 @@ impl AnalyzerRegistry {}}+ // `char_filter` never reaches the built pipeline: every analyzer is+ // registered as `AnalyzerPipeline::new(vec![], tokenizer, filters)` —+ // the char-filter slot is hard-coded empty. A declared `html_strip` or+ // `mapping` therefore strips/maps nothing, with no signal anywhere.+ if analysis+ .pointer("/char_filter")+ .and_then(|v| v.as_object())+ .is_some_and(|m| !m.is_empty())+ {+ problems.push(+ "char filters are not supported (analysis.char_filter is declared but no \+ char filter is ever applied)"+ .to_string(),+ );+ }+ // `normalizer` is the keyword-field analogue and is equally unbuilt.+ if analysis+ .pointer("/normalizer")+ .and_then(|v| v.as_object())+ .is_some_and(|m| !m.is_empty())+ {+ problems.push(+ "normalizers are not supported (analysis.normalizer is declared but never \+ built, so keyword fields naming one are indexed unchanged)"+ .to_string(),+ );+ }+if let Some(analyzer_map) = analysis.pointer("/analyzer").and_then(|v| v.as_object()) {for (analyzer_name, analyzer_def) in analyzer_map {let analyzer_type = analyzer_def@@ -1401,6 +1493,49 @@ impl AnalyzerRegistry {continue;}+ // Per-analyzer char filters: same hard-coded-empty slot.+ if analyzer_def+ .get("char_filter")+ .is_some_and(|v| !matches!(v, serde_json::Value::Array(a) if a.is_empty()))+ {+ problems.push(format!(+ "analyzer [{analyzer_name}]: `char_filter` is not supported — the \+ declared char filters are never applied"+ ));+ }++ // Shape errors are silent losses, not type errors:+ // `apply_settings` reads `tokenizer` with `as_str()` and+ // `filter` with `as_array()`, so a wrong-shaped value is+ // dropped and the analyzer is built with `standard` / no+ // filters — accepted, and not what was written.+ match analyzer_def.get("tokenizer") {+ None | Some(serde_json::Value::String(_)) => {}+ Some(other) => problems.push(format!(+ "analyzer [{analyzer_name}]: `tokenizer` must be a tokenizer name \+ (string), got {other} — it would be ignored and `standard` used"+ )),+ }+ let mut filter_names: Vec<&str> = Vec::new();+ match analyzer_def.get("filter") {+ None => {}+ Some(serde_json::Value::Array(arr)) => {+ for v in arr {+ match v.as_str() {+ Some(s) => filter_names.push(s),+ None => problems.push(format!(+ "analyzer [{analyzer_name}]: `filter` entries must be \+ filter names (strings), got {v} — it would be ignored"+ )),+ }+ }+ }+ Some(other) => problems.push(format!(+ "analyzer [{analyzer_name}]: `filter` must be an array of filter \+ names, got {other} — it would be ignored and no filters applied"+ )),+ }+let tokenizer_name = analyzer_def.get("tokenizer").and_then(|v| v.as_str())@@ -1413,11 +1548,6 @@ impl AnalyzerRegistry {));}- let filter_names: Vec<&str> = analyzer_def- .get("filter")- .and_then(|v| v.as_array())- .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())- .unwrap_or_default();for fname in filter_names {if !declared_filters.contains(fname)&& probe.resolve_builtin_filter(fname).is_none()@@ -1431,9 +1561,72 @@ impl AnalyzerRegistry {}problems.sort();+ problems.dedup();problems}+ /// Does a declared token filter resolve, by NAME, to a built-in that does+ /// exactly what the declaration asks for?+ ///+ /// This is the fallback `apply_settings` genuinely takes: a `type` it+ /// cannot build leaves `custom_filters` without an entry, and the analyzer+ /// loop then calls [`Self::resolve_builtin_filter`] with the *declared+ /// name*. Judging such a declaration on its `type` alone reports a filter+ /// that is in fact honoured — and, through the create-time gate, 400s a+ /// `PUT /{index}` that previously worked.+ ///+ /// Deliberately narrow: only the exact option sets the built-in reproduces+ /// return `true`. A custom `stopwords` list under the name `english_stop`+ /// is still a problem, because the English list is what would actually run.+ fn builtin_filter_honours(name: &str, def: &serde_json::Value) -> bool {+ let ty = def.get("type").and_then(|v| v.as_str()).unwrap_or("");+ match (name, ty) {+ // `LowercaseFilter`. ES's `lowercase` filter takes an optional+ // `language` (greek/irish/turkish); only the default is this.+ ("lowercase", "lowercase") => def.get("language").is_none(),+ // `StopwordsFilter::english()`. ES's `stop` defaults to+ // `_english_`, which is exactly the list built here.+ ("stop" | "english_stop", "stop") => {+ def.get("stopwords_path").is_none()+ && match def.get("stopwords") {+ None => true,+ Some(serde_json::Value::String(s)) => s == "_english_",+ _ => false,+ }+ && def.get("ignore_case").is_none()+ && def.get("remove_trailing").is_none()+ }+ // `StemmerFilter::english()` — rust-stemmers' Snowball English.+ ("stemmer" | "english_stemmer", "stemmer") => {+ match def.get("language").or_else(|| def.get("name")) {+ None => true,+ Some(serde_json::Value::String(s)) => {+ matches!(s.as_str(), "english" | "porter2")+ }+ _ => false,+ }+ }+ _ => false,+ }+ }++ /// Tokenizer counterpart of [`Self::builtin_filter_honours`].+ ///+ /// `apply_settings` resolves an unbuilt tokenizer by declared name through+ /// [`Self::resolve_builtin_tokenizer`], so `{"standard": {"type":+ /// "standard"}}` is honoured exactly as written.+ fn builtin_tokenizer_honours(name: &str, def: &serde_json::Value) -> bool {+ let ty = def.get("type").and_then(|v| v.as_str()).unwrap_or("");+ match (name, ty) {+ // `max_token_length` is not implemented by `StandardTokenizer`.+ ("standard", "standard") => def.get("max_token_length").is_none(),+ ("whitespace", "whitespace") => def.get("max_token_length").is_none(),+ // `buffer_size` is a performance knob with no semantic effect.+ ("keyword", "keyword") => true,+ _ => false,+ }+ }+/// Locate the `analysis` block inside an index-settings object.////// ES accepts both the shorthand (`settings.analysis.*`) and the canonical@@ -2373,4 +2566,150 @@ mod unsupported_analysis_tests {);}}++ /// The canonical Elasticsearch-docs `rebuilt_english` shape, cut down to the+ /// parts xerj can serve. Judging `english_stop` on its `type` alone made+ /// `unsupported_analysis` report it as unsupported — and, through the+ /// create-time gate, 400 a `PUT /{index}` that xerj had always accepted AND+ /// analysed correctly. Both halves are asserted here: no complaint, and the+ /// registry built from the same block really does strip English stopwords.+ #[test]+ fn declared_filter_resolved_by_name_is_honoured_not_reported() {+ let settings = json!({+ "analysis": {+ "filter": {+ "english_stop": { "type": "stop", "stopwords": "_english_" }+ },+ "analyzer": {+ "rebuilt_english": {+ "type": "custom",+ "tokenizer": "standard",+ "filter": ["lowercase", "english_stop"]+ }+ }+ }+ });+ assert_eq!(+ AnalyzerRegistry::unsupported_analysis(&settings),+ Vec::<String>::new()+ );++ let mut registry = AnalyzerRegistry::with_defaults();+ registry.apply_settings(&settings);+ let terms = registry+ .get_analyzer("rebuilt_english")+ .expect("analyzer must be registered")+ .analyze_to_terms("The Quick Brown Fox");+ assert_eq!(+ terms,+ vec!["quick".to_string(), "brown".to_string(), "fox".to_string()],+ "the built registry must apply the stopword filter the block declares"+ );+ }++ /// …but only when the built-in is what the declaration actually asks for.+ /// A custom stopword list under the same name is NOT honoured — the English+ /// list would run instead — so it must still be reported, and with a message+ /// about the options rather than a misleading "unsupported type".+ #[test]+ fn declared_filter_with_options_the_builtin_cannot_reproduce_is_reported() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "filter": {+ "english_stop": { "type": "stop", "stopwords": ["pelican", "walrus"] }+ }+ }+ }));+ assert_eq!(problems.len(), 1, "{problems:?}");+ assert!(+ problems[0].contains("not supported") && problems[0].contains("english_stop"),+ "{problems:?}"+ );+ }++ #[test]+ fn builtin_tokenizer_resolved_by_name_is_honoured_not_reported() {+ assert_eq!(+ AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "tokenizer": { "whitespace": { "type": "whitespace" } },+ "analyzer": { "a": { "type": "custom", "tokenizer": "whitespace" } }+ }+ })),+ Vec::<String>::new()+ );+ }++ /// `AnalyzerPipeline::new(vec![], …)` — the char-filter slot is hard-coded+ /// empty, so a declared `html_strip` strips nothing. Measured pre-fix:+ /// `<b>hello</b>` tokenised to `["b", "hello", "b"]` with `problems == []`.+ #[test]+ fn char_filters_are_reported_because_they_are_never_built() {+ let settings = json!({+ "analysis": {+ "char_filter": { "strip_html": { "type": "html_strip" } },+ "analyzer": {+ "a": {+ "type": "custom",+ "tokenizer": "standard",+ "char_filter": ["strip_html"]+ }+ }+ }+ });+ let problems = AnalyzerRegistry::unsupported_analysis(&settings);+ assert_eq!(problems.len(), 2, "{problems:?}");+ assert!(+ problems.iter().all(|p| p.contains("char filter")),+ "{problems:?}"+ );++ // The reason it must be reported: nothing strips the tags.+ let mut registry = AnalyzerRegistry::with_defaults();+ registry.apply_settings(&settings);+ assert!(+ registry+ .get_analyzer("a")+ .expect("registered")+ .analyze_to_terms("<b>hello</b>")+ .contains(&"b".to_string()),+ "char filters really are not applied — that is why they are refused"+ );+ }++ #[test]+ fn normalizers_are_reported_because_they_are_never_built() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "normalizer": { "lower": { "type": "custom", "filter": ["lowercase"] } }+ }+ }));+ assert_eq!(problems.len(), 1, "{problems:?}");+ assert!(problems[0].contains("normalizer"), "{problems:?}");+ }++ /// `apply_settings` reads `filter` with `as_array()`, so a bare string is+ /// dropped and the analyzer is built with no filters at all. Measured+ /// pre-fix: `problems == []`, terms `["ABC"]` from `"ABC"` — the+ /// `lowercase` the caller wrote never ran.+ #[test]+ fn wrong_shaped_analyzer_keys_are_reported() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "analyzer": {+ "a": { "type": "custom", "tokenizer": "standard", "filter": "lowercase" },+ "b": { "type": "custom", "tokenizer": ["standard"] }+ }+ }+ }));+ assert_eq!(problems.len(), 2, "{problems:?}");+ assert!(+ problems.iter().any(|p| p.contains("`filter` must be")),+ "{problems:?}"+ );+ assert!(+ problems.iter().any(|p| p.contains("`tokenizer` must be")),+ "{problems:?}"+ );+ }}diff --git a/engine/crates/xerj-wasm/src/builtins.rs b/engine/crates/xerj-wasm/src/builtins.rsindex 5c40fdfc2..69e32d3ef 100644--- a/engine/crates/xerj-wasm/src/builtins.rs+++ b/engine/crates/xerj-wasm/src/builtins.rs@@ -873,6 +873,68 @@ impl TransformPlugin for SetPlugin {}}+// ─────────────────────────────────────────────────────────────────────────────+// AppendPlugin+// ─────────────────────────────────────────────────────────────────────────────++/// Append a value to a field, creating or widening a list as needed.+///+/// Elasticsearch's `append` semantics, which is what this exists for: the field+/// becomes a list if it is not one already, a list `value` extends rather than+/// nests, and a missing field is created as a single-element list+/// (`IngestDocument.appendFieldValue`, read for semantics only).+///+/// Issue #204: `append` used to be *mapped onto `set`* in the ES→xerj processor+/// translation (`es_compat.rs`), so `{"append": {"field": "tags", "value":+/// "b"}}` REPLACED `["a"]` with `"b"` instead of producing `["a", "b"]` —+/// under a `200 {"acknowledged": true}`, and disagreeing with the `_simulate`+/// interpreter, which has always implemented a real append. Two behaviours for+/// one pipeline; this is the one that runs at ingest.+pub struct AppendPlugin {+ field: String,+ value: Value,+}++impl AppendPlugin {+ pub fn new(field: impl Into<String>, value: Value) -> Self {+ Self {+ field: field.into(),+ value,+ }+ }+}++impl TransformPlugin for AppendPlugin {+ fn name(&self) -> &str {+ "append"+ }++ fn process(&self, doc: &mut Value) -> ProcessAction {+ let Some(obj) = doc.as_object_mut() else {+ return ProcessAction::Pass;+ };+ let slot = obj+ .entry(self.field.clone())+ .or_insert_with(|| Value::Array(Vec::new()));+ match slot {+ Value::Array(existing) => match self.value.clone() {+ Value::Array(more) => existing.extend(more),+ one => existing.push(one),+ },+ scalar => {+ let previous = scalar.clone();+ let mut out = vec![previous];+ match self.value.clone() {+ Value::Array(more) => out.extend(more),+ one => out.push(one),+ }+ *scalar = Value::Array(out);+ }+ }+ ProcessAction::Pass+ }+}+// ─────────────────────────────────────────────────────────────────────────────// RemoveNullPlugin// ─────────────────────────────────────────────────────────────────────────────diff --git a/engine/crates/xerj-wasm/src/lib.rs b/engine/crates/xerj-wasm/src/lib.rsindex 9af56875d..4fa0a8de9 100644--- a/engine/crates/xerj-wasm/src/lib.rs+++ b/engine/crates/xerj-wasm/src/lib.rs@@ -62,6 +62,24 @@ pub enum WasmError {#[error("unsupported stage type '{stage}'")]UnsupportedStage { stage: String },+ /// The stage type IS implemented, but the caller set an option this build+ /// cannot honour — an Elasticsearch processor-level `if` guard, an+ /// `on_failure` recovery chain, a `grok` `patterns` array.+ ///+ /// Issue #204 again, one level down. Reporting these as `InvalidConfig`+ /// would tell a caller their valid ES processor is malformed; ignoring them+ /// is what the sweep exists to kill — a `set` that runs on documents its+ /// `if` excludes is a silently wrong document, not a cosmetic gap. Handled+ /// exactly like [`WasmError::UnsupportedStage`] at the API boundary: the+ /// definition is accepted (ES accepts it) and recorded as unrunnable, so+ /// every ingest through it refuses loudly.+ #[error("stage '{stage}': option '{option}' is not supported by this build ({reason})")]+ UnsupportedOption {+ stage: String,+ option: String,+ reason: String,+ },+/// The pipeline is registered but carries a stage this build cannot run,/// so no document may pass through it.#[error("pipeline '{pipeline}' is registered but cannot run: {reason}")]diff --git a/engine/crates/xerj-wasm/src/pipeline.rs b/engine/crates/xerj-wasm/src/pipeline.rsindex 303b9d0b3..de8c583f0 100644--- a/engine/crates/xerj-wasm/src/pipeline.rs+++ b/engine/crates/xerj-wasm/src/pipeline.rs@@ -11,10 +11,10 @@ use tracing::{debug, warn};use crate::{builtins::{- AddFieldPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin, FieldRenamePlugin,- GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin, RemoveNullPlugin,- RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin, UppercasePlugin,- UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,+ AddFieldPlugin, AppendPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin,+ FieldRenamePlugin, GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin,+ RemoveNullPlugin, RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin,+ UppercasePlugin, UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,},Result, TransformPlugin, WasmError,};@@ -99,15 +99,17 @@ pub struct Pipeline {impl Pipeline {/// Build a [`Pipeline`] from a [`PipelineConfig`].///- /// Returns [`WasmError::InvalidConfig`] when a plugin-specific config is- /// malformed, and [`WasmError::UnsupportedStage`] when the stage type- /// itself is one xerj does not implement.+ /// Three outcomes, deliberately distinct (issue #204):///- /// Those two are deliberately distinct (issue #204). A malformed config is- /// a caller error and the definition must be refused. An unimplemented- /// stage type is a *xerj capability gap*: the caller wrote something- /// legitimate that we cannot run, and the honest answer is to say so at the- /// point of use rather than pretend either way.+ /// - [`WasmError::InvalidConfig`] — the caller's mistake. The definition+ /// must be refused.+ /// - [`WasmError::UnsupportedStage`] — a *xerj capability gap*: the caller+ /// wrote something legitimate that we cannot run. The honest answer is to+ /// say so at the point of use rather than pretend either way.+ /// - [`WasmError::UnsupportedOption`] — the same gap one level down: the+ /// stage exists, an option on it does not (a processor-level `if` guard,+ /// a `grok` `patterns` array). Silently dropping these is what makes a+ /// compiled pipeline write the wrong document under a `201`.pub fn from_config(name: impl Into<String>, config: &PipelineConfig) -> Result<Self> {let name = name.into();let mut stages: Vec<Arc<dyn TransformPlugin>> = Vec::new();@@ -116,6 +118,13 @@ impl Pipeline {let plugin =build_plugin(&stage_cfg.stage_type, &stage_cfg.config).map_err(|e| match e {StageBuildError::Unsupported(stage) => WasmError::UnsupportedStage { stage },+ StageBuildError::UnsupportedOption { option, reason } => {+ WasmError::UnsupportedOption {+ stage: stage_cfg.stage_type.clone(),+ option,+ reason,+ }+ }StageBuildError::Config(reason) => WasmError::InvalidConfig {plugin: stage_cfg.stage_type.clone(),reason,@@ -191,14 +200,17 @@ impl std::fmt::Debug for Pipeline {/// Why [`build_plugin`] refused a stage.///-/// Split in two on purpose (issue #204): `Config` is the caller's mistake and-/// must be refused at definition time; `Unsupported` is xerj's gap and has to-/// be reported as such so callers are not told their processor is invalid when-/// it is merely unimplemented here.+/// Split three ways on purpose (issue #204): `Config` is the caller's mistake+/// and must be refused at definition time; `Unsupported` and+/// `UnsupportedOption` are xerj's gaps and have to be reported as such, so+/// callers are not told their processor is invalid when it is merely+/// unimplemented here.#[derive(Debug, Clone, PartialEq, Eq)]pub(crate) enum StageBuildError {/// The stage type is not implemented by this build.Unsupported(String),+ /// The stage type IS implemented but an option on it is not.+ UnsupportedOption { option: String, reason: String },/// The stage type exists but its config cannot be honoured.Config(String),}@@ -209,11 +221,47 @@ impl From<String> for StageBuildError {}}+/// Elasticsearch processor-level keys that decide **whether** a processor runs+/// or **what happens when it fails**, none of which the compiled pipeline can+/// act on.+///+/// Issue #204, applied to this PR's own new surface. `PUT /_ingest/pipeline`+/// now means "compiled ⇒ xerj honours this"; a `{"set": {…, "if": "ctx.foo ==+/// 'never'"}}` that compiles and then sets the field on every document makes+/// that promise false, and writes the wrong document under a `201`. The+/// pipeline is accepted (ES accepts it) and recorded as unrunnable, so the gap+/// surfaces at ingest instead of being silently resolved in the caller's face.+///+/// `tag` and `description` are deliberately NOT here: they are identification+/// metadata that changes no document and no decision on this path.+const UNHONOURED_PROCESSOR_KEYS: &[(&str, &str)] = &[+ (+ "if",+ "the guard is not evaluated, so the processor would run on documents it excludes",+ ),+ (+ "on_failure",+ "the recovery processors are not run; the pipeline's `on_error` policy applies instead",+ ),+ (+ "ignore_failure",+ "a failing processor is not ignored; the pipeline's `on_error` policy applies instead",+ ),+];+/// Instantiate a built-in plugin by name.fn build_plugin(stage_type: &str,config: &Value,) -> std::result::Result<Arc<dyn TransformPlugin>, StageBuildError> {+ for (key, reason) in UNHONOURED_PROCESSOR_KEYS {+ if config.get(key).is_some() {+ return Err(StageBuildError::UnsupportedOption {+ option: (*key).to_string(),+ reason: (*reason).to_string(),+ });+ }+ }match stage_type {"json_parse" => {let field = str_field(config, "field")?;@@ -280,6 +328,27 @@ fn build_plugin("grok" => {let field = str_field(config, "field")?;+ // Elasticsearch's `grok` processor is driven by `patterns` — an+ // ARRAY of grok expressions (`["%{IP:client} %{WORD:method}"]`),+ // optionally with `pattern_definitions`. xerj's `grok` is a+ // different, narrower thing: one NAMED pattern out of+ // `GROK_PATTERN_NAMES`. Reading `pattern` and defaulting to+ // `SYSLOG` meant an ES grok processor compiled clean, answered+ // `200 acknowledged` under this PR's "compiled ⇒ honoured"+ // contract, and left the document untouched (issue #204 — measured:+ // `{"message": "10.0.0.1 GET"}` came back with no `client` field).+ for key in ["patterns", "pattern_definitions"] {+ if config.get(key).is_some() {+ return Err(StageBuildError::UnsupportedOption {+ option: key.to_string(),+ reason: format!(+ "xerj's grok stage takes a named `pattern` from [{}], not \+ arbitrary grok expressions",+ GROK_PATTERN_NAMES.join(", ")+ ),+ });+ }+ }let pattern_name = match config.get("pattern") {None => "SYSLOG".to_string(),Some(Value::String(s)) => {@@ -371,13 +440,47 @@ fn build_plugin(.get("value").cloned().ok_or_else(|| "missing 'value'".to_string())?;- let override_existing = config- .get("override")- .and_then(Value::as_bool)- .unwrap_or(false);+ // Elasticsearch's `set` processor defaults `override` to TRUE+ // (`SetProcessor.java:135` — `readBooleanProperty(…, "override",+ // true)`; ES is read for semantics only, no code reproduced). This+ // was `unwrap_or(false)`, so `{"set": {"field": "env", "value":+ // "prod"}}` left an existing `{"env": "dev"}` untouched: a weaker+ // fallback on user configuration, in the file the #204 sweep+ // rewrote. A present-but-non-boolean `override` is a caller error+ // rather than a silent fall-through to the default.+ let override_existing = match config.get("override") {+ None => true,+ Some(v) => v+ .as_bool()+ .ok_or_else(|| format!("'override' must be a boolean, got {v}"))?,+ };Ok(Arc::new(SetPlugin::new(field, value, override_existing)))}+ "append" => {+ let field = str_field(config, "field")?;+ let value = config+ .get("value")+ .cloned()+ .ok_or_else(|| "missing 'value'".to_string())?;+ // ES's `allow_duplicates` defaults to true (`AppendProcessor+ // .java:121`) and that is what `AppendPlugin` does; the+ // de-duplicating variant is not implemented, so asking for it must+ // not be silently ignored.+ match config.get("allow_duplicates") {+ None | Some(Value::Bool(true)) => {}+ Some(_) => {+ return Err(StageBuildError::UnsupportedOption {+ option: "allow_duplicates".to_string(),+ reason: "values are always appended; de-duplicating append is not \+ implemented"+ .to_string(),+ })+ }+ }+ Ok(Arc::new(AppendPlugin::new(field, value)))+ }+"remove_null" => Ok(Arc::new(RemoveNullPlugin)),"url_decode" => {@@ -667,4 +770,150 @@ mod fallback_regression_tests {// Absent `formats` is still the documented epoch/RFC-3339 default.assert!(build("timestamp_parse", serde_json::json!({ "field": "ts" })).is_ok());}++ /// A processor-level `if` guard is not evaluated on this path. Compiling it+ /// meant `PUT /_ingest/pipeline` answered 200 — which this change redefined+ /// to mean "xerj honours this" — and then ran the processor on exactly the+ /// documents the guard excludes. Measured pre-fix: `{"foo":+ /// "something-else"}` came out `{"foo": "something-else", "env": "prod"}`.+ #[test]+ fn a_processor_level_if_guard_is_not_silently_dropped() {+ let err = build(+ "set",+ serde_json::json!({+ "field": "env", "value": "prod", "if": "ctx.foo == 'never'"+ }),+ )+ .expect_err("an unhonourable `if` must not compile");+ assert!(+ matches!(&err, WasmError::UnsupportedOption { stage, option, .. }+ if stage == "set" && option == "if"),+ "expected UnsupportedOption(if), got: {err:?}"+ );+ }++ #[test]+ fn on_failure_and_ignore_failure_are_not_silently_dropped() {+ for key in ["on_failure", "ignore_failure"] {+ let mut cfg = serde_json::json!({ "field": "env", "value": "prod" });+ cfg[key] = serde_json::json!(if key == "ignore_failure" {+ serde_json::json!(true)+ } else {+ serde_json::json!([{ "set": { "field": "x", "value": 1 } }])+ });+ let err = build("set", cfg).expect_err("{key} must not be dropped");+ assert!(+ matches!(&err, WasmError::UnsupportedOption { option, .. } if option == key),+ "expected UnsupportedOption({key}), got: {err:?}"+ );+ }+ }++ /// `tag` and `description` are identification metadata — they change no+ /// document and no decision on this path — so they stay accepted.+ #[test]+ fn processor_metadata_keys_still_compile() {+ assert!(build(+ "set",+ serde_json::json!({+ "field": "env", "value": "prod",+ "tag": "t1", "description": "sets env"+ })+ )+ .is_ok());+ }++ /// ES's `grok` is driven by `patterns` (an array of grok expressions);+ /// xerj's takes one NAMED pattern. Reading `pattern` and defaulting to+ /// `SYSLOG` meant an ES grok processor compiled clean and left the document+ /// untouched — measured pre-fix: `{"message": "10.0.0.1 GET"}` with the+ /// `%{IP:client}` capture nowhere.+ #[test]+ fn es_grok_patterns_array_is_not_silently_replaced_by_syslog() {+ let err = build(+ "grok",+ serde_json::json!({+ "field": "message", "patterns": ["%{IP:client} %{WORD:method}"]+ }),+ )+ .expect_err("`patterns` must not compile to the SYSLOG default");+ assert!(+ matches!(&err, WasmError::UnsupportedOption { stage, option, .. }+ if stage == "grok" && option == "patterns"),+ "expected UnsupportedOption(patterns), got: {err:?}"+ );+ }++ /// Elasticsearch's `set` defaults `override` to true (SetProcessor.java:135+ /// — read for semantics only). `unwrap_or(false)` made `{"set": {"field":+ /// "env", "value": "prod"}}` a no-op against an existing `{"env": "dev"}`.+ #[test]+ fn set_overrides_by_default_like_elasticsearch() {+ let pl = build(+ "set",+ serde_json::json!({ "field": "env", "value": "prod" }),+ )+ .expect("set builds");+ let mut doc = serde_json::json!({ "env": "dev" });+ pl.process(&mut doc);+ assert_eq!(doc["env"], "prod");++ // …and the opt-out still works.+ let pl = build(+ "set",+ serde_json::json!({ "field": "env", "value": "prod", "override": false }),+ )+ .expect("set builds");+ let mut doc = serde_json::json!({ "env": "dev" });+ pl.process(&mut doc);+ assert_eq!(doc["env"], "dev");++ // A non-boolean `override` is refused rather than defaulted.+ assert!(build(+ "set",+ serde_json::json!({ "field": "env", "value": "prod", "override": "yes" })+ )+ .is_err());+ }++ /// `append` used to be mapped onto `set` in the ES translation, so it+ /// REPLACED the field. It is now a real append and agrees with the+ /// `_simulate` interpreter.+ #[test]+ fn append_extends_rather_than_replacing() {+ let pl = build(+ "append",+ serde_json::json!({ "field": "tags", "value": "b" }),+ )+ .expect("append builds");++ let mut doc = serde_json::json!({ "tags": ["a"] });+ pl.process(&mut doc);+ assert_eq!(doc["tags"], serde_json::json!(["a", "b"]));++ // A scalar widens into a list; a missing field becomes a 1-element one.+ let mut doc = serde_json::json!({ "tags": "a" });+ pl.process(&mut doc);+ assert_eq!(doc["tags"], serde_json::json!(["a", "b"]));+ let mut doc = serde_json::json!({});+ pl.process(&mut doc);+ assert_eq!(doc["tags"], serde_json::json!(["b"]));++ // A list value extends rather than nesting.+ let pl = build(+ "append",+ serde_json::json!({ "field": "tags", "value": ["b", "c"] }),+ )+ .expect("append builds");+ let mut doc = serde_json::json!({ "tags": ["a"] });+ pl.process(&mut doc);+ assert_eq!(doc["tags"], serde_json::json!(["a", "b", "c"]));++ // De-duplicating append is not implemented; asking for it is refused.+ assert!(build(+ "append",+ serde_json::json!({ "field": "tags", "value": "b", "allow_duplicates": false })+ )+ .is_err());+ }}