Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine,api): fail closed on user configuration xerj cannot honour

xerj-org2b50d533e900
diff --git a/engine/crates/xerj-api/src/authz.rs b/engine/crates/xerj-api/src/authz.rs
index b6a3df603..bc5358067 100644
--- a/engine/crates/xerj-api/src/authz.rs
+++ b/engine/crates/xerj-api/src/authz.rs
@@ -1477,7 +1477,23 @@ async fn prune_response(
match serde_json::from_slice::<Value>(&bytes) {
Ok(mut v) => {
prune_json(&mut v, sites, principal, &known);
- serde_json::to_vec(&v).unwrap_or_else(|_| bytes.to_vec())
+ match serde_json::to_vec(&v) {
+ Ok(v) => v,
+ // Issue #204: this was
+ // `.unwrap_or_else(|_| bytes.to_vec())` — a failure to
+ // re-serialise the PRUNED document fell back to the
+ // ORIGINAL bytes, i.e. the scoped principal received the
+ // unfiltered listing this function exists to withhold.
+ // Refuse, exactly as the buffering failure above does: a
+ // filter that cannot run must not fail open.
+ Err(e) => {
+ tracing::error!(error = %e, "could not re-serialise pruned response");
+ return es_error(
+ StatusCode::INTERNAL_SERVER_ERROR,
+ "response could not be authorized".to_string(),
+ );
+ }
+ }
}
// Not JSON after all (some `_cat` handlers answer text under a
// JSON content type); fall through to the line filter.
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 91c93f51d..95a9801a0 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -13412,6 +13412,128 @@ mod malformed_bulk_route_tests {
}
}
+/// Issue #204 — an ingest pipeline that cannot be compiled must be REFUSED,
+/// not stored, acknowledged and quietly skipped.
+#[cfg(test)]
+mod ingest_pipeline_fail_closed_tests {
+ use super::*;
+ use axum::{
+ body::{to_bytes, Body},
+ http::{Request, StatusCode},
+ };
+ use tower::ServiceExt;
+ use xerj_common::{
+ config::{Config, WalSync},
+ metrics::Metrics,
+ };
+ use xerj_engine::Engine;
+
+ fn test_state() -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = WalSync::Async;
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ async fn put_pipeline(state: AppState, id: &str, body: Value) -> (StatusCode, Value) {
+ let response = crate::router::build_es_compat_router(state)
+ .oneshot(
+ Request::put(format!("/_ingest/pipeline/{id}"))
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .expect("request"),
+ )
+ .await
+ .expect("route response");
+ let status = response.status();
+ let bytes = to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("response bytes");
+ (
+ status,
+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),
+ )
+ }
+
+ #[tokio::test]
+ async fn an_uncompilable_pipeline_is_rejected_and_not_stored() {
+ let state = test_state();
+
+ // `social_security` is not a PII type xerj can redact. Pre-fix this
+ // answered 200/acknowledged, `GET` echoed it back, and every document
+ // routed through it kept its PII.
+ let (status, body) = put_pipeline(
+ state.clone(),
+ "redact",
+ json!({
+ "processors": [
+ { "pii_redaction": { "types": ["social_security"] } }
+ ]
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "body: {body}");
+
+ // Nothing was stored: GET must 404 rather than echo a pipeline that
+ // does not exist in compiled form.
+ let response = crate::router::build_es_compat_router(state.clone())
+ .oneshot(
+ Request::get("/_ingest/pipeline/redact")
+ .body(Body::empty())
+ .expect("request"),
+ )
+ .await
+ .expect("route response");
+ assert_eq!(response.status(), StatusCode::NOT_FOUND);
+ assert!(!state.engine.pipelines.contains_key("redact"));
+ assert!(!state.engine.transform_pipelines.contains_key("redact"));
+ }
+
+ #[tokio::test]
+ async fn a_processor_entry_that_is_not_a_processor_is_rejected() {
+ // Pre-fix `filter_map` dropped this silently and the pipeline compiled
+ // with zero stages.
+ let (status, body) = put_pipeline(
+ test_state(),
+ "broken",
+ json!({ "processors": [ "lowercase" ] }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_REQUEST, "body: {body}");
+ }
+
+ #[tokio::test]
+ async fn a_compilable_pipeline_still_round_trips() {
+ let state = test_state();
+ let (status, _) = put_pipeline(
+ state.clone(),
+ "ok",
+ json!({
+ "processors": [
+ { "pii_redaction": { "types": ["email"] } },
+ { "set": { "field": "env", "value": "prod" } }
+ ]
+ }),
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert!(state.engine.transform_pipelines.contains_key("ok"));
+
+ let response = crate::router::build_es_compat_router(state)
+ .oneshot(
+ Request::get("/_ingest/pipeline/ok")
+ .body(Body::empty())
+ .expect("request"),
+ )
+ .await
+ .expect("route response");
+ assert_eq!(response.status(), StatusCode::OK);
+ }
+}
+
#[cfg(test)]
mod script_query_end_to_end_tests {
//! Regression coverage for a real bug: the `script` query's unit tests
@@ -21389,12 +21511,17 @@ pub async fn put_ingest_pipeline(
Path(id): Path<String>,
Json(body): Json<Value>,
) -> impl IntoResponse {
- // Store the raw JSON for GET /_ingest/pipeline.
- state.engine.pipelines.insert(id.clone(), body.clone());
// Convert ES processor format → xerj stage format, then compile.
// ES: {"processors": [{"set": {"field":"x","value":"y"}}]}
// xerj: {"stages": [{"type": "set", "config": {"field":"x","value":"y"}}]}
//
+ // Issue #204 — the raw body used to be stored FIRST and the compile
+ // failure only warn-logged, so a pipeline with an unsupported processor
+ // was answered `200 {"acknowledged": true}`, echoed back verbatim by
+ // `GET /_ingest/pipeline/{id}`, and never ran: the redaction / grok / date
+ // parsing the caller asked for silently did not happen. Compile first,
+ // store only on success, and answer 400 like ES does.
+ //
// ES → xerj name mapping for processors whose names differ:
fn map_proc_name(es_name: &str) -> String {
match es_name {
@@ -21408,23 +21535,31 @@ pub async fn put_ingest_pipeline(
other => other.to_string(),
}
}
- let xerj_config =
- if let Some(processors) = body.get("processors").and_then(Value::as_array) {
- let stages: Vec<Value> = processors.iter().filter_map(|proc| {
- let obj = proc.as_object()?;
- let (proc_type, proc_config) = obj.iter().next()?;
+ let xerj_config = if let Some(processors) = body.get("processors").and_then(Value::as_array) {
+ let mut stages: Vec<Value> = Vec::with_capacity(processors.len());
+ for (i, proc) in processors.iter().enumerate() {
+ // A processor entry that is not a single-key object used to be
+ // dropped by `filter_map` — the pipeline then compiled with fewer
+ // stages than were written, and still acknowledged.
+ let Some((proc_type, proc_config)) = proc.as_object().and_then(|o| o.iter().next())
+ else {
+ let e = xerj_common::XerjError::invalid_mapping(format!(
+ "pipeline [{id}] processor[{i}] is not a processor object \
+ (expected {{\"<processor>\": {{...}}}})"
+ ));
+ return ApiError::new(e).into_response();
+ };
let xerj_type = map_proc_name(proc_type.as_str());
// Adapt ES config shapes to xerj config shapes where they differ.
let adapted_config = match proc_type.as_str() {
"remove" => {
// ES: {"field": "x"} or {"field": ["x","y"]}
// xerj drop_field: {"fields": ["x","y"]}
- let fields = match proc_config.get("field") {
+ match proc_config.get("field") {
Some(Value::String(s)) => json!({"fields": [s]}),
Some(Value::Array(a)) => json!({"fields": a}),
_ => proc_config.clone(),
- };
- fields
+ }
}
"rename" => {
// ES: {"field": "old", "target_field": "new"}
@@ -21436,17 +21571,26 @@ pub async fn put_ingest_pipeline(
}
_ => proc_config.clone(),
};
- Some(json!({"type": xerj_type, "config": adapted_config}))
- }).collect();
- json!({
- "description": body.get("description").and_then(Value::as_str).unwrap_or(""),
- "stages": stages
- })
- } else {
- body.clone()
- };
+ stages.push(json!({"type": xerj_type, "config": adapted_config}));
+ }
+ json!({
+ "description": body.get("description").and_then(Value::as_str).unwrap_or(""),
+ "stages": stages
+ })
+ } else {
+ body.clone()
+ };
+ // `create_pipeline` publishes into BOTH the compiled map and the
+ // GET-visible `pipelines` map, and only on success — so a rejected
+ // definition leaves no phantom entry to echo back. (On success the stored
+ // blob is the translated xerj config, exactly as before: `create_pipeline`
+ // used to overwrite the raw body this handler had already inserted.)
if let Err(e) = state.engine.create_pipeline(&id, xerj_config) {
- tracing::warn!(pipeline = %id, error = %e, "pipeline stored but failed to compile");
+ tracing::warn!(pipeline = %id, error = %e, "rejected ingest pipeline definition");
+ let err = xerj_common::XerjError::invalid_mapping(format!(
+ "pipeline [{id}] cannot be created: {e}"
+ ));
+ return ApiError::new(err).into_response();
}
Json(json!({ "acknowledged": true })).into_response()
}
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index 824dc13b3..199f2f062 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -821,11 +821,29 @@ pub async fn admin_backup(State(state): State<AppState>, body: Bytes) -> impl In
let started = Instant::now();
let request_id = Uuid::new_v4().to_string();
- // Body is optional and lenient — `{}` or empty both mean "defaults".
+ // Body is optional — empty and `{}` both mean "defaults". A body that is
+ // PRESENT but unparseable is not: issue #204, this was
+ // `.unwrap_or_default()`, so a malformed request silently discarded the
+ // caller's `repo_path` / `name` / `indices` and ran a *different* backup
+ // (default location, every index) under a `201 Created`. An operator
+ // reading "created" had no way to know the archive they asked for is not
+ // the archive that exists.
let req: BackupRequest = if body.is_empty() {
BackupRequest::default()
} else {
- serde_json::from_slice(&body).unwrap_or_default()
+ match serde_json::from_slice(&body) {
+ Ok(r) => r,
+ Err(e) => {
+ return native_error(
+ xerj_common::XerjError::invalid_document_json(format!(
+ "backup request body is not valid BackupRequest JSON: {e}"
+ )),
+ Some(&request_id),
+ started.elapsed().as_millis() as u64,
+ )
+ .into_response();
+ }
+ }
};
let data_dir = state.engine.config().server.data_dir.clone();
diff --git a/engine/crates/xerj-api/src/state.rs b/engine/crates/xerj-api/src/state.rs
index 34d7534bb..4491bdb0d 100644
--- a/engine/crates/xerj-api/src/state.rs
+++ b/engine/crates/xerj-api/src/state.rs
@@ -304,30 +304,63 @@ impl MlDetector {
}
/// Load all persisted detectors from `<data_dir>/_ml/detectors.json`.
- /// Missing/corrupt files yield an empty registry (best effort).
+ ///
+ /// An ABSENT file is a fresh node and yields an empty registry silently. A
+ /// file that is present but unreadable or unparseable also yields an empty
+ /// registry — every detector the operator configured disappears — so it is
+ /// logged at ERROR rather than swallowed (issue #204). Boot is not failed
+ /// on it: a node that refuses to start is worse than one that starts with
+ /// its ML jobs missing and says so.
pub fn load_all(data_dir: &str) -> DashMap<String, MlDetector> {
let map = DashMap::new();
let path = Self::registry_path(data_dir);
- if let Ok(bytes) = std::fs::read(&path) {
- if let Ok(list) = serde_json::from_slice::<Vec<MlDetector>>(&bytes) {
- for d in list {
- map.insert(d.detector_id.clone(), d);
+ match std::fs::read(&path) {
+ Ok(bytes) => match serde_json::from_slice::<Vec<MlDetector>>(&bytes) {
+ Ok(list) => {
+ for d in list {
+ map.insert(d.detector_id.clone(), d);
+ }
}
- }
+ Err(e) => tracing::error!(
+ path = %path.display(), error = %e,
+ "ml detector registry is unparseable — ALL configured detectors are \
+ being dropped; move the file aside to stop it being overwritten"
+ ),
+ },
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+ Err(e) => tracing::error!(
+ path = %path.display(), error = %e,
+ "ml detector registry could not be read — ALL configured detectors are \
+ being dropped"
+ ),
}
map
}
- /// Persist the full registry to disk (best effort — swallows I/O errors).
+ /// Persist the full registry to disk.
+ ///
+ /// Issue #204: the write used to be `let _ = ..`, so a detector accepted
+ /// with a 200 could fail to reach disk and vanish at the next restart with
+ /// no signal anywhere. Still best-effort — the caller has already
+ /// acknowledged — but no longer silent.
pub fn save_all(data_dir: &str, registry: &DashMap<String, MlDetector>) {
let path = Self::registry_path(data_dir);
- if let Some(parent) = path.parent() {
- let _ = std::fs::create_dir_all(parent);
- }
let list: Vec<MlDetector> = registry.iter().map(|e| e.value().clone()).collect();
- if let Ok(bytes) = serde_json::to_vec_pretty(&list) {
- let _ = std::fs::write(&path, bytes);
+ if let Err(e) = Self::write_registry(&path, &list) {
+ tracing::error!(
+ path = %path.display(), error = %e,
+ "failed to persist ml detector registry — detectors are live in memory \
+ but WILL BE LOST on restart"
+ );
+ }
+ }
+
+ fn write_registry(path: &std::path::Path, list: &[MlDetector]) -> std::io::Result<()> {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)?;
}
+ let bytes = serde_json::to_vec_pretty(list)?;
+ std::fs::write(path, bytes)
}
}
@@ -396,31 +429,58 @@ impl MlDatafeed {
/// Load all persisted datafeeds. Runtime state (`state`, cursor) is reset:
/// background scoring tasks and the results store are in-memory only and do
/// not survive a restart, so a freshly-loaded datafeed is always `stopped`.
+ ///
+ /// A present-but-unreadable registry drops every configured datafeed; that
+ /// is logged at ERROR rather than swallowed (issue #204). An absent file is
+ /// a fresh node and stays quiet.
pub fn load_all(data_dir: &str) -> DashMap<String, MlDatafeed> {
let map = DashMap::new();
let path = Self::registry_path(data_dir);
- if let Ok(bytes) = std::fs::read(&path) {
- if let Ok(list) = serde_json::from_slice::<Vec<MlDatafeed>>(&bytes) {
- for mut d in list {
- d.state = "stopped".to_string();
- d.last_scored_ms = None;
- map.insert(d.datafeed_id.clone(), d);
+ match std::fs::read(&path) {
+ Ok(bytes) => match serde_json::from_slice::<Vec<MlDatafeed>>(&bytes) {
+ Ok(list) => {
+ for mut d in list {
+ d.state = "stopped".to_string();
+ d.last_scored_ms = None;
+ map.insert(d.datafeed_id.clone(), d);
+ }
}
- }
+ Err(e) => tracing::error!(
+ path = %path.display(), error = %e,
+ "ml datafeed registry is unparseable — ALL configured datafeeds are \
+ being dropped; move the file aside to stop it being overwritten"
+ ),
+ },
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
+ Err(e) => tracing::error!(
+ path = %path.display(), error = %e,
+ "ml datafeed registry could not be read — ALL configured datafeeds are \
+ being dropped"
+ ),
}
map
}
- /// Persist the full registry to disk (best effort — swallows I/O errors).
+ /// Persist the full registry to disk. Best-effort but not silent — see
+ /// [`MlDetector::save_all`] (issue #204).
pub fn save_all(data_dir: &str, registry: &DashMap<String, MlDatafeed>) {
let path = Self::registry_path(data_dir);
- if let Some(parent) = path.parent() {
- let _ = std::fs::create_dir_all(parent);
- }
let list: Vec<MlDatafeed> = registry.iter().map(|e| e.value().clone()).collect();
- if let Ok(bytes) = serde_json::to_vec_pretty(&list) {
- let _ = std::fs::write(&path, bytes);
+ if let Err(e) = Self::write_registry(&path, &list) {
+ tracing::error!(
+ path = %path.display(), error = %e,
+ "failed to persist ml datafeed registry — datafeeds are live in memory \
+ but WILL BE LOST on restart"
+ );
+ }
+ }
+
+ fn write_registry(path: &std::path::Path, list: &[MlDatafeed]) -> std::io::Result<()> {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)?;
}
+ let bytes = serde_json::to_vec_pretty(list)?;
+ std::fs::write(path, bytes)
}
}
diff --git a/engine/crates/xerj-console-api/src/auth/store.rs b/engine/crates/xerj-console-api/src/auth/store.rs
index cfbfec380..2ad73c3af 100644
--- a/engine/crates/xerj-console-api/src/auth/store.rs
+++ b/engine/crates/xerj-console-api/src/auth/store.rs
@@ -209,7 +209,13 @@ pub async fn count_passkeys_for_user(engine: &Engine, user_id: &str) -> ConsoleR
pub async fn delete_passkey(engine: &Engine, credential_id: &str) -> ConsoleResult<()> {
let idx = engine.get_index(indices::PASSKEYS)?;
- let _ = idx.delete_document(credential_id).await;
+ // Issue #204: this was `let _ = ..`, so a delete refused by the engine
+ // (index write block, disk flood stage, storage error) still produced
+ // `204 No Content` — the console said the passkey was revoked while it
+ // remained on disk and kept authenticating. `delete_document` returns
+ // `Ok(false)` for an already-absent credential, so propagating cannot turn
+ // a repeat revoke into an error; only a genuine failure surfaces.
+ idx.delete_document(credential_id).await?;
Ok(())
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 66a2a0ccf..71cd398a0 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -5223,6 +5223,9 @@ pub struct Index {
/// because we only ever take/replace under it, never hold across
/// any awaits.
pub(crate) merge_task: Arc<parking_lot::Mutex<Option<tokio::task::JoinHandle<()>>>>,
+ /// Failed writes of `schema.json` during dynamic-mapping evolution.
+ /// See [`Index::persist_evolved_schema`] (issue #204).
+ schema_persist_failures: AtomicU64,
}
impl Index {
@@ -5341,6 +5344,25 @@ impl Index {
config: &Config,
data_dir: &Path,
) -> Result<Arc<Self>> {
+ // Issue #204 — refuse an analysis block we would only pretend to
+ // honour. `apply_settings` is total (it has to be: it also runs on the
+ // open path), so an unresolvable tokenizer silently became `standard`
+ // and an unresolvable filter was silently dropped — the index was then
+ // built with an analyzer the caller never asked for and nothing in the
+ // 200 response said so. Checked BEFORE the index directory is created
+ // so a rejected create leaves nothing behind.
+ let analysis_problems = AnalyzerRegistry::unsupported_analysis(&settings);
+ if !analysis_problems.is_empty() {
+ return Err(EngineError::Common(xerj_common::XerjError::config(
+ format!(
+ "index [{}] cannot be created: xerj cannot honour this analysis \
+ configuration — {}",
+ name.as_str(),
+ analysis_problems.join("; ")
+ ),
+ )));
+ }
+
let index_dir = data_dir.join(name.as_str());
// Defense-in-depth: IndexName::validate rejects '.', '..', separators,
// and '..' substrings, so join cannot escape data_dir. The debug_assert
@@ -5544,6 +5566,7 @@ impl Index {
flush_signal: Arc::new(SyncFlushCoord::new()),
external_versions: Arc::new(dashmap::DashMap::new()),
merge_task: Arc::new(parking_lot::Mutex::new(None)),
+ schema_persist_failures: AtomicU64::new(0),
});
// Kick off the background merge pass. 5 s cadence is aggressive
// enough to collapse a burst of flushes quickly without burning a
@@ -5891,6 +5914,7 @@ impl Index {
flush_signal: Arc::new(SyncFlushCoord::new()),
external_versions: Arc::new(dashmap::DashMap::new()),
merge_task: Arc::new(parking_lot::Mutex::new(None)),
+ schema_persist_failures: AtomicU64::new(0),
});
index.spawn_merge_task(5);
// RC4 W2 item 17: a flush-time-stale HNSW snapshot used to stay
@@ -17642,7 +17666,7 @@ impl Index {
}
}
if schema_changed {
- let _ = self.save_schema(&schema).await;
+ self.persist_evolved_schema(&schema).await;
}
}
@@ -17716,10 +17740,45 @@ impl Index {
}
}
if schema_changed {
- let _ = self.save_schema(&schema).await;
+ self.persist_evolved_schema(&schema).await;
+ }
+ }
+
+ /// Persist a dynamic-mapping evolution, complaining loudly if it does not
+ /// stick.
+ ///
+ /// Issue #204: both callers used to be `let _ = self.save_schema(..)`. The
+ /// in-memory schema had the new fields, `schema.json` did not, and the next
+ /// restart reopened the index with a mapping that had silently rolled back
+ /// — documents already on disk carrying those fields, no mapping for them,
+ /// and not one line anywhere saying why.
+ ///
+ /// This is a log-and-surface fix, not a fail-closed one, and deliberately
+ /// so: schema evolution runs *after* the document has been admitted, so
+ /// there is no caller left to fail. Losing the durable mapping is the lesser
+ /// evil against rejecting a document that is already indexed — but it is
+ /// an ERROR, not a shrug. `schema_persist_failures` counts it so a health
+ /// check can see it without reading logs.
+ async fn persist_evolved_schema(&self, schema: &ManagedSchema) {
+ if let Err(e) = self.save_schema(schema).await {
+ self.schema_persist_failures.fetch_add(1, Ordering::Relaxed);
+ tracing::error!(
+ index = self.name.as_str(),
+ path = %self.data_dir.join("schema.json").display(),
+ error = %e,
+ "failed to persist dynamic mapping evolution — the new fields are \
+ live in memory but WILL BE LOST on restart"
+ );
}
}
+ /// Number of times [`Self::persist_evolved_schema`] could not write
+ /// `schema.json`. Non-zero means the on-disk mapping is behind the
+ /// in-memory one and a restart will lose dynamically-mapped fields.
+ pub fn schema_persist_failures(&self) -> u64 {
+ self.schema_persist_failures.load(Ordering::Relaxed)
+ }
+
async fn save_schema(&self, schema: &ManagedSchema) -> Result<()> {
let path = self.data_dir.join("schema.json");
let bytes = serde_json::to_vec_pretty(schema)?;
diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rs
index 208918097..6654e2b49 100644
--- a/engine/crates/xerj-engine/tests/integration.rs
+++ b/engine/crates/xerj-engine/tests/integration.rs
@@ -6471,3 +6471,116 @@ async fn test_bare_count_bucket_script_inside_a_terms_bucket_agrees_on_both_path
}
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Issue #204 — an analysis block xerj cannot honour must be REFUSED at create
+// time, not accepted and quietly replaced with something weaker.
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn create_index_rejects_an_analysis_block_it_cannot_honour() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ // Pre-fix: `autocomplete_tok` was never declared, `apply_settings`
+ // substituted a StandardTokenizer, and the caller got a 200 plus an index
+ // that does not do prefix matching at all.
+ let settings = json!({
+ "analysis": {
+ "analyzer": {
+ "default": {
+ "type": "custom",
+ "tokenizer": "autocomplete_tok",
+ "filter": ["lowercase"]
+ }
+ }
+ }
+ });
+ let err = engine
+ .create_index_with_settings("bad_tok_idx", Schema::empty(), settings)
+ .expect_err("an unresolvable tokenizer must fail index creation");
+ let msg = err.to_string();
+ assert!(msg.contains("autocomplete_tok"), "unhelpful error: {msg}");
+
+ // Refused before anything hit the disk, and not registered in the engine.
+ assert!(engine.get_index("bad_tok_idx").is_err());
+ assert!(
+ !dir.path().join("bad_tok_idx").exists(),
+ "a rejected create must not leave an index directory behind"
+ );
+
+ // An unresolvable token filter is refused for the same reason: dropping
+ // `lowercase` silently makes every match case-sensitive.
+ let err = engine
+ .create_index_with_settings(
+ "bad_filter_idx",
+ Schema::empty(),
+ json!({
+ "analysis": {
+ "analyzer": {
+ "default": {
+ "type": "custom",
+ "tokenizer": "standard",
+ "filter": ["lowercse"]
+ }
+ }
+ }
+ }),
+ )
+ .expect_err("an unresolvable token filter must fail index creation");
+ assert!(err.to_string().contains("lowercse"), "{err}");
+}
+
+#[tokio::test]
+async fn create_index_still_accepts_every_analysis_construct_we_support() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ // The full supported surface in one settings block — this is the guard
+ // against the #204 check over-rejecting.
+ let settings = json!({
+ "analysis": {
+ "filter": {
+ "my_synonyms": { "type": "synonym", "synonyms": ["fast,quick"] },
+ "my_length": { "type": "length", "min": 2, "max": 40 },
+ "my_shingle": { "type": "shingle", "max_shingle_size": 2 },
+ "my_fold": { "type": "asciifolding" }
+ },
+ "tokenizer": {
+ "ng": { "type": "ngram", "min_gram": 2, "max_gram": 3 },
+ "edge": { "type": "edge_ngram", "min_gram": 1, "max_gram": 8 },
+ "pat": { "type": "pattern", "pattern": "\\W+" }
+ },
+ "analyzer": {
+ "default": {
+ "type": "custom",
+ "tokenizer": "edge",
+ "filter": ["lowercase", "stop", "stemmer", "asciifolding",
+ "my_synonyms", "my_length", "my_shingle", "my_fold"]
+ },
+ "plain": { "type": "english" }
+ }
+ }
+ });
+ engine
+ .create_index_with_settings("good_idx", Schema::empty(), settings)
+ .expect("a fully supported analysis block must still create");
+
+ let idx = engine.get_index("good_idx").unwrap();
+ idx.index_document(Some("1".into()), json!({ "title": "javascript" }))
+ .await
+ .unwrap();
+ let result = idx
+ .search(&make_search(json!({"match": {"title": "java"}})))
+ .await
+ .unwrap();
+ assert_eq!(
+ result.total.value, 1,
+ "the edge-ngram analyzer must actually be in force"
+ );
+
+ // An index with no analysis block at all is untouched by the check.
+ engine
+ .create_index_with_settings("plain_idx", Schema::empty(), json!({}))
+ .expect("empty settings must still create");
+}
diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rs
index 9924d9f9b..031250d70 100644
--- a/engine/crates/xerj-fts/src/analyzer.rs
+++ b/engine/crates/xerj-fts/src/analyzer.rs
@@ -894,6 +894,15 @@ fn nfkc_normalize(s: &str) -> String {
// ── Analyzer registry ─────────────────────────────────────────────────────────
+/// `analysis.filter.*.type` values [`AnalyzerRegistry::apply_settings`] can
+/// actually build. Must stay in step with the `match filter_type` arms there —
+/// `supported_analysis_types_all_build` pins that.
+pub const SUPPORTED_FILTER_TYPES: &[&str] = &["synonym", "length", "shingle", "asciifolding"];
+
+/// `analysis.tokenizer.*.type` values [`AnalyzerRegistry::apply_settings`] can
+/// actually build. Must stay in step with the `match tok_type` arms there.
+pub const SUPPORTED_TOKENIZER_TYPES: &[&str] = &["ngram", "edge_ngram", "pattern"];
+
/// Central registry that maps analyzer names to their pipelines.
///
/// Built-in analyzers are registered by default; custom analyzers can be
@@ -1055,6 +1064,12 @@ impl AnalyzerRegistry {
/// Extend this registry with custom analyzer definitions parsed from an
/// index `settings.analysis` block.
///
+ /// **Total by design.** Every construct this method cannot honour is
+ /// skipped or replaced rather than raised, because it also runs on the
+ /// index-open path where a hard failure would make an existing index
+ /// unopenable. Callers on the *create* path must gate on
+ /// [`AnalyzerRegistry::unsupported_analysis`] first — see issue #204.
+ ///
/// Accepts the ES-compatible JSON format:
/// ```json
/// {
@@ -1076,7 +1091,7 @@ impl AnalyzerRegistry {
/// }
/// ```
pub fn apply_settings(&mut self, settings: &serde_json::Value) {
- let analysis = match settings.pointer("/analysis") {
+ let analysis = match Self::analysis_block(settings) {
Some(a) => a,
None => return,
};
@@ -1215,8 +1230,16 @@ impl AnalyzerRegistry {
if analyzer_type != "custom" {
// For non-custom types, look up the built-in by type name.
- if let Some(builtin) = self.get_analyzer(analyzer_type) {
- self.analyzers.insert(analyzer_name.clone(), builtin);
+ match self.get_analyzer(analyzer_type) {
+ Some(builtin) => {
+ self.analyzers.insert(analyzer_name.clone(), builtin);
+ }
+ None => tracing::warn!(
+ analyzer = analyzer_name.as_str(),
+ analyzer_type,
+ "unknown analyzer type — analyzer not registered, \
+ fields referencing it fall back to `standard`"
+ ),
}
continue;
}
@@ -1231,7 +1254,22 @@ impl AnalyzerRegistry {
.get(tokenizer_name)
.cloned()
.or_else(|| self.resolve_builtin_tokenizer(tokenizer_name))
- .unwrap_or_else(|| Arc::new(StandardTokenizer));
+ .unwrap_or_else(|| {
+ // Not equivalent: substituting `standard` for (say)
+ // `edge_ngram` builds a completely different index.
+ // `unsupported_analysis` rejects this at index-create
+ // time; reaching it here means we are re-opening an
+ // index whose settings.json predates that check, so
+ // say so loudly rather than degrade in silence
+ // (issue #204).
+ tracing::warn!(
+ analyzer = analyzer_name.as_str(),
+ tokenizer = tokenizer_name,
+ "unknown tokenizer in custom analyzer — falling back to \
+ `standard`; this index does NOT tokenize as configured"
+ );
+ Arc::new(StandardTokenizer)
+ });
// Resolve token filters.
let filter_names: Vec<&str> = analyzer_def
@@ -1263,6 +1301,155 @@ impl AnalyzerRegistry {
}
}
+ /// Report every `settings.analysis` construct that [`apply_settings`] would
+ /// accept and then silently NOT honour.
+ ///
+ /// [`apply_settings`]: AnalyzerRegistry::apply_settings
+ ///
+ /// Issue #204 — "degrade loudly or not at all". `apply_settings` is
+ /// deliberately total: it never fails, because it also runs on the
+ /// index-*open* path where refusing to build a registry would brick an
+ /// existing index. That leniency is right at open time and wrong at create
+ /// time, where every one of these constructs used to be accepted with a
+ /// `200 {"acknowledged": true}` and then quietly replaced by something
+ /// weaker:
+ ///
+ /// - an unresolvable `tokenizer` became `standard` — an `edge_ngram`
+ /// autocomplete index that matches nothing;
+ /// - an unresolvable `filter` was dropped — a missing `lowercase` turns
+ /// every match case-sensitive;
+ /// - an unsupported filter/tokenizer `type` was skipped entirely;
+ /// - an unknown non-custom analyzer `type` left the analyzer unregistered,
+ /// so fields naming it silently got `standard`.
+ ///
+ /// Returns one human-readable message per problem, empty when the block can
+ /// be honoured exactly as written. `Index::create_with_settings` turns a
+ /// non-empty result into a 400 so the caller learns at the door.
+ ///
+ /// `settings` may be either the full envelope (`{"settings": {"analysis":
+ /// …}}`) or the inner settings object — the same two shapes
+ /// `build_registry_from_settings` accepts.
+ pub fn unsupported_analysis(settings: &serde_json::Value) -> Vec<String> {
+ let root = settings.pointer("/settings").unwrap_or(settings);
+ let Some(analysis) = Self::analysis_block(root) else {
+ return Vec::new();
+ };
+ let probe = Self::with_defaults();
+ let mut problems = Vec::new();
+
+ // Declared names are collected even when their `type` is unsupported,
+ // so a bad type is reported once (as a type problem) rather than twice
+ // (again as a dangling reference from every analyzer that uses it).
+ let mut declared_filters: HashSet<&str> = HashSet::new();
+ if let Some(filter_map) = analysis.pointer("/filter").and_then(|v| v.as_object()) {
+ for (filter_name, filter_def) in filter_map {
+ declared_filters.insert(filter_name.as_str());
+ let filter_type = filter_def
+ .get("type")
+ .and_then(|v| v.as_str())
+ .unwrap_or("");
+ if !SUPPORTED_FILTER_TYPES.contains(&filter_type) {
+ problems.push(format!(
+ "token filter [{filter_name}]: unsupported type [{filter_type}] \
+ (supported: {})",
+ SUPPORTED_FILTER_TYPES.join(", ")
+ ));
+ }
+ }
+ }
+
+ let mut declared_tokenizers: HashSet<&str> = HashSet::new();
+ if let Some(tok_map) = analysis.pointer("/tokenizer").and_then(|v| v.as_object()) {
+ for (tok_name, tok_def) in tok_map {
+ declared_tokenizers.insert(tok_name.as_str());
+ let tok_type = tok_def.get("type").and_then(|v| v.as_str()).unwrap_or("");
+ if !SUPPORTED_TOKENIZER_TYPES.contains(&tok_type) {
+ problems.push(format!(
+ "tokenizer [{tok_name}]: unsupported type [{tok_type}] \
+ (supported: {})",
+ SUPPORTED_TOKENIZER_TYPES.join(", ")
+ ));
+ continue;
+ }
+ if tok_type == "pattern" {
+ let pattern = tok_def
+ .get("pattern")
+ .and_then(|v| v.as_str())
+ .unwrap_or(r"\W+");
+ if PatternTokenizer::new(pattern).is_err() {
+ problems.push(format!(
+ "tokenizer [{tok_name}]: invalid pattern regex [{pattern}]"
+ ));
+ }
+ }
+ }
+ }
+
+ if let Some(analyzer_map) = analysis.pointer("/analyzer").and_then(|v| v.as_object()) {
+ for (analyzer_name, analyzer_def) in analyzer_map {
+ let analyzer_type = analyzer_def
+ .get("type")
+ .and_then(|v| v.as_str())
+ .unwrap_or("custom");
+
+ if analyzer_type != "custom" {
+ if probe.get_analyzer(analyzer_type).is_none() {
+ problems.push(format!(
+ "analyzer [{analyzer_name}]: unknown analyzer type [{analyzer_type}]"
+ ));
+ }
+ continue;
+ }
+
+ let tokenizer_name = analyzer_def
+ .get("tokenizer")
+ .and_then(|v| v.as_str())
+ .unwrap_or("standard");
+ if !declared_tokenizers.contains(tokenizer_name)
+ && probe.resolve_builtin_tokenizer(tokenizer_name).is_none()
+ {
+ problems.push(format!(
+ "analyzer [{analyzer_name}]: unknown tokenizer [{tokenizer_name}]"
+ ));
+ }
+
+ let filter_names: Vec<&str> = analyzer_def
+ .get("filter")
+ .and_then(|v| v.as_array())
+ .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
+ .unwrap_or_default();
+ for fname in filter_names {
+ if !declared_filters.contains(fname)
+ && probe.resolve_builtin_filter(fname).is_none()
+ {
+ problems.push(format!(
+ "analyzer [{analyzer_name}]: unknown token filter [{fname}]"
+ ));
+ }
+ }
+ }
+ }
+
+ problems.sort();
+ problems
+ }
+
+ /// Locate the `analysis` block inside an index-settings object.
+ ///
+ /// ES accepts both the shorthand (`settings.analysis.*`) and the canonical
+ /// namespaced form (`settings.index.analysis.*`). Only the shorthand used
+ /// to be read here, so a settings body written the canonical way was
+ /// accepted, echoed back by `GET /{index}/_settings`, and then contributed
+ /// no analyzers at all — every field naming one silently got `standard`
+ /// (issue #204). Kept as one helper so the builder and
+ /// [`AnalyzerRegistry::unsupported_analysis`] can never disagree about
+ /// which block is in force.
+ fn analysis_block(settings: &serde_json::Value) -> Option<&serde_json::Value> {
+ settings
+ .pointer("/analysis")
+ .or_else(|| settings.pointer("/index/analysis"))
+ }
+
/// Resolve a tokenizer by its built-in name.
fn resolve_builtin_tokenizer(&self, name: &str) -> Option<Arc<dyn Tokenizer>> {
match name {
@@ -1998,3 +2185,192 @@ mod tests {
assert!(terms.contains(&"test".to_string()), "terms={terms:?}");
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Issue #204 — `settings.analysis` constructs that used to degrade in silence
+// ─────────────────────────────────────────────────────────────────────────────
+#[cfg(test)]
+mod unsupported_analysis_tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn a_settings_block_we_can_honour_reports_nothing() {
+ let settings = json!({
+ "analysis": {
+ "filter": {
+ "my_synonyms": { "type": "synonym", "synonyms": ["fast,quick"] },
+ "my_length": { "type": "length", "min": 3, "max": 50 }
+ },
+ "tokenizer": {
+ "autocomplete_tok": { "type": "edge_ngram", "min_gram": 1, "max_gram": 10 }
+ },
+ "analyzer": {
+ "a": { "type": "custom", "tokenizer": "autocomplete_tok",
+ "filter": ["lowercase", "my_synonyms", "my_length"] },
+ "b": { "type": "english" }
+ }
+ }
+ });
+ assert!(
+ AnalyzerRegistry::unsupported_analysis(&settings).is_empty(),
+ "{:?}",
+ AnalyzerRegistry::unsupported_analysis(&settings)
+ );
+ // Absent analysis block, and the outer-envelope shape, both work.
+ assert!(AnalyzerRegistry::unsupported_analysis(&json!({})).is_empty());
+ assert!(
+ AnalyzerRegistry::unsupported_analysis(&json!({ "settings": settings })).is_empty()
+ );
+ }
+
+ /// ES's canonical `index.analysis.*` form must be BUILT, not ignored — and
+ /// therefore also validated.
+ #[test]
+ fn the_index_namespaced_form_is_honoured_by_builder_and_validator() {
+ let namespaced = json!({
+ "index": {
+ "analysis": {
+ "tokenizer": { "edge": { "type": "edge_ngram", "min_gram": 1, "max_gram": 8 } },
+ "analyzer": { "ac": { "type": "custom", "tokenizer": "edge" } }
+ }
+ }
+ });
+ assert!(
+ AnalyzerRegistry::unsupported_analysis(&namespaced).is_empty(),
+ "{:?}",
+ AnalyzerRegistry::unsupported_analysis(&namespaced)
+ );
+
+ let mut registry = AnalyzerRegistry::with_defaults();
+ registry.apply_settings(&namespaced);
+ let ac = registry
+ .get_analyzer("ac")
+ .expect("index.analysis.analyzer.ac must be registered, not silently dropped");
+ assert!(
+ ac.analyze_to_terms("java").contains(&"ja".to_string()),
+ "the edge-ngram tokenizer must be the one in force"
+ );
+
+ // And a broken one in that form is caught rather than ignored.
+ let bad = json!({ "index": { "analysis": { "analyzer": {
+ "ac": { "type": "custom", "tokenizer": "nope" } } } } });
+ assert_eq!(AnalyzerRegistry::unsupported_analysis(&bad).len(), 1);
+ }
+
+ #[test]
+ fn unresolvable_tokenizer_is_reported() {
+ // Pre-fix: this analyzer was registered with a StandardTokenizer and
+ // the index tokenized nothing like the caller asked for.
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "analyzer": {
+ "autocomplete": { "type": "custom", "tokenizer": "edge_ngram_tok" }
+ }
+ }
+ }));
+ assert_eq!(problems.len(), 1, "{problems:?}");
+ assert!(problems[0].contains("edge_ngram_tok"), "{problems:?}");
+ }
+
+ #[test]
+ fn unresolvable_filter_is_reported() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "analyzer": {
+ "a": { "type": "custom", "tokenizer": "standard",
+ "filter": ["lowercse", "word_delimiter"] }
+ }
+ }
+ }));
+ assert_eq!(problems.len(), 2, "{problems:?}");
+ assert!(
+ problems.iter().any(|p| p.contains("lowercse")),
+ "{problems:?}"
+ );
+ assert!(
+ problems.iter().any(|p| p.contains("word_delimiter")),
+ "{problems:?}"
+ );
+ }
+
+ #[test]
+ fn unsupported_component_type_is_reported_once_not_twice() {
+ // A filter declared with an unsupported `type` is skipped by
+ // apply_settings, so every analyzer naming it also loses it. Report
+ // the actionable cause (the type) and not the knock-on reference.
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": {
+ "filter": { "stem": { "type": "hunspell", "locale": "en_US" } },
+ "tokenizer": { "tok": { "type": "char_group" } },
+ "analyzer": {
+ "a": { "type": "custom", "tokenizer": "tok", "filter": ["stem"] }
+ }
+ }
+ }));
+ assert_eq!(problems.len(), 2, "{problems:?}");
+ assert!(
+ problems.iter().any(|p| p.contains("hunspell")),
+ "{problems:?}"
+ );
+ assert!(
+ problems.iter().any(|p| p.contains("char_group")),
+ "{problems:?}"
+ );
+ }
+
+ #[test]
+ fn unknown_non_custom_analyzer_type_is_reported() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": { "analyzer": { "a": { "type": "kuromoji" } } }
+ }));
+ assert_eq!(problems.len(), 1, "{problems:?}");
+ assert!(problems[0].contains("kuromoji"), "{problems:?}");
+ }
+
+ #[test]
+ fn invalid_pattern_tokenizer_regex_is_reported() {
+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({
+ "analysis": { "tokenizer": { "t": { "type": "pattern", "pattern": "[unclosed" } } }
+ }));
+ assert_eq!(problems.len(), 1, "{problems:?}");
+ assert!(
+ problems[0].contains("invalid pattern regex"),
+ "{problems:?}"
+ );
+ }
+
+ /// The allow-lists are the contract `unsupported_analysis` enforces; if one
+ /// drifts from the `match` arms in `apply_settings` we would start
+ /// rejecting configurations we can serve (or accepting ones we cannot).
+ #[test]
+ fn supported_analysis_types_all_build() {
+ for t in SUPPORTED_FILTER_TYPES {
+ let mut registry = AnalyzerRegistry::with_defaults();
+ registry.apply_settings(&json!({
+ "analysis": {
+ "filter": { "f": { "type": t, "synonyms": ["a,b"] } },
+ "analyzer": { "a": { "type": "custom", "tokenizer": "standard",
+ "filter": ["f"] } }
+ }
+ }));
+ assert!(
+ registry.get_analyzer("a").is_some(),
+ "filter type `{t}` is on the supported list but did not build"
+ );
+ }
+ for t in SUPPORTED_TOKENIZER_TYPES {
+ let mut registry = AnalyzerRegistry::with_defaults();
+ registry.apply_settings(&json!({
+ "analysis": {
+ "tokenizer": { "tk": { "type": t } },
+ "analyzer": { "a": { "type": "custom", "tokenizer": "tk" } }
+ }
+ }));
+ assert!(
+ registry.get_analyzer("a").is_some(),
+ "tokenizer type `{t}` is on the supported list but did not build"
+ );
+ }
+ }
+}
diff --git a/engine/crates/xerj-wasm/src/builtins.rs b/engine/crates/xerj-wasm/src/builtins.rs
index 5711d3c6f..5c40fdfc2 100644
--- a/engine/crates/xerj-wasm/src/builtins.rs
+++ b/engine/crates/xerj-wasm/src/builtins.rs
@@ -420,6 +420,23 @@ impl TransformPlugin for GrokPlugin {
}
}
+/// Built-in grok pattern names accepted by the `grok` stage.
+///
+/// `GENERIC` is the explicit opt-in for the catch-everything-as-`message`
+/// pattern. It used to be reachable by *typo*: `grok_pattern` fell through to
+/// it for any unrecognised name, so `"pattern": "NGINX_COMBINE"` compiled
+/// happily and then extracted exactly one field (`message`) out of every
+/// access log line — an accepted-and-ignored configuration (issue #204).
+/// `build_plugin` now rejects names outside this list, so the generic pattern
+/// is only ever used when it was actually asked for.
+pub const GROK_PATTERN_NAMES: &[&str] = &[
+ "NGINX_COMBINED",
+ "APACHE_COMBINED",
+ "SYSLOG",
+ "POSTGRESQL",
+ "GENERIC",
+];
+
/// Returns `(regex_str, field_names)` for a built-in grok pattern.
fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {
match name {
@@ -453,11 +470,9 @@ fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {
"message",
],
),
- _ => (
- // Generic: capture everything as `message`.
- r#"^(?P<message>.+)$"#,
- &["message"],
- ),
+ // "GENERIC" (and, defensively, anything that slipped past
+ // `build_plugin`'s allow-list): capture everything as `message`.
+ _ => (r#"^(?P<message>.+)$"#, &["message"]),
}
}
@@ -479,11 +494,22 @@ pub struct PiiRedactionPlugin {
patterns: Vec<(Regex, &'static str)>,
}
+/// PII type names this plugin knows how to redact.
+///
+/// A name outside this list used to be accepted and then quietly matched
+/// nothing: `PiiRedactionPlugin::new(vec!["social_security".into()])` compiled
+/// to an EMPTY pattern set, so the stage ran on every document, redacted
+/// nothing, and reported success — unredacted PII in the index with no signal
+/// anywhere (issue #204). `build_plugin` now rejects unknown names at
+/// pipeline-creation time.
+pub const PII_TYPE_NAMES: &[&str] = &["email", "ip", "credit_card", "ssn", "phone"];
+
impl PiiRedactionPlugin {
/// Build a new plugin that redacts the given PII types.
///
- /// `types` is a list of strings from `["email", "ip", "credit_card",
- /// "ssn", "phone"]`. An empty list enables all types.
+ /// `types` is a list of strings from [`PII_TYPE_NAMES`]. An empty list
+ /// enables all types. Callers must validate the names first (see
+ /// `build_plugin`) — an unknown name silently contributes no pattern.
pub fn new(types: Vec<String>) -> Self {
let all = types.is_empty();
@@ -611,10 +637,23 @@ impl TransformPlugin for CopyFieldPlugin {
// ConvertTypePlugin
// ─────────────────────────────────────────────────────────────────────────────
+/// Target types the `convert` stage can produce.
+///
+/// An unknown target type used to be accepted at pipeline-creation time and
+/// then hit `_ => None` per document: the field was left exactly as it was and
+/// the document passed. `"type": "int"` (instead of `"integer"`) therefore
+/// produced a pipeline that acknowledged the conversion and never performed it
+/// (issue #204). `build_plugin` now rejects unknown target types.
+pub const CONVERT_TARGET_TYPES: &[&str] = &["integer", "float", "string", "boolean"];
+
/// Convert a field's value to a different type.
///
-/// Supported target types: `integer`, `float`, `string`, `boolean`.
-/// If conversion fails the document is passed through unchanged.
+/// Supported target types are [`CONVERT_TARGET_TYPES`]; an unknown one is
+/// rejected when the pipeline is created. If the *value* cannot be coerced
+/// (e.g. `"abc"` to `integer`) the document is passed through unchanged — a
+/// per-document data condition, not a configuration error. (ES's `convert`
+/// processor fails the document instead; that divergence is pre-existing and
+/// out of scope for the #204 sweep.)
pub struct ConvertTypePlugin {
field: String,
target_type: String,
diff --git a/engine/crates/xerj-wasm/src/pipeline.rs b/engine/crates/xerj-wasm/src/pipeline.rs
index ce3fad251..4395babf1 100644
--- a/engine/crates/xerj-wasm/src/pipeline.rs
+++ b/engine/crates/xerj-wasm/src/pipeline.rs
@@ -14,7 +14,7 @@ use crate::{
AddFieldPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin, FieldRenamePlugin,
GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin, RemoveNullPlugin,
RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin, UppercasePlugin,
- UrlDecodePlugin,
+ UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,
},
Result, TransformPlugin, WasmError,
};
@@ -199,15 +199,7 @@ fn build_plugin(
"timestamp_parse" => {
let field = str_field(config, "field")?;
- let formats: Vec<String> = config
- .get("formats")
- .and_then(Value::as_array)
- .map(|arr| {
- arr.iter()
- .filter_map(|v| v.as_str().map(str::to_string))
- .collect()
- })
- .unwrap_or_default();
+ let formats = optional_string_array(config, "formats")?.unwrap_or_default();
let target = config
.get("target")
.and_then(Value::as_str)
@@ -260,24 +252,45 @@ fn build_plugin(
"grok" => {
let field = str_field(config, "field")?;
- let pattern_name = config
- .get("pattern")
- .and_then(Value::as_str)
- .unwrap_or("SYSLOG")
- .to_string();
+ let pattern_name = match config.get("pattern") {
+ None => "SYSLOG".to_string(),
+ Some(Value::String(s)) => {
+ // Fail closed: an unrecognised pattern name used to fall
+ // through to the generic catch-all, which extracts a
+ // single `message` field and nothing else (issue #204).
+ if !GROK_PATTERN_NAMES.contains(&s.as_str()) {
+ return Err(format!(
+ "unknown grok pattern '{s}' (known patterns: {})",
+ GROK_PATTERN_NAMES.join(", ")
+ ));
+ }
+ s.clone()
+ }
+ Some(other) => {
+ return Err(format!("'pattern' must be a string, got {other}"));
+ }
+ };
Ok(Arc::new(GrokPlugin::new(field, pattern_name)))
}
"pii_redaction" => {
- let types: Vec<String> = config
- .get("types")
- .and_then(Value::as_array)
- .map(|arr| {
- arr.iter()
- .filter_map(|v| v.as_str().map(str::to_string))
- .collect()
- })
- .unwrap_or_else(|| vec!["email".into(), "ip".into(), "credit_card".into()]);
+ // Fail closed on an unknown PII type: it used to contribute no
+ // regex, so a typo'd `types` list produced a stage that redacted
+ // nothing while reporting success (issue #204).
+ let types = match optional_string_array(config, "types")? {
+ None => vec!["email".into(), "ip".into(), "credit_card".into()],
+ Some(requested) => {
+ for t in &requested {
+ if !PII_TYPE_NAMES.contains(&t.as_str()) {
+ return Err(format!(
+ "unknown pii type '{t}' (known types: {})",
+ PII_TYPE_NAMES.join(", ")
+ ));
+ }
+ }
+ requested
+ }
+ };
Ok(Arc::new(PiiRedactionPlugin::new(types)))
}
@@ -290,6 +303,14 @@ fn build_plugin(
"convert" => {
let field = str_field(config, "field")?;
let target_type = str_field(config, "type")?;
+ // Fail closed: an unknown target type used to no-op per document
+ // rather than fail the pipeline definition (issue #204).
+ if !CONVERT_TARGET_TYPES.contains(&target_type.as_str()) {
+ return Err(format!(
+ "unknown convert target type '{target_type}' (known types: {})",
+ CONVERT_TARGET_TYPES.join(", ")
+ ));
+ }
Ok(Arc::new(ConvertTypePlugin::new(field, target_type)))
}
@@ -350,6 +371,33 @@ fn str_field(config: &Value, key: &str) -> std::result::Result<String, String> {
.ok_or_else(|| format!("missing required string field '{key}'"))
}
+/// Read an OPTIONAL array-of-strings config key.
+///
+/// `Ok(None)` means the key is absent (the caller applies its documented
+/// default). A present-but-wrong-shaped value is an error rather than a silent
+/// fall-through to the default — `"formats": "%Y-%m-%d"` used to be dropped by
+/// `and_then(Value::as_array)` and the stage then behaved as if no formats had
+/// been configured at all (issue #204).
+fn optional_string_array(
+ config: &Value,
+ key: &str,
+) -> std::result::Result<Option<Vec<String>>, String> {
+ let Some(raw) = config.get(key) else {
+ return Ok(None);
+ };
+ let arr = raw
+ .as_array()
+ .ok_or_else(|| format!("'{key}' must be an array of strings"))?;
+ let mut out = Vec::with_capacity(arr.len());
+ for v in arr {
+ let s = v
+ .as_str()
+ .ok_or_else(|| format!("'{key}' must contain only strings, got {v}"))?;
+ out.push(s.to_string());
+ }
+ Ok(Some(out))
+}
+
fn string_array(config: &Value, key: &str) -> std::result::Result<Vec<String>, String> {
let arr = config
.get(key)
@@ -440,3 +488,126 @@ mod tests {
assert!(Pipeline::from_config("bad", &cfg).is_err());
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Issue #204 — weaker fallbacks on user-supplied processor configuration
+//
+// Each case below used to build a pipeline that was accepted, stored, and
+// reported healthy, and then did LESS than the caller asked for with no signal
+// at all. They now fail closed at `Pipeline::from_config`, which is where the
+// caller can still act on the error.
+// ─────────────────────────────────────────────────────────────────────────────
+#[cfg(test)]
+mod fallback_regression_tests {
+ use super::*;
+
+ fn build(stage_type: &str, config: Value) -> std::result::Result<Pipeline, WasmError> {
+ Pipeline::from_config(
+ "t",
+ &PipelineConfig {
+ description: String::new(),
+ stages: vec![PipelineStageConfig {
+ stage_type: stage_type.into(),
+ config,
+ }],
+ on_error: ErrorPolicy::Drop,
+ timeout_ms: 0,
+ },
+ )
+ }
+
+ #[test]
+ fn unknown_pii_type_is_rejected_not_silently_unredacted() {
+ // Pre-fix: `social_security` matched no `wants(..)` arm, the pattern
+ // list came out EMPTY, and the stage passed every document through
+ // with its PII intact while reporting success.
+ let err = build(
+ "pii_redaction",
+ serde_json::json!({ "types": ["ssn", "social_security"] }),
+ )
+ .expect_err("unknown pii type must fail the pipeline definition");
+ assert!(
+ err.to_string().contains("social_security"),
+ "error must name the offending type: {err}"
+ );
+
+ // The supported spelling still builds and still redacts.
+ let pl = build("pii_redaction", serde_json::json!({ "types": ["ssn"] }))
+ .expect("known pii type builds");
+ let mut doc = serde_json::json!({ "note": "ssn 123-45-6789" });
+ pl.process(&mut doc);
+ assert_eq!(doc["note"], "ssn [REDACTED_SSN]");
+ }
+
+ #[test]
+ fn non_array_pii_types_is_rejected_not_defaulted() {
+ // Pre-fix: `and_then(Value::as_array)` returned None for a bare string
+ // and the stage silently fell back to the built-in default type list.
+ assert!(build("pii_redaction", serde_json::json!({ "types": "ssn" })).is_err());
+ }
+
+ #[test]
+ fn absent_pii_types_still_uses_the_documented_default() {
+ let pl = build("pii_redaction", serde_json::json!({})).expect("default types build");
+ let mut doc = serde_json::json!({ "note": "mail me at [email protected]" });
+ pl.process(&mut doc);
+ assert_eq!(doc["note"], "mail me at [REDACTED_EMAIL]");
+ }
+
+ #[test]
+ fn unknown_grok_pattern_is_rejected_not_downgraded_to_catch_all() {
+ // Pre-fix: this compiled to `^(?P<message>.+)$`, so an nginx pipeline
+ // extracted one field instead of ten and nothing said so.
+ let err = build(
+ "grok",
+ serde_json::json!({ "field": "msg", "pattern": "NGINX_COMBINE" }),
+ )
+ .expect_err("typo'd grok pattern must fail");
+ assert!(
+ err.to_string().contains("NGINX_COMBINE"),
+ "error must name the offending pattern: {err}"
+ );
+
+ // The catch-all is still reachable — by asking for it.
+ let pl = build(
+ "grok",
+ serde_json::json!({ "field": "msg", "pattern": "GENERIC" }),
+ )
+ .expect("GENERIC is an explicit, supported pattern");
+ let mut doc = serde_json::json!({ "msg": "anything at all" });
+ pl.process(&mut doc);
+ assert_eq!(doc["message"], "anything at all");
+ }
+
+ #[test]
+ fn unknown_convert_target_type_is_rejected_not_a_per_document_noop() {
+ // Pre-fix: `"int"` built fine and then hit `_ => None` on every
+ // document, leaving the field a string forever.
+ let err = build(
+ "convert",
+ serde_json::json!({ "field": "status", "type": "int" }),
+ )
+ .expect_err("unknown convert target type must fail");
+ assert!(err.to_string().contains("int"), "{err}");
+
+ let pl = build(
+ "convert",
+ serde_json::json!({ "field": "status", "type": "integer" }),
+ )
+ .expect("known target type builds");
+ let mut doc = serde_json::json!({ "status": "404" });
+ pl.process(&mut doc);
+ assert_eq!(doc["status"], 404);
+ }
+
+ #[test]
+ fn non_array_timestamp_formats_is_rejected_not_dropped() {
+ assert!(build(
+ "timestamp_parse",
+ serde_json::json!({ "field": "ts", "formats": "%Y-%m-%d" })
+ )
+ .is_err());
+ // Absent `formats` is still the documented epoch/RFC-3339 default.
+ assert!(build("timestamp_parse", serde_json::json!({ "field": "ts" })).is_ok());
+ }
+}