AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine,api): fail closed on user configuration xerj cannot honour
xerj-org2b50d533e900
diff --git a/engine/crates/xerj-api/src/authz.rs b/engine/crates/xerj-api/src/authz.rsindex b6a3df603..bc5358067 100644--- a/engine/crates/xerj-api/src/authz.rs+++ b/engine/crates/xerj-api/src/authz.rs@@ -1477,7 +1477,23 @@ async fn prune_response(match serde_json::from_slice::<Value>(&bytes) {Ok(mut v) => {prune_json(&mut v, sites, principal, &known);- serde_json::to_vec(&v).unwrap_or_else(|_| bytes.to_vec())+ match serde_json::to_vec(&v) {+ Ok(v) => v,+ // Issue #204: this was+ // `.unwrap_or_else(|_| bytes.to_vec())` — a failure to+ // re-serialise the PRUNED document fell back to the+ // ORIGINAL bytes, i.e. the scoped principal received the+ // unfiltered listing this function exists to withhold.+ // Refuse, exactly as the buffering failure above does: a+ // filter that cannot run must not fail open.+ Err(e) => {+ tracing::error!(error = %e, "could not re-serialise pruned response");+ return es_error(+ StatusCode::INTERNAL_SERVER_ERROR,+ "response could not be authorized".to_string(),+ );+ }+ }}// Not JSON after all (some `_cat` handlers answer text under a// JSON content type); fall through to the line filter.diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 91c93f51d..95a9801a0 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -13412,6 +13412,128 @@ mod malformed_bulk_route_tests {}}+/// Issue #204 — an ingest pipeline that cannot be compiled must be REFUSED,+/// not stored, acknowledged and quietly skipped.+#[cfg(test)]+mod ingest_pipeline_fail_closed_tests {+ use super::*;+ use axum::{+ body::{to_bytes, Body},+ http::{Request, StatusCode},+ };+ use tower::ServiceExt;+ use xerj_common::{+ config::{Config, WalSync},+ metrics::Metrics,+ };+ use xerj_engine::Engine;++ fn test_state() -> AppState {+ let dir = tempfile::tempdir().expect("tempdir").keep();+ let mut config = Config::default();+ config.server.data_dir = dir.to_string_lossy().into_owned();+ config.storage.wal_sync = WalSync::Async;+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ async fn put_pipeline(state: AppState, id: &str, body: Value) -> (StatusCode, Value) {+ let response = crate::router::build_es_compat_router(state)+ .oneshot(+ Request::put(format!("/_ingest/pipeline/{id}"))+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))+ .expect("request"),+ )+ .await+ .expect("route response");+ let status = response.status();+ let bytes = to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("response bytes");+ (+ status,+ serde_json::from_slice(&bytes).unwrap_or(Value::Null),+ )+ }++ #[tokio::test]+ async fn an_uncompilable_pipeline_is_rejected_and_not_stored() {+ let state = test_state();++ // `social_security` is not a PII type xerj can redact. Pre-fix this+ // answered 200/acknowledged, `GET` echoed it back, and every document+ // routed through it kept its PII.+ let (status, body) = put_pipeline(+ state.clone(),+ "redact",+ json!({+ "processors": [+ { "pii_redaction": { "types": ["social_security"] } }+ ]+ }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "body: {body}");++ // Nothing was stored: GET must 404 rather than echo a pipeline that+ // does not exist in compiled form.+ let response = crate::router::build_es_compat_router(state.clone())+ .oneshot(+ Request::get("/_ingest/pipeline/redact")+ .body(Body::empty())+ .expect("request"),+ )+ .await+ .expect("route response");+ assert_eq!(response.status(), StatusCode::NOT_FOUND);+ assert!(!state.engine.pipelines.contains_key("redact"));+ assert!(!state.engine.transform_pipelines.contains_key("redact"));+ }++ #[tokio::test]+ async fn a_processor_entry_that_is_not_a_processor_is_rejected() {+ // Pre-fix `filter_map` dropped this silently and the pipeline compiled+ // with zero stages.+ let (status, body) = put_pipeline(+ test_state(),+ "broken",+ json!({ "processors": [ "lowercase" ] }),+ )+ .await;+ assert_eq!(status, StatusCode::BAD_REQUEST, "body: {body}");+ }++ #[tokio::test]+ async fn a_compilable_pipeline_still_round_trips() {+ let state = test_state();+ let (status, _) = put_pipeline(+ state.clone(),+ "ok",+ json!({+ "processors": [+ { "pii_redaction": { "types": ["email"] } },+ { "set": { "field": "env", "value": "prod" } }+ ]+ }),+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert!(state.engine.transform_pipelines.contains_key("ok"));++ let response = crate::router::build_es_compat_router(state)+ .oneshot(+ Request::get("/_ingest/pipeline/ok")+ .body(Body::empty())+ .expect("request"),+ )+ .await+ .expect("route response");+ assert_eq!(response.status(), StatusCode::OK);+ }+}+#[cfg(test)]mod script_query_end_to_end_tests {//! Regression coverage for a real bug: the `script` query's unit tests@@ -21389,12 +21511,17 @@ pub async fn put_ingest_pipeline(Path(id): Path<String>,Json(body): Json<Value>,) -> impl IntoResponse {- // Store the raw JSON for GET /_ingest/pipeline.- state.engine.pipelines.insert(id.clone(), body.clone());// Convert ES processor format → xerj stage format, then compile.// ES: {"processors": [{"set": {"field":"x","value":"y"}}]}// xerj: {"stages": [{"type": "set", "config": {"field":"x","value":"y"}}]}//+ // Issue #204 — the raw body used to be stored FIRST and the compile+ // failure only warn-logged, so a pipeline with an unsupported processor+ // was answered `200 {"acknowledged": true}`, echoed back verbatim by+ // `GET /_ingest/pipeline/{id}`, and never ran: the redaction / grok / date+ // parsing the caller asked for silently did not happen. Compile first,+ // store only on success, and answer 400 like ES does.+ //// ES → xerj name mapping for processors whose names differ:fn map_proc_name(es_name: &str) -> String {match es_name {@@ -21408,23 +21535,31 @@ pub async fn put_ingest_pipeline(other => other.to_string(),}}- let xerj_config =- if let Some(processors) = body.get("processors").and_then(Value::as_array) {- let stages: Vec<Value> = processors.iter().filter_map(|proc| {- let obj = proc.as_object()?;- let (proc_type, proc_config) = obj.iter().next()?;+ let xerj_config = if let Some(processors) = body.get("processors").and_then(Value::as_array) {+ let mut stages: Vec<Value> = Vec::with_capacity(processors.len());+ for (i, proc) in processors.iter().enumerate() {+ // A processor entry that is not a single-key object used to be+ // dropped by `filter_map` — the pipeline then compiled with fewer+ // stages than were written, and still acknowledged.+ let Some((proc_type, proc_config)) = proc.as_object().and_then(|o| o.iter().next())+ else {+ let e = xerj_common::XerjError::invalid_mapping(format!(+ "pipeline [{id}] processor[{i}] is not a processor object \+ (expected {{\"<processor>\": {{...}}}})"+ ));+ return ApiError::new(e).into_response();+ };let xerj_type = map_proc_name(proc_type.as_str());// Adapt ES config shapes to xerj config shapes where they differ.let adapted_config = match proc_type.as_str() {"remove" => {// ES: {"field": "x"} or {"field": ["x","y"]}// xerj drop_field: {"fields": ["x","y"]}- let fields = match proc_config.get("field") {+ match proc_config.get("field") {Some(Value::String(s)) => json!({"fields": [s]}),Some(Value::Array(a)) => json!({"fields": a}),_ => proc_config.clone(),- };- fields+ }}"rename" => {// ES: {"field": "old", "target_field": "new"}@@ -21436,17 +21571,26 @@ pub async fn put_ingest_pipeline(}_ => proc_config.clone(),};- Some(json!({"type": xerj_type, "config": adapted_config}))- }).collect();- json!({- "description": body.get("description").and_then(Value::as_str).unwrap_or(""),- "stages": stages- })- } else {- body.clone()- };+ stages.push(json!({"type": xerj_type, "config": adapted_config}));+ }+ json!({+ "description": body.get("description").and_then(Value::as_str).unwrap_or(""),+ "stages": stages+ })+ } else {+ body.clone()+ };+ // `create_pipeline` publishes into BOTH the compiled map and the+ // GET-visible `pipelines` map, and only on success — so a rejected+ // definition leaves no phantom entry to echo back. (On success the stored+ // blob is the translated xerj config, exactly as before: `create_pipeline`+ // used to overwrite the raw body this handler had already inserted.)if let Err(e) = state.engine.create_pipeline(&id, xerj_config) {- tracing::warn!(pipeline = %id, error = %e, "pipeline stored but failed to compile");+ tracing::warn!(pipeline = %id, error = %e, "rejected ingest pipeline definition");+ let err = xerj_common::XerjError::invalid_mapping(format!(+ "pipeline [{id}] cannot be created: {e}"+ ));+ return ApiError::new(err).into_response();}Json(json!({ "acknowledged": true })).into_response()}diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rsindex 824dc13b3..199f2f062 100644--- a/engine/crates/xerj-api/src/native.rs+++ b/engine/crates/xerj-api/src/native.rs@@ -821,11 +821,29 @@ pub async fn admin_backup(State(state): State<AppState>, body: Bytes) -> impl Inlet started = Instant::now();let request_id = Uuid::new_v4().to_string();- // Body is optional and lenient — `{}` or empty both mean "defaults".+ // Body is optional — empty and `{}` both mean "defaults". A body that is+ // PRESENT but unparseable is not: issue #204, this was+ // `.unwrap_or_default()`, so a malformed request silently discarded the+ // caller's `repo_path` / `name` / `indices` and ran a *different* backup+ // (default location, every index) under a `201 Created`. An operator+ // reading "created" had no way to know the archive they asked for is not+ // the archive that exists.let req: BackupRequest = if body.is_empty() {BackupRequest::default()} else {- serde_json::from_slice(&body).unwrap_or_default()+ match serde_json::from_slice(&body) {+ Ok(r) => r,+ Err(e) => {+ return native_error(+ xerj_common::XerjError::invalid_document_json(format!(+ "backup request body is not valid BackupRequest JSON: {e}"+ )),+ Some(&request_id),+ started.elapsed().as_millis() as u64,+ )+ .into_response();+ }+ }};let data_dir = state.engine.config().server.data_dir.clone();diff --git a/engine/crates/xerj-api/src/state.rs b/engine/crates/xerj-api/src/state.rsindex 34d7534bb..4491bdb0d 100644--- a/engine/crates/xerj-api/src/state.rs+++ b/engine/crates/xerj-api/src/state.rs@@ -304,30 +304,63 @@ impl MlDetector {}/// Load all persisted detectors from `<data_dir>/_ml/detectors.json`.- /// Missing/corrupt files yield an empty registry (best effort).+ ///+ /// An ABSENT file is a fresh node and yields an empty registry silently. A+ /// file that is present but unreadable or unparseable also yields an empty+ /// registry — every detector the operator configured disappears — so it is+ /// logged at ERROR rather than swallowed (issue #204). Boot is not failed+ /// on it: a node that refuses to start is worse than one that starts with+ /// its ML jobs missing and says so.pub fn load_all(data_dir: &str) -> DashMap<String, MlDetector> {let map = DashMap::new();let path = Self::registry_path(data_dir);- if let Ok(bytes) = std::fs::read(&path) {- if let Ok(list) = serde_json::from_slice::<Vec<MlDetector>>(&bytes) {- for d in list {- map.insert(d.detector_id.clone(), d);+ match std::fs::read(&path) {+ Ok(bytes) => match serde_json::from_slice::<Vec<MlDetector>>(&bytes) {+ Ok(list) => {+ for d in list {+ map.insert(d.detector_id.clone(), d);+ }}- }+ Err(e) => tracing::error!(+ path = %path.display(), error = %e,+ "ml detector registry is unparseable — ALL configured detectors are \+ being dropped; move the file aside to stop it being overwritten"+ ),+ },+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => tracing::error!(+ path = %path.display(), error = %e,+ "ml detector registry could not be read — ALL configured detectors are \+ being dropped"+ ),}map}- /// Persist the full registry to disk (best effort — swallows I/O errors).+ /// Persist the full registry to disk.+ ///+ /// Issue #204: the write used to be `let _ = ..`, so a detector accepted+ /// with a 200 could fail to reach disk and vanish at the next restart with+ /// no signal anywhere. Still best-effort — the caller has already+ /// acknowledged — but no longer silent.pub fn save_all(data_dir: &str, registry: &DashMap<String, MlDetector>) {let path = Self::registry_path(data_dir);- if let Some(parent) = path.parent() {- let _ = std::fs::create_dir_all(parent);- }let list: Vec<MlDetector> = registry.iter().map(|e| e.value().clone()).collect();- if let Ok(bytes) = serde_json::to_vec_pretty(&list) {- let _ = std::fs::write(&path, bytes);+ if let Err(e) = Self::write_registry(&path, &list) {+ tracing::error!(+ path = %path.display(), error = %e,+ "failed to persist ml detector registry — detectors are live in memory \+ but WILL BE LOST on restart"+ );+ }+ }++ fn write_registry(path: &std::path::Path, list: &[MlDetector]) -> std::io::Result<()> {+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent)?;}+ let bytes = serde_json::to_vec_pretty(list)?;+ std::fs::write(path, bytes)}}@@ -396,31 +429,58 @@ impl MlDatafeed {/// Load all persisted datafeeds. Runtime state (`state`, cursor) is reset:/// background scoring tasks and the results store are in-memory only and do/// not survive a restart, so a freshly-loaded datafeed is always `stopped`.+ ///+ /// A present-but-unreadable registry drops every configured datafeed; that+ /// is logged at ERROR rather than swallowed (issue #204). An absent file is+ /// a fresh node and stays quiet.pub fn load_all(data_dir: &str) -> DashMap<String, MlDatafeed> {let map = DashMap::new();let path = Self::registry_path(data_dir);- if let Ok(bytes) = std::fs::read(&path) {- if let Ok(list) = serde_json::from_slice::<Vec<MlDatafeed>>(&bytes) {- for mut d in list {- d.state = "stopped".to_string();- d.last_scored_ms = None;- map.insert(d.datafeed_id.clone(), d);+ match std::fs::read(&path) {+ Ok(bytes) => match serde_json::from_slice::<Vec<MlDatafeed>>(&bytes) {+ Ok(list) => {+ for mut d in list {+ d.state = "stopped".to_string();+ d.last_scored_ms = None;+ map.insert(d.datafeed_id.clone(), d);+ }}- }+ Err(e) => tracing::error!(+ path = %path.display(), error = %e,+ "ml datafeed registry is unparseable — ALL configured datafeeds are \+ being dropped; move the file aside to stop it being overwritten"+ ),+ },+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}+ Err(e) => tracing::error!(+ path = %path.display(), error = %e,+ "ml datafeed registry could not be read — ALL configured datafeeds are \+ being dropped"+ ),}map}- /// Persist the full registry to disk (best effort — swallows I/O errors).+ /// Persist the full registry to disk. Best-effort but not silent — see+ /// [`MlDetector::save_all`] (issue #204).pub fn save_all(data_dir: &str, registry: &DashMap<String, MlDatafeed>) {let path = Self::registry_path(data_dir);- if let Some(parent) = path.parent() {- let _ = std::fs::create_dir_all(parent);- }let list: Vec<MlDatafeed> = registry.iter().map(|e| e.value().clone()).collect();- if let Ok(bytes) = serde_json::to_vec_pretty(&list) {- let _ = std::fs::write(&path, bytes);+ if let Err(e) = Self::write_registry(&path, &list) {+ tracing::error!(+ path = %path.display(), error = %e,+ "failed to persist ml datafeed registry — datafeeds are live in memory \+ but WILL BE LOST on restart"+ );+ }+ }++ fn write_registry(path: &std::path::Path, list: &[MlDatafeed]) -> std::io::Result<()> {+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent)?;}+ let bytes = serde_json::to_vec_pretty(list)?;+ std::fs::write(path, bytes)}}diff --git a/engine/crates/xerj-console-api/src/auth/store.rs b/engine/crates/xerj-console-api/src/auth/store.rsindex cfbfec380..2ad73c3af 100644--- a/engine/crates/xerj-console-api/src/auth/store.rs+++ b/engine/crates/xerj-console-api/src/auth/store.rs@@ -209,7 +209,13 @@ pub async fn count_passkeys_for_user(engine: &Engine, user_id: &str) -> ConsoleRpub async fn delete_passkey(engine: &Engine, credential_id: &str) -> ConsoleResult<()> {let idx = engine.get_index(indices::PASSKEYS)?;- let _ = idx.delete_document(credential_id).await;+ // Issue #204: this was `let _ = ..`, so a delete refused by the engine+ // (index write block, disk flood stage, storage error) still produced+ // `204 No Content` — the console said the passkey was revoked while it+ // remained on disk and kept authenticating. `delete_document` returns+ // `Ok(false)` for an already-absent credential, so propagating cannot turn+ // a repeat revoke into an error; only a genuine failure surfaces.+ idx.delete_document(credential_id).await?;Ok(())}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 66a2a0ccf..71cd398a0 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -5223,6 +5223,9 @@ pub struct Index {/// because we only ever take/replace under it, never hold across/// any awaits.pub(crate) merge_task: Arc<parking_lot::Mutex<Option<tokio::task::JoinHandle<()>>>>,+ /// Failed writes of `schema.json` during dynamic-mapping evolution.+ /// See [`Index::persist_evolved_schema`] (issue #204).+ schema_persist_failures: AtomicU64,}impl Index {@@ -5341,6 +5344,25 @@ impl Index {config: &Config,data_dir: &Path,) -> Result<Arc<Self>> {+ // Issue #204 — refuse an analysis block we would only pretend to+ // honour. `apply_settings` is total (it has to be: it also runs on the+ // open path), so an unresolvable tokenizer silently became `standard`+ // and an unresolvable filter was silently dropped — the index was then+ // built with an analyzer the caller never asked for and nothing in the+ // 200 response said so. Checked BEFORE the index directory is created+ // so a rejected create leaves nothing behind.+ let analysis_problems = AnalyzerRegistry::unsupported_analysis(&settings);+ if !analysis_problems.is_empty() {+ return Err(EngineError::Common(xerj_common::XerjError::config(+ format!(+ "index [{}] cannot be created: xerj cannot honour this analysis \+ configuration — {}",+ name.as_str(),+ analysis_problems.join("; ")+ ),+ )));+ }+let index_dir = data_dir.join(name.as_str());// Defense-in-depth: IndexName::validate rejects '.', '..', separators,// and '..' substrings, so join cannot escape data_dir. The debug_assert@@ -5544,6 +5566,7 @@ impl Index {flush_signal: Arc::new(SyncFlushCoord::new()),external_versions: Arc::new(dashmap::DashMap::new()),merge_task: Arc::new(parking_lot::Mutex::new(None)),+ schema_persist_failures: AtomicU64::new(0),});// Kick off the background merge pass. 5 s cadence is aggressive// enough to collapse a burst of flushes quickly without burning a@@ -5891,6 +5914,7 @@ impl Index {flush_signal: Arc::new(SyncFlushCoord::new()),external_versions: Arc::new(dashmap::DashMap::new()),merge_task: Arc::new(parking_lot::Mutex::new(None)),+ schema_persist_failures: AtomicU64::new(0),});index.spawn_merge_task(5);// RC4 W2 item 17: a flush-time-stale HNSW snapshot used to stay@@ -17642,7 +17666,7 @@ impl Index {}}if schema_changed {- let _ = self.save_schema(&schema).await;+ self.persist_evolved_schema(&schema).await;}}@@ -17716,10 +17740,45 @@ impl Index {}}if schema_changed {- let _ = self.save_schema(&schema).await;+ self.persist_evolved_schema(&schema).await;+ }+ }++ /// Persist a dynamic-mapping evolution, complaining loudly if it does not+ /// stick.+ ///+ /// Issue #204: both callers used to be `let _ = self.save_schema(..)`. The+ /// in-memory schema had the new fields, `schema.json` did not, and the next+ /// restart reopened the index with a mapping that had silently rolled back+ /// — documents already on disk carrying those fields, no mapping for them,+ /// and not one line anywhere saying why.+ ///+ /// This is a log-and-surface fix, not a fail-closed one, and deliberately+ /// so: schema evolution runs *after* the document has been admitted, so+ /// there is no caller left to fail. Losing the durable mapping is the lesser+ /// evil against rejecting a document that is already indexed — but it is+ /// an ERROR, not a shrug. `schema_persist_failures` counts it so a health+ /// check can see it without reading logs.+ async fn persist_evolved_schema(&self, schema: &ManagedSchema) {+ if let Err(e) = self.save_schema(schema).await {+ self.schema_persist_failures.fetch_add(1, Ordering::Relaxed);+ tracing::error!(+ index = self.name.as_str(),+ path = %self.data_dir.join("schema.json").display(),+ error = %e,+ "failed to persist dynamic mapping evolution — the new fields are \+ live in memory but WILL BE LOST on restart"+ );}}+ /// Number of times [`Self::persist_evolved_schema`] could not write+ /// `schema.json`. Non-zero means the on-disk mapping is behind the+ /// in-memory one and a restart will lose dynamically-mapped fields.+ pub fn schema_persist_failures(&self) -> u64 {+ self.schema_persist_failures.load(Ordering::Relaxed)+ }+async fn save_schema(&self, schema: &ManagedSchema) -> Result<()> {let path = self.data_dir.join("schema.json");let bytes = serde_json::to_vec_pretty(schema)?;diff --git a/engine/crates/xerj-engine/tests/integration.rs b/engine/crates/xerj-engine/tests/integration.rsindex 208918097..6654e2b49 100644--- a/engine/crates/xerj-engine/tests/integration.rs+++ b/engine/crates/xerj-engine/tests/integration.rs@@ -6471,3 +6471,116 @@ async fn test_bare_count_bucket_script_inside_a_terms_bucket_agrees_on_both_path}}}++// ─────────────────────────────────────────────────────────────────────────────+// Issue #204 — an analysis block xerj cannot honour must be REFUSED at create+// time, not accepted and quietly replaced with something weaker.+// ─────────────────────────────────────────────────────────────────────────────++#[tokio::test]+async fn create_index_rejects_an_analysis_block_it_cannot_honour() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ // Pre-fix: `autocomplete_tok` was never declared, `apply_settings`+ // substituted a StandardTokenizer, and the caller got a 200 plus an index+ // that does not do prefix matching at all.+ let settings = json!({+ "analysis": {+ "analyzer": {+ "default": {+ "type": "custom",+ "tokenizer": "autocomplete_tok",+ "filter": ["lowercase"]+ }+ }+ }+ });+ let err = engine+ .create_index_with_settings("bad_tok_idx", Schema::empty(), settings)+ .expect_err("an unresolvable tokenizer must fail index creation");+ let msg = err.to_string();+ assert!(msg.contains("autocomplete_tok"), "unhelpful error: {msg}");++ // Refused before anything hit the disk, and not registered in the engine.+ assert!(engine.get_index("bad_tok_idx").is_err());+ assert!(+ !dir.path().join("bad_tok_idx").exists(),+ "a rejected create must not leave an index directory behind"+ );++ // An unresolvable token filter is refused for the same reason: dropping+ // `lowercase` silently makes every match case-sensitive.+ let err = engine+ .create_index_with_settings(+ "bad_filter_idx",+ Schema::empty(),+ json!({+ "analysis": {+ "analyzer": {+ "default": {+ "type": "custom",+ "tokenizer": "standard",+ "filter": ["lowercse"]+ }+ }+ }+ }),+ )+ .expect_err("an unresolvable token filter must fail index creation");+ assert!(err.to_string().contains("lowercse"), "{err}");+}++#[tokio::test]+async fn create_index_still_accepts_every_analysis_construct_we_support() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ // The full supported surface in one settings block — this is the guard+ // against the #204 check over-rejecting.+ let settings = json!({+ "analysis": {+ "filter": {+ "my_synonyms": { "type": "synonym", "synonyms": ["fast,quick"] },+ "my_length": { "type": "length", "min": 2, "max": 40 },+ "my_shingle": { "type": "shingle", "max_shingle_size": 2 },+ "my_fold": { "type": "asciifolding" }+ },+ "tokenizer": {+ "ng": { "type": "ngram", "min_gram": 2, "max_gram": 3 },+ "edge": { "type": "edge_ngram", "min_gram": 1, "max_gram": 8 },+ "pat": { "type": "pattern", "pattern": "\\W+" }+ },+ "analyzer": {+ "default": {+ "type": "custom",+ "tokenizer": "edge",+ "filter": ["lowercase", "stop", "stemmer", "asciifolding",+ "my_synonyms", "my_length", "my_shingle", "my_fold"]+ },+ "plain": { "type": "english" }+ }+ }+ });+ engine+ .create_index_with_settings("good_idx", Schema::empty(), settings)+ .expect("a fully supported analysis block must still create");++ let idx = engine.get_index("good_idx").unwrap();+ idx.index_document(Some("1".into()), json!({ "title": "javascript" }))+ .await+ .unwrap();+ let result = idx+ .search(&make_search(json!({"match": {"title": "java"}})))+ .await+ .unwrap();+ assert_eq!(+ result.total.value, 1,+ "the edge-ngram analyzer must actually be in force"+ );++ // An index with no analysis block at all is untouched by the check.+ engine+ .create_index_with_settings("plain_idx", Schema::empty(), json!({}))+ .expect("empty settings must still create");+}diff --git a/engine/crates/xerj-fts/src/analyzer.rs b/engine/crates/xerj-fts/src/analyzer.rsindex 9924d9f9b..031250d70 100644--- a/engine/crates/xerj-fts/src/analyzer.rs+++ b/engine/crates/xerj-fts/src/analyzer.rs@@ -894,6 +894,15 @@ fn nfkc_normalize(s: &str) -> String {// ── Analyzer registry ─────────────────────────────────────────────────────────+/// `analysis.filter.*.type` values [`AnalyzerRegistry::apply_settings`] can+/// actually build. Must stay in step with the `match filter_type` arms there —+/// `supported_analysis_types_all_build` pins that.+pub const SUPPORTED_FILTER_TYPES: &[&str] = &["synonym", "length", "shingle", "asciifolding"];++/// `analysis.tokenizer.*.type` values [`AnalyzerRegistry::apply_settings`] can+/// actually build. Must stay in step with the `match tok_type` arms there.+pub const SUPPORTED_TOKENIZER_TYPES: &[&str] = &["ngram", "edge_ngram", "pattern"];+/// Central registry that maps analyzer names to their pipelines.////// Built-in analyzers are registered by default; custom analyzers can be@@ -1055,6 +1064,12 @@ impl AnalyzerRegistry {/// Extend this registry with custom analyzer definitions parsed from an/// index `settings.analysis` block.///+ /// **Total by design.** Every construct this method cannot honour is+ /// skipped or replaced rather than raised, because it also runs on the+ /// index-open path where a hard failure would make an existing index+ /// unopenable. Callers on the *create* path must gate on+ /// [`AnalyzerRegistry::unsupported_analysis`] first — see issue #204.+ ////// Accepts the ES-compatible JSON format:/// ```json/// {@@ -1076,7 +1091,7 @@ impl AnalyzerRegistry {/// }/// ```pub fn apply_settings(&mut self, settings: &serde_json::Value) {- let analysis = match settings.pointer("/analysis") {+ let analysis = match Self::analysis_block(settings) {Some(a) => a,None => return,};@@ -1215,8 +1230,16 @@ impl AnalyzerRegistry {if analyzer_type != "custom" {// For non-custom types, look up the built-in by type name.- if let Some(builtin) = self.get_analyzer(analyzer_type) {- self.analyzers.insert(analyzer_name.clone(), builtin);+ match self.get_analyzer(analyzer_type) {+ Some(builtin) => {+ self.analyzers.insert(analyzer_name.clone(), builtin);+ }+ None => tracing::warn!(+ analyzer = analyzer_name.as_str(),+ analyzer_type,+ "unknown analyzer type — analyzer not registered, \+ fields referencing it fall back to `standard`"+ ),}continue;}@@ -1231,7 +1254,22 @@ impl AnalyzerRegistry {.get(tokenizer_name).cloned().or_else(|| self.resolve_builtin_tokenizer(tokenizer_name))- .unwrap_or_else(|| Arc::new(StandardTokenizer));+ .unwrap_or_else(|| {+ // Not equivalent: substituting `standard` for (say)+ // `edge_ngram` builds a completely different index.+ // `unsupported_analysis` rejects this at index-create+ // time; reaching it here means we are re-opening an+ // index whose settings.json predates that check, so+ // say so loudly rather than degrade in silence+ // (issue #204).+ tracing::warn!(+ analyzer = analyzer_name.as_str(),+ tokenizer = tokenizer_name,+ "unknown tokenizer in custom analyzer — falling back to \+ `standard`; this index does NOT tokenize as configured"+ );+ Arc::new(StandardTokenizer)+ });// Resolve token filters.let filter_names: Vec<&str> = analyzer_def@@ -1263,6 +1301,155 @@ impl AnalyzerRegistry {}}+ /// Report every `settings.analysis` construct that [`apply_settings`] would+ /// accept and then silently NOT honour.+ ///+ /// [`apply_settings`]: AnalyzerRegistry::apply_settings+ ///+ /// Issue #204 — "degrade loudly or not at all". `apply_settings` is+ /// deliberately total: it never fails, because it also runs on the+ /// index-*open* path where refusing to build a registry would brick an+ /// existing index. That leniency is right at open time and wrong at create+ /// time, where every one of these constructs used to be accepted with a+ /// `200 {"acknowledged": true}` and then quietly replaced by something+ /// weaker:+ ///+ /// - an unresolvable `tokenizer` became `standard` — an `edge_ngram`+ /// autocomplete index that matches nothing;+ /// - an unresolvable `filter` was dropped — a missing `lowercase` turns+ /// every match case-sensitive;+ /// - an unsupported filter/tokenizer `type` was skipped entirely;+ /// - an unknown non-custom analyzer `type` left the analyzer unregistered,+ /// so fields naming it silently got `standard`.+ ///+ /// Returns one human-readable message per problem, empty when the block can+ /// be honoured exactly as written. `Index::create_with_settings` turns a+ /// non-empty result into a 400 so the caller learns at the door.+ ///+ /// `settings` may be either the full envelope (`{"settings": {"analysis":+ /// …}}`) or the inner settings object — the same two shapes+ /// `build_registry_from_settings` accepts.+ pub fn unsupported_analysis(settings: &serde_json::Value) -> Vec<String> {+ let root = settings.pointer("/settings").unwrap_or(settings);+ let Some(analysis) = Self::analysis_block(root) else {+ return Vec::new();+ };+ let probe = Self::with_defaults();+ let mut problems = Vec::new();++ // Declared names are collected even when their `type` is unsupported,+ // so a bad type is reported once (as a type problem) rather than twice+ // (again as a dangling reference from every analyzer that uses it).+ let mut declared_filters: HashSet<&str> = HashSet::new();+ if let Some(filter_map) = analysis.pointer("/filter").and_then(|v| v.as_object()) {+ for (filter_name, filter_def) in filter_map {+ declared_filters.insert(filter_name.as_str());+ let filter_type = filter_def+ .get("type")+ .and_then(|v| v.as_str())+ .unwrap_or("");+ if !SUPPORTED_FILTER_TYPES.contains(&filter_type) {+ problems.push(format!(+ "token filter [{filter_name}]: unsupported type [{filter_type}] \+ (supported: {})",+ SUPPORTED_FILTER_TYPES.join(", ")+ ));+ }+ }+ }++ let mut declared_tokenizers: HashSet<&str> = HashSet::new();+ if let Some(tok_map) = analysis.pointer("/tokenizer").and_then(|v| v.as_object()) {+ for (tok_name, tok_def) in tok_map {+ declared_tokenizers.insert(tok_name.as_str());+ let tok_type = tok_def.get("type").and_then(|v| v.as_str()).unwrap_or("");+ if !SUPPORTED_TOKENIZER_TYPES.contains(&tok_type) {+ problems.push(format!(+ "tokenizer [{tok_name}]: unsupported type [{tok_type}] \+ (supported: {})",+ SUPPORTED_TOKENIZER_TYPES.join(", ")+ ));+ continue;+ }+ if tok_type == "pattern" {+ let pattern = tok_def+ .get("pattern")+ .and_then(|v| v.as_str())+ .unwrap_or(r"\W+");+ if PatternTokenizer::new(pattern).is_err() {+ problems.push(format!(+ "tokenizer [{tok_name}]: invalid pattern regex [{pattern}]"+ ));+ }+ }+ }+ }++ if let Some(analyzer_map) = analysis.pointer("/analyzer").and_then(|v| v.as_object()) {+ for (analyzer_name, analyzer_def) in analyzer_map {+ let analyzer_type = analyzer_def+ .get("type")+ .and_then(|v| v.as_str())+ .unwrap_or("custom");++ if analyzer_type != "custom" {+ if probe.get_analyzer(analyzer_type).is_none() {+ problems.push(format!(+ "analyzer [{analyzer_name}]: unknown analyzer type [{analyzer_type}]"+ ));+ }+ continue;+ }++ let tokenizer_name = analyzer_def+ .get("tokenizer")+ .and_then(|v| v.as_str())+ .unwrap_or("standard");+ if !declared_tokenizers.contains(tokenizer_name)+ && probe.resolve_builtin_tokenizer(tokenizer_name).is_none()+ {+ problems.push(format!(+ "analyzer [{analyzer_name}]: unknown tokenizer [{tokenizer_name}]"+ ));+ }++ let filter_names: Vec<&str> = analyzer_def+ .get("filter")+ .and_then(|v| v.as_array())+ .map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())+ .unwrap_or_default();+ for fname in filter_names {+ if !declared_filters.contains(fname)+ && probe.resolve_builtin_filter(fname).is_none()+ {+ problems.push(format!(+ "analyzer [{analyzer_name}]: unknown token filter [{fname}]"+ ));+ }+ }+ }+ }++ problems.sort();+ problems+ }++ /// Locate the `analysis` block inside an index-settings object.+ ///+ /// ES accepts both the shorthand (`settings.analysis.*`) and the canonical+ /// namespaced form (`settings.index.analysis.*`). Only the shorthand used+ /// to be read here, so a settings body written the canonical way was+ /// accepted, echoed back by `GET /{index}/_settings`, and then contributed+ /// no analyzers at all — every field naming one silently got `standard`+ /// (issue #204). Kept as one helper so the builder and+ /// [`AnalyzerRegistry::unsupported_analysis`] can never disagree about+ /// which block is in force.+ fn analysis_block(settings: &serde_json::Value) -> Option<&serde_json::Value> {+ settings+ .pointer("/analysis")+ .or_else(|| settings.pointer("/index/analysis"))+ }+/// Resolve a tokenizer by its built-in name.fn resolve_builtin_tokenizer(&self, name: &str) -> Option<Arc<dyn Tokenizer>> {match name {@@ -1998,3 +2185,192 @@ mod tests {assert!(terms.contains(&"test".to_string()), "terms={terms:?}");}}++// ─────────────────────────────────────────────────────────────────────────────+// Issue #204 — `settings.analysis` constructs that used to degrade in silence+// ─────────────────────────────────────────────────────────────────────────────+#[cfg(test)]+mod unsupported_analysis_tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn a_settings_block_we_can_honour_reports_nothing() {+ let settings = json!({+ "analysis": {+ "filter": {+ "my_synonyms": { "type": "synonym", "synonyms": ["fast,quick"] },+ "my_length": { "type": "length", "min": 3, "max": 50 }+ },+ "tokenizer": {+ "autocomplete_tok": { "type": "edge_ngram", "min_gram": 1, "max_gram": 10 }+ },+ "analyzer": {+ "a": { "type": "custom", "tokenizer": "autocomplete_tok",+ "filter": ["lowercase", "my_synonyms", "my_length"] },+ "b": { "type": "english" }+ }+ }+ });+ assert!(+ AnalyzerRegistry::unsupported_analysis(&settings).is_empty(),+ "{:?}",+ AnalyzerRegistry::unsupported_analysis(&settings)+ );+ // Absent analysis block, and the outer-envelope shape, both work.+ assert!(AnalyzerRegistry::unsupported_analysis(&json!({})).is_empty());+ assert!(+ AnalyzerRegistry::unsupported_analysis(&json!({ "settings": settings })).is_empty()+ );+ }++ /// ES's canonical `index.analysis.*` form must be BUILT, not ignored — and+ /// therefore also validated.+ #[test]+ fn the_index_namespaced_form_is_honoured_by_builder_and_validator() {+ let namespaced = json!({+ "index": {+ "analysis": {+ "tokenizer": { "edge": { "type": "edge_ngram", "min_gram": 1, "max_gram": 8 } },+ "analyzer": { "ac": { "type": "custom", "tokenizer": "edge" } }+ }+ }+ });+ assert!(+ AnalyzerRegistry::unsupported_analysis(&namespaced).is_empty(),+ "{:?}",+ AnalyzerRegistry::unsupported_analysis(&namespaced)+ );++ let mut registry = AnalyzerRegistry::with_defaults();+ registry.apply_settings(&namespaced);+ let ac = registry+ .get_analyzer("ac")+ .expect("index.analysis.analyzer.ac must be registered, not silently dropped");+ assert!(+ ac.analyze_to_terms("java").contains(&"ja".to_string()),+ "the edge-ngram tokenizer must be the one in force"+ );++ // And a broken one in that form is caught rather than ignored.+ let bad = json!({ "index": { "analysis": { "analyzer": {+ "ac": { "type": "custom", "tokenizer": "nope" } } } } });+ assert_eq!(AnalyzerRegistry::unsupported_analysis(&bad).len(), 1);+ }++ #[test]+ fn unresolvable_tokenizer_is_reported() {+ // Pre-fix: this analyzer was registered with a StandardTokenizer and+ // the index tokenized nothing like the caller asked for.+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "analyzer": {+ "autocomplete": { "type": "custom", "tokenizer": "edge_ngram_tok" }+ }+ }+ }));+ assert_eq!(problems.len(), 1, "{problems:?}");+ assert!(problems[0].contains("edge_ngram_tok"), "{problems:?}");+ }++ #[test]+ fn unresolvable_filter_is_reported() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "analyzer": {+ "a": { "type": "custom", "tokenizer": "standard",+ "filter": ["lowercse", "word_delimiter"] }+ }+ }+ }));+ assert_eq!(problems.len(), 2, "{problems:?}");+ assert!(+ problems.iter().any(|p| p.contains("lowercse")),+ "{problems:?}"+ );+ assert!(+ problems.iter().any(|p| p.contains("word_delimiter")),+ "{problems:?}"+ );+ }++ #[test]+ fn unsupported_component_type_is_reported_once_not_twice() {+ // A filter declared with an unsupported `type` is skipped by+ // apply_settings, so every analyzer naming it also loses it. Report+ // the actionable cause (the type) and not the knock-on reference.+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": {+ "filter": { "stem": { "type": "hunspell", "locale": "en_US" } },+ "tokenizer": { "tok": { "type": "char_group" } },+ "analyzer": {+ "a": { "type": "custom", "tokenizer": "tok", "filter": ["stem"] }+ }+ }+ }));+ assert_eq!(problems.len(), 2, "{problems:?}");+ assert!(+ problems.iter().any(|p| p.contains("hunspell")),+ "{problems:?}"+ );+ assert!(+ problems.iter().any(|p| p.contains("char_group")),+ "{problems:?}"+ );+ }++ #[test]+ fn unknown_non_custom_analyzer_type_is_reported() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": { "analyzer": { "a": { "type": "kuromoji" } } }+ }));+ assert_eq!(problems.len(), 1, "{problems:?}");+ assert!(problems[0].contains("kuromoji"), "{problems:?}");+ }++ #[test]+ fn invalid_pattern_tokenizer_regex_is_reported() {+ let problems = AnalyzerRegistry::unsupported_analysis(&json!({+ "analysis": { "tokenizer": { "t": { "type": "pattern", "pattern": "[unclosed" } } }+ }));+ assert_eq!(problems.len(), 1, "{problems:?}");+ assert!(+ problems[0].contains("invalid pattern regex"),+ "{problems:?}"+ );+ }++ /// The allow-lists are the contract `unsupported_analysis` enforces; if one+ /// drifts from the `match` arms in `apply_settings` we would start+ /// rejecting configurations we can serve (or accepting ones we cannot).+ #[test]+ fn supported_analysis_types_all_build() {+ for t in SUPPORTED_FILTER_TYPES {+ let mut registry = AnalyzerRegistry::with_defaults();+ registry.apply_settings(&json!({+ "analysis": {+ "filter": { "f": { "type": t, "synonyms": ["a,b"] } },+ "analyzer": { "a": { "type": "custom", "tokenizer": "standard",+ "filter": ["f"] } }+ }+ }));+ assert!(+ registry.get_analyzer("a").is_some(),+ "filter type `{t}` is on the supported list but did not build"+ );+ }+ for t in SUPPORTED_TOKENIZER_TYPES {+ let mut registry = AnalyzerRegistry::with_defaults();+ registry.apply_settings(&json!({+ "analysis": {+ "tokenizer": { "tk": { "type": t } },+ "analyzer": { "a": { "type": "custom", "tokenizer": "tk" } }+ }+ }));+ assert!(+ registry.get_analyzer("a").is_some(),+ "tokenizer type `{t}` is on the supported list but did not build"+ );+ }+ }+}diff --git a/engine/crates/xerj-wasm/src/builtins.rs b/engine/crates/xerj-wasm/src/builtins.rsindex 5711d3c6f..5c40fdfc2 100644--- a/engine/crates/xerj-wasm/src/builtins.rs+++ b/engine/crates/xerj-wasm/src/builtins.rs@@ -420,6 +420,23 @@ impl TransformPlugin for GrokPlugin {}}+/// Built-in grok pattern names accepted by the `grok` stage.+///+/// `GENERIC` is the explicit opt-in for the catch-everything-as-`message`+/// pattern. It used to be reachable by *typo*: `grok_pattern` fell through to+/// it for any unrecognised name, so `"pattern": "NGINX_COMBINE"` compiled+/// happily and then extracted exactly one field (`message`) out of every+/// access log line — an accepted-and-ignored configuration (issue #204).+/// `build_plugin` now rejects names outside this list, so the generic pattern+/// is only ever used when it was actually asked for.+pub const GROK_PATTERN_NAMES: &[&str] = &[+ "NGINX_COMBINED",+ "APACHE_COMBINED",+ "SYSLOG",+ "POSTGRESQL",+ "GENERIC",+];+/// Returns `(regex_str, field_names)` for a built-in grok pattern.fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {match name {@@ -453,11 +470,9 @@ fn grok_pattern(name: &str) -> (&'static str, &'static [&'static str]) {"message",],),- _ => (- // Generic: capture everything as `message`.- r#"^(?P<message>.+)$"#,- &["message"],- ),+ // "GENERIC" (and, defensively, anything that slipped past+ // `build_plugin`'s allow-list): capture everything as `message`.+ _ => (r#"^(?P<message>.+)$"#, &["message"]),}}@@ -479,11 +494,22 @@ pub struct PiiRedactionPlugin {patterns: Vec<(Regex, &'static str)>,}+/// PII type names this plugin knows how to redact.+///+/// A name outside this list used to be accepted and then quietly matched+/// nothing: `PiiRedactionPlugin::new(vec!["social_security".into()])` compiled+/// to an EMPTY pattern set, so the stage ran on every document, redacted+/// nothing, and reported success — unredacted PII in the index with no signal+/// anywhere (issue #204). `build_plugin` now rejects unknown names at+/// pipeline-creation time.+pub const PII_TYPE_NAMES: &[&str] = &["email", "ip", "credit_card", "ssn", "phone"];+impl PiiRedactionPlugin {/// Build a new plugin that redacts the given PII types.///- /// `types` is a list of strings from `["email", "ip", "credit_card",- /// "ssn", "phone"]`. An empty list enables all types.+ /// `types` is a list of strings from [`PII_TYPE_NAMES`]. An empty list+ /// enables all types. Callers must validate the names first (see+ /// `build_plugin`) — an unknown name silently contributes no pattern.pub fn new(types: Vec<String>) -> Self {let all = types.is_empty();@@ -611,10 +637,23 @@ impl TransformPlugin for CopyFieldPlugin {// ConvertTypePlugin// ─────────────────────────────────────────────────────────────────────────────+/// Target types the `convert` stage can produce.+///+/// An unknown target type used to be accepted at pipeline-creation time and+/// then hit `_ => None` per document: the field was left exactly as it was and+/// the document passed. `"type": "int"` (instead of `"integer"`) therefore+/// produced a pipeline that acknowledged the conversion and never performed it+/// (issue #204). `build_plugin` now rejects unknown target types.+pub const CONVERT_TARGET_TYPES: &[&str] = &["integer", "float", "string", "boolean"];+/// Convert a field's value to a different type.///-/// Supported target types: `integer`, `float`, `string`, `boolean`.-/// If conversion fails the document is passed through unchanged.+/// Supported target types are [`CONVERT_TARGET_TYPES`]; an unknown one is+/// rejected when the pipeline is created. If the *value* cannot be coerced+/// (e.g. `"abc"` to `integer`) the document is passed through unchanged — a+/// per-document data condition, not a configuration error. (ES's `convert`+/// processor fails the document instead; that divergence is pre-existing and+/// out of scope for the #204 sweep.)pub struct ConvertTypePlugin {field: String,target_type: String,diff --git a/engine/crates/xerj-wasm/src/pipeline.rs b/engine/crates/xerj-wasm/src/pipeline.rsindex ce3fad251..4395babf1 100644--- a/engine/crates/xerj-wasm/src/pipeline.rs+++ b/engine/crates/xerj-wasm/src/pipeline.rs@@ -14,7 +14,7 @@ use crate::{AddFieldPlugin, ConvertTypePlugin, CopyFieldPlugin, DropFieldPlugin, FieldRenamePlugin,GrokPlugin, JsonParsePlugin, LowercasePlugin, PiiRedactionPlugin, RemoveNullPlugin,RoutePlugin, SetPlugin, SplitPlugin, TimestampParsePlugin, UppercasePlugin,- UrlDecodePlugin,+ UrlDecodePlugin, CONVERT_TARGET_TYPES, GROK_PATTERN_NAMES, PII_TYPE_NAMES,},Result, TransformPlugin, WasmError,};@@ -199,15 +199,7 @@ fn build_plugin("timestamp_parse" => {let field = str_field(config, "field")?;- let formats: Vec<String> = config- .get("formats")- .and_then(Value::as_array)- .map(|arr| {- arr.iter()- .filter_map(|v| v.as_str().map(str::to_string))- .collect()- })- .unwrap_or_default();+ let formats = optional_string_array(config, "formats")?.unwrap_or_default();let target = config.get("target").and_then(Value::as_str)@@ -260,24 +252,45 @@ fn build_plugin("grok" => {let field = str_field(config, "field")?;- let pattern_name = config- .get("pattern")- .and_then(Value::as_str)- .unwrap_or("SYSLOG")- .to_string();+ let pattern_name = match config.get("pattern") {+ None => "SYSLOG".to_string(),+ Some(Value::String(s)) => {+ // Fail closed: an unrecognised pattern name used to fall+ // through to the generic catch-all, which extracts a+ // single `message` field and nothing else (issue #204).+ if !GROK_PATTERN_NAMES.contains(&s.as_str()) {+ return Err(format!(+ "unknown grok pattern '{s}' (known patterns: {})",+ GROK_PATTERN_NAMES.join(", ")+ ));+ }+ s.clone()+ }+ Some(other) => {+ return Err(format!("'pattern' must be a string, got {other}"));+ }+ };Ok(Arc::new(GrokPlugin::new(field, pattern_name)))}"pii_redaction" => {- let types: Vec<String> = config- .get("types")- .and_then(Value::as_array)- .map(|arr| {- arr.iter()- .filter_map(|v| v.as_str().map(str::to_string))- .collect()- })- .unwrap_or_else(|| vec!["email".into(), "ip".into(), "credit_card".into()]);+ // Fail closed on an unknown PII type: it used to contribute no+ // regex, so a typo'd `types` list produced a stage that redacted+ // nothing while reporting success (issue #204).+ let types = match optional_string_array(config, "types")? {+ None => vec!["email".into(), "ip".into(), "credit_card".into()],+ Some(requested) => {+ for t in &requested {+ if !PII_TYPE_NAMES.contains(&t.as_str()) {+ return Err(format!(+ "unknown pii type '{t}' (known types: {})",+ PII_TYPE_NAMES.join(", ")+ ));+ }+ }+ requested+ }+ };Ok(Arc::new(PiiRedactionPlugin::new(types)))}@@ -290,6 +303,14 @@ fn build_plugin("convert" => {let field = str_field(config, "field")?;let target_type = str_field(config, "type")?;+ // Fail closed: an unknown target type used to no-op per document+ // rather than fail the pipeline definition (issue #204).+ if !CONVERT_TARGET_TYPES.contains(&target_type.as_str()) {+ return Err(format!(+ "unknown convert target type '{target_type}' (known types: {})",+ CONVERT_TARGET_TYPES.join(", ")+ ));+ }Ok(Arc::new(ConvertTypePlugin::new(field, target_type)))}@@ -350,6 +371,33 @@ fn str_field(config: &Value, key: &str) -> std::result::Result<String, String> {.ok_or_else(|| format!("missing required string field '{key}'"))}+/// Read an OPTIONAL array-of-strings config key.+///+/// `Ok(None)` means the key is absent (the caller applies its documented+/// default). A present-but-wrong-shaped value is an error rather than a silent+/// fall-through to the default — `"formats": "%Y-%m-%d"` used to be dropped by+/// `and_then(Value::as_array)` and the stage then behaved as if no formats had+/// been configured at all (issue #204).+fn optional_string_array(+ config: &Value,+ key: &str,+) -> std::result::Result<Option<Vec<String>>, String> {+ let Some(raw) = config.get(key) else {+ return Ok(None);+ };+ let arr = raw+ .as_array()+ .ok_or_else(|| format!("'{key}' must be an array of strings"))?;+ let mut out = Vec::with_capacity(arr.len());+ for v in arr {+ let s = v+ .as_str()+ .ok_or_else(|| format!("'{key}' must contain only strings, got {v}"))?;+ out.push(s.to_string());+ }+ Ok(Some(out))+}+fn string_array(config: &Value, key: &str) -> std::result::Result<Vec<String>, String> {let arr = config.get(key)@@ -440,3 +488,126 @@ mod tests {assert!(Pipeline::from_config("bad", &cfg).is_err());}}++// ─────────────────────────────────────────────────────────────────────────────+// Issue #204 — weaker fallbacks on user-supplied processor configuration+//+// Each case below used to build a pipeline that was accepted, stored, and+// reported healthy, and then did LESS than the caller asked for with no signal+// at all. They now fail closed at `Pipeline::from_config`, which is where the+// caller can still act on the error.+// ─────────────────────────────────────────────────────────────────────────────+#[cfg(test)]+mod fallback_regression_tests {+ use super::*;++ fn build(stage_type: &str, config: Value) -> std::result::Result<Pipeline, WasmError> {+ Pipeline::from_config(+ "t",+ &PipelineConfig {+ description: String::new(),+ stages: vec![PipelineStageConfig {+ stage_type: stage_type.into(),+ config,+ }],+ on_error: ErrorPolicy::Drop,+ timeout_ms: 0,+ },+ )+ }++ #[test]+ fn unknown_pii_type_is_rejected_not_silently_unredacted() {+ // Pre-fix: `social_security` matched no `wants(..)` arm, the pattern+ // list came out EMPTY, and the stage passed every document through+ // with its PII intact while reporting success.+ let err = build(+ "pii_redaction",+ serde_json::json!({ "types": ["ssn", "social_security"] }),+ )+ .expect_err("unknown pii type must fail the pipeline definition");+ assert!(+ err.to_string().contains("social_security"),+ "error must name the offending type: {err}"+ );++ // The supported spelling still builds and still redacts.+ let pl = build("pii_redaction", serde_json::json!({ "types": ["ssn"] }))+ .expect("known pii type builds");+ let mut doc = serde_json::json!({ "note": "ssn 123-45-6789" });+ pl.process(&mut doc);+ assert_eq!(doc["note"], "ssn [REDACTED_SSN]");+ }++ #[test]+ fn non_array_pii_types_is_rejected_not_defaulted() {+ // Pre-fix: `and_then(Value::as_array)` returned None for a bare string+ // and the stage silently fell back to the built-in default type list.+ assert!(build("pii_redaction", serde_json::json!({ "types": "ssn" })).is_err());+ }++ #[test]+ fn absent_pii_types_still_uses_the_documented_default() {+ let pl = build("pii_redaction", serde_json::json!({})).expect("default types build");+ let mut doc = serde_json::json!({ "note": "mail me at [email protected]" });+ pl.process(&mut doc);+ assert_eq!(doc["note"], "mail me at [REDACTED_EMAIL]");+ }++ #[test]+ fn unknown_grok_pattern_is_rejected_not_downgraded_to_catch_all() {+ // Pre-fix: this compiled to `^(?P<message>.+)$`, so an nginx pipeline+ // extracted one field instead of ten and nothing said so.+ let err = build(+ "grok",+ serde_json::json!({ "field": "msg", "pattern": "NGINX_COMBINE" }),+ )+ .expect_err("typo'd grok pattern must fail");+ assert!(+ err.to_string().contains("NGINX_COMBINE"),+ "error must name the offending pattern: {err}"+ );++ // The catch-all is still reachable — by asking for it.+ let pl = build(+ "grok",+ serde_json::json!({ "field": "msg", "pattern": "GENERIC" }),+ )+ .expect("GENERIC is an explicit, supported pattern");+ let mut doc = serde_json::json!({ "msg": "anything at all" });+ pl.process(&mut doc);+ assert_eq!(doc["message"], "anything at all");+ }++ #[test]+ fn unknown_convert_target_type_is_rejected_not_a_per_document_noop() {+ // Pre-fix: `"int"` built fine and then hit `_ => None` on every+ // document, leaving the field a string forever.+ let err = build(+ "convert",+ serde_json::json!({ "field": "status", "type": "int" }),+ )+ .expect_err("unknown convert target type must fail");+ assert!(err.to_string().contains("int"), "{err}");++ let pl = build(+ "convert",+ serde_json::json!({ "field": "status", "type": "integer" }),+ )+ .expect("known target type builds");+ let mut doc = serde_json::json!({ "status": "404" });+ pl.process(&mut doc);+ assert_eq!(doc["status"], 404);+ }++ #[test]+ fn non_array_timestamp_formats_is_rejected_not_dropped() {+ assert!(build(+ "timestamp_parse",+ serde_json::json!({ "field": "ts", "formats": "%Y-%m-%d" })+ )+ .is_err());+ // Absent `formats` is still the documented epoch/RFC-3339 default.+ assert!(build("timestamp_parse", serde_json::json!({ "field": "ts" })).is_ok());+ }+}