AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): honor date_detection:false + reject non-dates in inferred date fields
xerj-org4ed5540aedb3
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rsindex 5b1e9d229..bc6d7fdda 100644--- a/engine/crates/xerj-engine/src/bulk.rs+++ b/engine/crates/xerj-engine/src/bulk.rs@@ -1244,6 +1244,9 @@ pub async fn process_bulk_with_opts(block_type,..}) if block_type.contains("read_only_allow_delete") => 429,+ // Default-format date rejection ("failed to parse+ // field [..] of type [date]") — ES answers 400.+ EngineError::Common(xerj_common::XerjError::InvalidMapping { .. }) => 400,_ => 500,};items[item_idx] = Some(BulkItemResult {@@ -1694,6 +1697,8 @@ pub async fn process_bulk_with_opts(block_type,..}) if block_type.contains("read_only_allow_delete") => 429,+ // Default-format date rejection — ES answers 400.+ EngineError::Common(xerj_common::XerjError::InvalidMapping { .. }) => 400,_ => 500,};items[item_idx] = Some(BulkItemResult {diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 03d64ea14..28716fd16 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -444,6 +444,12 @@ impl Engine {// so GET /_mapping and mapping-dependent code paths see// the same mapping as pre-restart.engine.load_persisted_es_mapping(&name_str);+ // The index isn't registered yet, so the propagation+ // inside load can't find it — set the toggles on the+ // local handle instead.+ if let Some(m) = engine.index_mappings.get(name_str.as_str()) {+ Engine::apply_date_mapping_flags(&idx, m.value());+ }engine.indices.insert(name_str, idx);}Err(e) => {@@ -785,9 +791,72 @@ impl Engine {}}}+ self.propagate_date_detection(name, &mapping);self.index_mappings.insert(name.to_string(), mapping);}+ /// Push the mapping's `date_detection` toggle (default true) down to the+ /// open `Index` so dynamic inference honors it. The blob shape varies by+ /// caller (`{"date_detection": ..}` from PUT /_mapping, or nested under+ /// `"mappings"` from index-create), so both levels are checked.+ fn propagate_date_detection(&self, name: &str, mapping: &Value) {+ if let Ok(idx) = self.get_index(name) {+ Self::apply_date_mapping_flags(&idx, mapping);+ }+ }++ /// Push both date-related mapping toggles down to an open index:+ /// the `date_detection` bool and the set of date fields excluded from+ /// default-format ingest validation (explicit `format` — those are+ /// validated against their own format by the bulk path — or+ /// `ignore_malformed`).+ pub(crate) fn apply_date_mapping_flags(idx: &crate::index::Index, mapping: &Value) {+ idx.set_date_detection(Self::mapping_date_detection(mapping));+ idx.set_date_format_exclusions(Self::mapping_date_exclusions(mapping));+ }++ /// Read the `date_detection` toggle out of a raw mapping blob+ /// (defaulting to true, like ES).+ pub(crate) fn mapping_date_detection(mapping: &Value) -> bool {+ mapping+ .get("date_detection")+ .or_else(|| mapping.get("mappings").and_then(|m| m.get("date_detection")))+ .and_then(Value::as_bool)+ .unwrap_or(true)+ }++ /// Top-level date fields carrying an explicit `format` or+ /// `ignore_malformed` in the raw mapping blob (either shape).+ fn mapping_date_exclusions(mapping: &Value) -> std::collections::HashSet<String> {+ let props = mapping.get("properties").or_else(|| {+ mapping+ .get("mappings")+ .and_then(|m| m.get("properties"))+ });+ let mut out = std::collections::HashSet::new();+ let Some(obj) = props.and_then(Value::as_object) else {+ return out;+ };+ for (fname, spec) in obj {+ let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");+ if ftype != "date" && ftype != "date_nanos" {+ continue;+ }+ let has_format = spec+ .get("format")+ .and_then(Value::as_str)+ .is_some_and(|f| !f.is_empty());+ let ignore_malformed = spec+ .get("ignore_malformed")+ .and_then(Value::as_bool)+ .unwrap_or(false);+ if has_format || ignore_malformed {+ out.insert(fname.clone());+ }+ }+ out+ }+/// Load a previously-persisted raw ES mapping blob for `name` (if any)/// into the in-memory `index_mappings` map. Called whenever an index is/// (re)opened from disk — boot scan and snapshot restore. A missing@@ -800,6 +869,7 @@ impl Engine {};match serde_json::from_slice::<Value>(&bytes) {Ok(mapping) => {+ self.propagate_date_detection(name, &mapping);self.index_mappings.insert(name.to_string(), mapping);}Err(e) => {@@ -1586,6 +1656,9 @@ impl Engine {// Snapshot dirs carry es_mapping.json — reload it so the// restored index serves the same mapping it was saved with.self.load_persisted_es_mapping(idx_name);+ if let Some(m) = self.index_mappings.get(idx_name) {+ Engine::apply_date_mapping_flags(&idx, m.value());+ }self.indices.insert(idx_name.clone(), idx);info!(index = idx_name, "index restored from snapshot");}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 4592da405..b0ac9b28a 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2,7 +2,7 @@use std::collections::{HashMap, HashSet};use std::path::{Path, PathBuf};-use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};use std::sync::Arc;use regex::Regex;@@ -412,6 +412,22 @@ fn l2_normalize_vec(v: &mut [f32]) {pub struct Index {name: IndexName,schema: Arc<RwLock<ManagedSchema>>,+ /// ES `date_detection` mapping toggle (default true). Consulted by+ /// dynamic inference only — an explicit `date_detection: false` in the+ /// index mapping keeps first-seen ISO-looking strings mapped as `text`.+ /// Owned here (not in `ManagedSchema`) because the raw mapping blob is+ /// API-layer state (`engine.index_mappings`); the engine propagates the+ /// flag on create/open/put-mapping via [`Index::set_date_detection`].+ date_detection: AtomicBool,+ /// Fast bail for ingest date validation: true once the schema holds any+ /// `date`-typed field (set at construction and when dynamic inference+ /// adds one; never unset — a stale `true` only costs a schema read).+ has_date_fields: AtomicBool,+ /// Date fields whose mapping declares an explicit `format` (validated+ /// against that format by `bulk::find_bad_date_field`) or+ /// `ignore_malformed` — excluded from the engine's default-format+ /// ingest validation. Engine-propagated alongside `date_detection`.+ date_format_exclusions: std::sync::RwLock<Arc<std::collections::HashSet<String>>>,store: Arc<IndexStore>,/// 16-shard FTS memtable. Replaces the pre-v16 single/// `Arc<RwLock<FtsMemtable>>` that serialized all concurrent bulk@@ -896,9 +912,16 @@ impl Index {let registry = Arc::new(build_registry_from_settings(&settings));info!(name = name.as_str(), "index created");+ let has_dates = managed+ .fields()+ .iter()+ .any(|f| matches!(f.field_type, FieldType::Date));let index = Arc::new(Self {name,schema: Arc::new(RwLock::new(managed)),+ date_detection: AtomicBool::new(true),+ has_date_fields: AtomicBool::new(has_dates),+ date_format_exclusions: std::sync::RwLock::new(Arc::new(Default::default())),store,memtable: Arc::new(crate::memtable::ShardedFtsMemtable::with_registry_and_shards(@@ -1148,9 +1171,16 @@ impl Index {doc_count = total_doc_count,"index opened");+ let has_dates = schema+ .fields()+ .iter()+ .any(|f| matches!(f.field_type, FieldType::Date));let index = Arc::new(Self {name,schema: Arc::new(RwLock::new(schema)),+ date_detection: AtomicBool::new(true),+ has_date_fields: AtomicBool::new(has_dates),+ date_format_exclusions: std::sync::RwLock::new(Arc::new(Default::default())),max_concurrent_queries: Arc::new(Semaphore::new(64)),enrichments: Arc::new(RwLock::new(HashMap::new())),store,@@ -1311,6 +1341,9 @@ impl Index {"write",)));}+ // Default-format date validation — before any durable write so a+ // rejected doc leaves no WAL/memtable trace.+ self.validate_default_format_dates(&source).await?;// Process-wide admission: disk flood-stage block (item 3) + parent// memory circuit breaker (item 1). Fires before any per-index work.self.governor_write_gate()?;@@ -10476,6 +10509,75 @@ impl Index {// ── Helpers ───────────────────────────────────────────────────────────────+ /// Set the ES `date_detection` toggle for dynamic inference. Called by+ /// the engine whenever the index mapping is created, loaded from disk,+ /// or replaced via PUT /_mapping.+ pub fn set_date_detection(&self, on: bool) {+ self.date_detection.store(on, Ordering::Relaxed);+ }++ /// Current `date_detection` state (default true, like ES).+ pub fn date_detection_enabled(&self) -> bool {+ self.date_detection.load(Ordering::Relaxed)+ }++ /// Replace the set of date fields excluded from default-format ingest+ /// validation (fields with an explicit mapping `format` — validated+ /// separately by `bulk::find_bad_date_field` — or `ignore_malformed`).+ /// Engine-propagated together with [`Index::set_date_detection`].+ pub fn set_date_format_exclusions(&self, excluded: std::collections::HashSet<String>) {+ if let Ok(mut guard) = self.date_format_exclusions.write() {+ *guard = Arc::new(excluded);+ }+ }++ /// Reject documents whose value for a DEFAULT-format date field (most+ /// importantly a dynamically-inferred one) cannot possibly parse under+ /// ES's default `strict_date_optional_time||epoch_millis` — real ES+ /// answers 400 `mapper_parsing_exception`; xerj previously accepted the+ /// doc silently, leaving the value invisible to range queries, sorts,+ /// and Kibana time filters. Runs on the per-doc write funnel only; the+ /// raw-bytes turbo batch paths intentionally skip it (they never parse+ /// doc bodies — same envelope as the pre-existing explicit-format+ /// validation, which routes strict-format indexes off turbo entirely).+ async fn validate_default_format_dates(&self, source: &Value) -> Result<()> {+ if !self.has_date_fields.load(Ordering::Relaxed) {+ return Ok(());+ }+ let Some(obj) = source.as_object() else {+ return Ok(());+ };+ let exclusions = self+ .date_format_exclusions+ .read()+ .map(|g| Arc::clone(&g))+ .unwrap_or_default();+ let schema = self.schema.read().await;+ for (key, val) in obj {+ if val.is_null() {+ continue;+ }+ let Some(fc) = schema.field(key) else {+ continue;+ };+ if !matches!(fc.field_type, FieldType::Date) || exclusions.contains(key) {+ continue;+ }+ if !default_format_date_value_ok(val) {+ let preview: String = match val {+ Value::String(s) => s.chars().take(64).collect(),+ other => other.to_string().chars().take(64).collect(),+ };+ return Err(EngineError::Common(xerj_common::XerjError::InvalidMapping {+ reason: format!(+ "failed to parse field [{key}] of type [date] with value [{preview}]"+ ),+ }));+ }+ }+ Ok(())+ }+/// Batch variant of [`evolve_schema_from_doc`]: one schema read-lock/// acquisition for the WHOLE batch instead of one per doc. The per-doc/// variant paid a tokio `RwLock::read().await` per document — measured@@ -10493,6 +10595,7 @@ impl Index {if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {return;}+ let date_detection = self.date_detection_enabled();let mut out: Vec<(String, FieldConfig)> = Vec::new();for source in sources {let Some(obj) = source.as_object() else {@@ -10500,7 +10603,7 @@ impl Index {};for (key, val) in obj {if !schema.schema.has_field(key) && !out.iter().any(|(k, _)| k == key) {- let ft = infer_field_type(val);+ let ft = infer_field_type(val, date_detection);out.push((key.clone(), FieldConfig::new(key.clone(), ft)));}}@@ -10519,6 +10622,9 @@ impl Index {let mut schema_changed = false;for (_, fc) in new_fields {if !schema.schema.has_field(&fc.name) {+ if matches!(fc.field_type, FieldType::Date) {+ self.has_date_fields.store(true, Ordering::Relaxed);+ }let _ = schema.schema.add_field(fc);schema_changed = true;}@@ -10543,10 +10649,11 @@ impl Index {if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {return;}+ let date_detection = self.date_detection_enabled();obj.iter().filter(|(key, _)| !schema.schema.has_field(key)).map(|(key, val)| {- let ft = infer_field_type(val);+ let ft = infer_field_type(val, date_detection);(key.clone(), FieldConfig::new(key.clone(), ft))}).collect()@@ -10566,6 +10673,9 @@ impl Index {let mut schema_changed = false;for (_, fc) in new_fields {if !schema.schema.has_field(&fc.name) {+ if matches!(fc.field_type, FieldType::Date) {+ self.has_date_fields.store(true, Ordering::Relaxed);+ }let _ = schema.schema.add_field(fc);schema_changed = true;}@@ -18358,10 +18468,11 @@ fn apply_copy_to(source: &Value, schema: &Schema) -> Value {/// ES's default `date_detection`: a string field with no explicit mapping/// (from a template or the user) is inferred as `date` when its value fully-/// parses as a standard date/time format — real ES tries-/// `strict_date_optional_time` (an ISO-8601 date or date-time, optional-/// fractional seconds/offset) plus `epoch_millis` before giving up and-/// falling back to `text`. xerj had no such check at all: every string+/// parses as a standard date/time format — real ES tries its default+/// `dynamic_date_formats` (`strict_date_optional_time` — an ISO-8601 date or+/// date-time with optional fractional seconds/offset — plus the+/// `yyyy/MM/dd`-style second default) before giving up and falling back to+/// `text`; numeric strings are NOT date-detected. xerj had no such check at all: every string/// field, including an ISO timestamp, was unconditionally inferred as/// `text` — e.g. xerj's own `xerj-monitoring` index (auto-created via/// `get_or_create_index`, no template covers it) ends up with `timestamp`@@ -18388,10 +18499,58 @@ fn looks_like_date_string(s: &str) -> bool {|| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f").is_ok()}-fn infer_field_type(val: &Value) -> FieldType {+/// Can `v` be a value of a DEFAULT-format date field+/// (`strict_date_optional_time||epoch_millis`)? Used at ingest to reject+/// values ES would refuse with `mapper_parsing_exception` — most importantly+/// for dynamically-inferred date fields, which previously swallowed any+/// later garbage silently (invisible to range queries, sorts, and Kibana+/// time filters).+///+/// Deliberately LENIENT in what it accepts (year-only "2024", year-month+/// "2024-07", any "yyyy-MM-ddT…" tail, epoch numbers/strings): a false+/// accept preserves the old behavior, a false reject would break valid+/// ingest, so only shapes that cannot possibly parse under the default+/// formats are rejected.+pub(crate) fn default_format_date_value_ok(v: &Value) -> bool {+ match v {+ Value::Null => true,+ Value::Number(_) => true, // epoch_millis+ Value::Array(arr) => arr.iter().all(default_format_date_value_ok),+ Value::String(s) => {+ let s = s.trim();+ if s.parse::<i64>().is_ok() {+ return true; // epoch_millis as string (also bare "2024")+ }+ let b = s.as_bytes();+ // yyyy prefix is mandatory for every non-epoch default format.+ if b.len() < 4 || !b[..4].iter().all(u8::is_ascii_digit) {+ return false;+ }+ match b.len() {+ 4 => true, // yyyy+ 7 => b[4] == b'-' && b[5].is_ascii_digit() && b[6].is_ascii_digit(), // yyyy-MM+ _ => {+ // yyyy-MM-dd with an optional T… time tail. The tail is+ // not re-validated (lenient by design, see above).+ b.len() >= 10+ && b[4] == b'-'+ && b[7] == b'-'+ && b[5].is_ascii_digit()+ && b[6].is_ascii_digit()+ && b[8].is_ascii_digit()+ && b[9].is_ascii_digit()+ && (b.len() == 10 || b[10] == b'T' || b[10] == b't')+ }+ }+ }+ Value::Bool(_) | Value::Object(_) => false,+ }+}++fn infer_field_type(val: &Value, date_detection: bool) -> FieldType {match val {Value::String(s) => {- if looks_like_date_string(s) {+ if date_detection && looks_like_date_string(s) {FieldType::Date} else {FieldType::Text@@ -18409,7 +18568,7 @@ fn infer_field_type(val: &Value) -> FieldType {// Detect type from first non-null element in the array.arr.iter().find(|v| !v.is_null())- .map(infer_field_type)+ .map(|v| infer_field_type(v, date_detection)).unwrap_or(FieldType::Text)}Value::Object(_) => FieldType::Object,@@ -25338,6 +25497,81 @@ fn build_registry_from_settings(settings: &Value) -> AnalyzerRegistry {registry}+#[cfg(test)]+mod date_detection_tests {+ use super::*;++ /// Pins the accept/reject envelope of dynamic date inference.+ #[test]+ fn looks_like_date_string_envelope() {+ for good in [+ "2026-07-25",+ "2026-07-25T10:30:00",+ "2026-07-25T10:30:00.123",+ "2026-07-25T10:30:00Z",+ "2026-07-25T10:30:00+02:00",+ ] {+ assert!(looks_like_date_string(good), "{good} should detect as date");+ }+ for bad in [+ "n/a",+ "hello",+ "2026", // conservative: bare year stays text+ "2026-07", // conservative: year-month stays text+ "2026/07/25", // slash format not in the detection set+ "2026-invoice-1", // date-shaped prefix, garbage tail+ "20260725",+ "",+ ] {+ assert!(!looks_like_date_string(bad), "{bad} should stay text");+ }+ }++ /// `date_detection: false` forces first-seen ISO strings to `text`.+ #[test]+ fn infer_respects_date_detection_toggle() {+ let iso = Value::String("2026-07-25T10:30:00Z".into());+ assert!(matches!(infer_field_type(&iso, true), FieldType::Date));+ assert!(matches!(infer_field_type(&iso, false), FieldType::Text));+ // Arrays follow the first non-null element under the same toggle.+ let arr = serde_json::json!([null, "2026-07-25"]);+ assert!(matches!(infer_field_type(&arr, true), FieldType::Date));+ assert!(matches!(infer_field_type(&arr, false), FieldType::Text));+ // Numbers are never date-detected.+ let num = serde_json::json!(1721900000000i64);+ assert!(matches!(infer_field_type(&num, true), FieldType::Long));+ }++ /// Ingest-time acceptance for default-format date fields: lenient on+ /// everything ES could parse, rejecting only impossible shapes.+ #[test]+ fn default_format_date_value_envelope() {+ for ok in [+ serde_json::json!(null),+ serde_json::json!(1721900000000i64),+ serde_json::json!("1721900000000"),+ serde_json::json!("2026"),+ serde_json::json!("2026-07"),+ serde_json::json!("2026-07-25"),+ serde_json::json!("2026-07-25T10:30"), // partial time is valid ES+ serde_json::json!("2026-07-25T10:30:00.123+02:00"),+ serde_json::json!(["2026-07-25", null, 1721900000000i64]),+ ] {+ assert!(default_format_date_value_ok(&ok), "{ok} must be accepted");+ }+ for bad in [+ serde_json::json!("n/a"),+ serde_json::json!("hello"),+ serde_json::json!("2026/07/25"),+ serde_json::json!(true),+ serde_json::json!({"nested": 1}),+ serde_json::json!(["2026-07-25", "n/a"]),+ ] {+ assert!(!default_format_date_value_ok(&bad), "{bad} must be rejected");+ }+ }+}+#[cfg(test)]mod fts_projection_tests {use super::*;