Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): honor date_detection:false + reject non-dates in inferred date fields

xerj-org4ed5540aedb3
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 5b1e9d229..bc6d7fdda 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -1244,6 +1244,9 @@ pub async fn process_bulk_with_opts(
block_type,
..
}) if block_type.contains("read_only_allow_delete") => 429,
+ // Default-format date rejection ("failed to parse
+ // field [..] of type [date]") — ES answers 400.
+ EngineError::Common(xerj_common::XerjError::InvalidMapping { .. }) => 400,
_ => 500,
};
items[item_idx] = Some(BulkItemResult {
@@ -1694,6 +1697,8 @@ pub async fn process_bulk_with_opts(
block_type,
..
}) if block_type.contains("read_only_allow_delete") => 429,
+ // Default-format date rejection — ES answers 400.
+ EngineError::Common(xerj_common::XerjError::InvalidMapping { .. }) => 400,
_ => 500,
};
items[item_idx] = Some(BulkItemResult {
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 03d64ea14..28716fd16 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -444,6 +444,12 @@ impl Engine {
// so GET /_mapping and mapping-dependent code paths see
// the same mapping as pre-restart.
engine.load_persisted_es_mapping(&name_str);
+ // The index isn't registered yet, so the propagation
+ // inside load can't find it — set the toggles on the
+ // local handle instead.
+ if let Some(m) = engine.index_mappings.get(name_str.as_str()) {
+ Engine::apply_date_mapping_flags(&idx, m.value());
+ }
engine.indices.insert(name_str, idx);
}
Err(e) => {
@@ -785,9 +791,72 @@ impl Engine {
}
}
}
+ self.propagate_date_detection(name, &mapping);
self.index_mappings.insert(name.to_string(), mapping);
}
+ /// Push the mapping's `date_detection` toggle (default true) down to the
+ /// open `Index` so dynamic inference honors it. The blob shape varies by
+ /// caller (`{"date_detection": ..}` from PUT /_mapping, or nested under
+ /// `"mappings"` from index-create), so both levels are checked.
+ fn propagate_date_detection(&self, name: &str, mapping: &Value) {
+ if let Ok(idx) = self.get_index(name) {
+ Self::apply_date_mapping_flags(&idx, mapping);
+ }
+ }
+
+ /// Push both date-related mapping toggles down to an open index:
+ /// the `date_detection` bool and the set of date fields excluded from
+ /// default-format ingest validation (explicit `format` — those are
+ /// validated against their own format by the bulk path — or
+ /// `ignore_malformed`).
+ pub(crate) fn apply_date_mapping_flags(idx: &crate::index::Index, mapping: &Value) {
+ idx.set_date_detection(Self::mapping_date_detection(mapping));
+ idx.set_date_format_exclusions(Self::mapping_date_exclusions(mapping));
+ }
+
+ /// Read the `date_detection` toggle out of a raw mapping blob
+ /// (defaulting to true, like ES).
+ pub(crate) fn mapping_date_detection(mapping: &Value) -> bool {
+ mapping
+ .get("date_detection")
+ .or_else(|| mapping.get("mappings").and_then(|m| m.get("date_detection")))
+ .and_then(Value::as_bool)
+ .unwrap_or(true)
+ }
+
+ /// Top-level date fields carrying an explicit `format` or
+ /// `ignore_malformed` in the raw mapping blob (either shape).
+ fn mapping_date_exclusions(mapping: &Value) -> std::collections::HashSet<String> {
+ let props = mapping.get("properties").or_else(|| {
+ mapping
+ .get("mappings")
+ .and_then(|m| m.get("properties"))
+ });
+ let mut out = std::collections::HashSet::new();
+ let Some(obj) = props.and_then(Value::as_object) else {
+ return out;
+ };
+ for (fname, spec) in obj {
+ let ftype = spec.get("type").and_then(Value::as_str).unwrap_or("");
+ if ftype != "date" && ftype != "date_nanos" {
+ continue;
+ }
+ let has_format = spec
+ .get("format")
+ .and_then(Value::as_str)
+ .is_some_and(|f| !f.is_empty());
+ let ignore_malformed = spec
+ .get("ignore_malformed")
+ .and_then(Value::as_bool)
+ .unwrap_or(false);
+ if has_format || ignore_malformed {
+ out.insert(fname.clone());
+ }
+ }
+ out
+ }
+
/// Load a previously-persisted raw ES mapping blob for `name` (if any)
/// into the in-memory `index_mappings` map. Called whenever an index is
/// (re)opened from disk — boot scan and snapshot restore. A missing
@@ -800,6 +869,7 @@ impl Engine {
};
match serde_json::from_slice::<Value>(&bytes) {
Ok(mapping) => {
+ self.propagate_date_detection(name, &mapping);
self.index_mappings.insert(name.to_string(), mapping);
}
Err(e) => {
@@ -1586,6 +1656,9 @@ impl Engine {
// Snapshot dirs carry es_mapping.json — reload it so the
// restored index serves the same mapping it was saved with.
self.load_persisted_es_mapping(idx_name);
+ if let Some(m) = self.index_mappings.get(idx_name) {
+ Engine::apply_date_mapping_flags(&idx, m.value());
+ }
self.indices.insert(idx_name.clone(), idx);
info!(index = idx_name, "index restored from snapshot");
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 4592da405..b0ac9b28a 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2,7 +2,7 @@
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use regex::Regex;
@@ -412,6 +412,22 @@ fn l2_normalize_vec(v: &mut [f32]) {
pub struct Index {
name: IndexName,
schema: Arc<RwLock<ManagedSchema>>,
+ /// ES `date_detection` mapping toggle (default true). Consulted by
+ /// dynamic inference only — an explicit `date_detection: false` in the
+ /// index mapping keeps first-seen ISO-looking strings mapped as `text`.
+ /// Owned here (not in `ManagedSchema`) because the raw mapping blob is
+ /// API-layer state (`engine.index_mappings`); the engine propagates the
+ /// flag on create/open/put-mapping via [`Index::set_date_detection`].
+ date_detection: AtomicBool,
+ /// Fast bail for ingest date validation: true once the schema holds any
+ /// `date`-typed field (set at construction and when dynamic inference
+ /// adds one; never unset — a stale `true` only costs a schema read).
+ has_date_fields: AtomicBool,
+ /// Date fields whose mapping declares an explicit `format` (validated
+ /// against that format by `bulk::find_bad_date_field`) or
+ /// `ignore_malformed` — excluded from the engine's default-format
+ /// ingest validation. Engine-propagated alongside `date_detection`.
+ date_format_exclusions: std::sync::RwLock<Arc<std::collections::HashSet<String>>>,
store: Arc<IndexStore>,
/// 16-shard FTS memtable. Replaces the pre-v16 single
/// `Arc<RwLock<FtsMemtable>>` that serialized all concurrent bulk
@@ -896,9 +912,16 @@ impl Index {
let registry = Arc::new(build_registry_from_settings(&settings));
info!(name = name.as_str(), "index created");
+ let has_dates = managed
+ .fields()
+ .iter()
+ .any(|f| matches!(f.field_type, FieldType::Date));
let index = Arc::new(Self {
name,
schema: Arc::new(RwLock::new(managed)),
+ date_detection: AtomicBool::new(true),
+ has_date_fields: AtomicBool::new(has_dates),
+ date_format_exclusions: std::sync::RwLock::new(Arc::new(Default::default())),
store,
memtable: Arc::new(
crate::memtable::ShardedFtsMemtable::with_registry_and_shards(
@@ -1148,9 +1171,16 @@ impl Index {
doc_count = total_doc_count,
"index opened"
);
+ let has_dates = schema
+ .fields()
+ .iter()
+ .any(|f| matches!(f.field_type, FieldType::Date));
let index = Arc::new(Self {
name,
schema: Arc::new(RwLock::new(schema)),
+ date_detection: AtomicBool::new(true),
+ has_date_fields: AtomicBool::new(has_dates),
+ date_format_exclusions: std::sync::RwLock::new(Arc::new(Default::default())),
max_concurrent_queries: Arc::new(Semaphore::new(64)),
enrichments: Arc::new(RwLock::new(HashMap::new())),
store,
@@ -1311,6 +1341,9 @@ impl Index {
"write",
)));
}
+ // Default-format date validation — before any durable write so a
+ // rejected doc leaves no WAL/memtable trace.
+ self.validate_default_format_dates(&source).await?;
// Process-wide admission: disk flood-stage block (item 3) + parent
// memory circuit breaker (item 1). Fires before any per-index work.
self.governor_write_gate()?;
@@ -10476,6 +10509,75 @@ impl Index {
// ── Helpers ───────────────────────────────────────────────────────────────
+ /// Set the ES `date_detection` toggle for dynamic inference. Called by
+ /// the engine whenever the index mapping is created, loaded from disk,
+ /// or replaced via PUT /_mapping.
+ pub fn set_date_detection(&self, on: bool) {
+ self.date_detection.store(on, Ordering::Relaxed);
+ }
+
+ /// Current `date_detection` state (default true, like ES).
+ pub fn date_detection_enabled(&self) -> bool {
+ self.date_detection.load(Ordering::Relaxed)
+ }
+
+ /// Replace the set of date fields excluded from default-format ingest
+ /// validation (fields with an explicit mapping `format` — validated
+ /// separately by `bulk::find_bad_date_field` — or `ignore_malformed`).
+ /// Engine-propagated together with [`Index::set_date_detection`].
+ pub fn set_date_format_exclusions(&self, excluded: std::collections::HashSet<String>) {
+ if let Ok(mut guard) = self.date_format_exclusions.write() {
+ *guard = Arc::new(excluded);
+ }
+ }
+
+ /// Reject documents whose value for a DEFAULT-format date field (most
+ /// importantly a dynamically-inferred one) cannot possibly parse under
+ /// ES's default `strict_date_optional_time||epoch_millis` — real ES
+ /// answers 400 `mapper_parsing_exception`; xerj previously accepted the
+ /// doc silently, leaving the value invisible to range queries, sorts,
+ /// and Kibana time filters. Runs on the per-doc write funnel only; the
+ /// raw-bytes turbo batch paths intentionally skip it (they never parse
+ /// doc bodies — same envelope as the pre-existing explicit-format
+ /// validation, which routes strict-format indexes off turbo entirely).
+ async fn validate_default_format_dates(&self, source: &Value) -> Result<()> {
+ if !self.has_date_fields.load(Ordering::Relaxed) {
+ return Ok(());
+ }
+ let Some(obj) = source.as_object() else {
+ return Ok(());
+ };
+ let exclusions = self
+ .date_format_exclusions
+ .read()
+ .map(|g| Arc::clone(&g))
+ .unwrap_or_default();
+ let schema = self.schema.read().await;
+ for (key, val) in obj {
+ if val.is_null() {
+ continue;
+ }
+ let Some(fc) = schema.field(key) else {
+ continue;
+ };
+ if !matches!(fc.field_type, FieldType::Date) || exclusions.contains(key) {
+ continue;
+ }
+ if !default_format_date_value_ok(val) {
+ let preview: String = match val {
+ Value::String(s) => s.chars().take(64).collect(),
+ other => other.to_string().chars().take(64).collect(),
+ };
+ return Err(EngineError::Common(xerj_common::XerjError::InvalidMapping {
+ reason: format!(
+ "failed to parse field [{key}] of type [date] with value [{preview}]"
+ ),
+ }));
+ }
+ }
+ Ok(())
+ }
+
/// Batch variant of [`evolve_schema_from_doc`]: one schema read-lock
/// acquisition for the WHOLE batch instead of one per doc. The per-doc
/// variant paid a tokio `RwLock::read().await` per document — measured
@@ -10493,6 +10595,7 @@ impl Index {
if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {
return;
}
+ let date_detection = self.date_detection_enabled();
let mut out: Vec<(String, FieldConfig)> = Vec::new();
for source in sources {
let Some(obj) = source.as_object() else {
@@ -10500,7 +10603,7 @@ impl Index {
};
for (key, val) in obj {
if !schema.schema.has_field(key) && !out.iter().any(|(k, _)| k == key) {
- let ft = infer_field_type(val);
+ let ft = infer_field_type(val, date_detection);
out.push((key.clone(), FieldConfig::new(key.clone(), ft)));
}
}
@@ -10519,6 +10622,9 @@ impl Index {
let mut schema_changed = false;
for (_, fc) in new_fields {
if !schema.schema.has_field(&fc.name) {
+ if matches!(fc.field_type, FieldType::Date) {
+ self.has_date_fields.store(true, Ordering::Relaxed);
+ }
let _ = schema.schema.add_field(fc);
schema_changed = true;
}
@@ -10543,10 +10649,11 @@ impl Index {
if !matches!(schema.dynamic, xerj_common::schema::DynamicMapping::Dynamic) {
return;
}
+ let date_detection = self.date_detection_enabled();
obj.iter()
.filter(|(key, _)| !schema.schema.has_field(key))
.map(|(key, val)| {
- let ft = infer_field_type(val);
+ let ft = infer_field_type(val, date_detection);
(key.clone(), FieldConfig::new(key.clone(), ft))
})
.collect()
@@ -10566,6 +10673,9 @@ impl Index {
let mut schema_changed = false;
for (_, fc) in new_fields {
if !schema.schema.has_field(&fc.name) {
+ if matches!(fc.field_type, FieldType::Date) {
+ self.has_date_fields.store(true, Ordering::Relaxed);
+ }
let _ = schema.schema.add_field(fc);
schema_changed = true;
}
@@ -18358,10 +18468,11 @@ fn apply_copy_to(source: &Value, schema: &Schema) -> Value {
/// ES's default `date_detection`: a string field with no explicit mapping
/// (from a template or the user) is inferred as `date` when its value fully
-/// parses as a standard date/time format — real ES tries
-/// `strict_date_optional_time` (an ISO-8601 date or date-time, optional
-/// fractional seconds/offset) plus `epoch_millis` before giving up and
-/// falling back to `text`. xerj had no such check at all: every string
+/// parses as a standard date/time format — real ES tries its default
+/// `dynamic_date_formats` (`strict_date_optional_time` — an ISO-8601 date or
+/// date-time with optional fractional seconds/offset — plus the
+/// `yyyy/MM/dd`-style second default) before giving up and falling back to
+/// `text`; numeric strings are NOT date-detected. xerj had no such check at all: every string
/// field, including an ISO timestamp, was unconditionally inferred as
/// `text` — e.g. xerj's own `xerj-monitoring` index (auto-created via
/// `get_or_create_index`, no template covers it) ends up with `timestamp`
@@ -18388,10 +18499,58 @@ fn looks_like_date_string(s: &str) -> bool {
|| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f").is_ok()
}
-fn infer_field_type(val: &Value) -> FieldType {
+/// Can `v` be a value of a DEFAULT-format date field
+/// (`strict_date_optional_time||epoch_millis`)? Used at ingest to reject
+/// values ES would refuse with `mapper_parsing_exception` — most importantly
+/// for dynamically-inferred date fields, which previously swallowed any
+/// later garbage silently (invisible to range queries, sorts, and Kibana
+/// time filters).
+///
+/// Deliberately LENIENT in what it accepts (year-only "2024", year-month
+/// "2024-07", any "yyyy-MM-ddT…" tail, epoch numbers/strings): a false
+/// accept preserves the old behavior, a false reject would break valid
+/// ingest, so only shapes that cannot possibly parse under the default
+/// formats are rejected.
+pub(crate) fn default_format_date_value_ok(v: &Value) -> bool {
+ match v {
+ Value::Null => true,
+ Value::Number(_) => true, // epoch_millis
+ Value::Array(arr) => arr.iter().all(default_format_date_value_ok),
+ Value::String(s) => {
+ let s = s.trim();
+ if s.parse::<i64>().is_ok() {
+ return true; // epoch_millis as string (also bare "2024")
+ }
+ let b = s.as_bytes();
+ // yyyy prefix is mandatory for every non-epoch default format.
+ if b.len() < 4 || !b[..4].iter().all(u8::is_ascii_digit) {
+ return false;
+ }
+ match b.len() {
+ 4 => true, // yyyy
+ 7 => b[4] == b'-' && b[5].is_ascii_digit() && b[6].is_ascii_digit(), // yyyy-MM
+ _ => {
+ // yyyy-MM-dd with an optional T… time tail. The tail is
+ // not re-validated (lenient by design, see above).
+ b.len() >= 10
+ && b[4] == b'-'
+ && b[7] == b'-'
+ && b[5].is_ascii_digit()
+ && b[6].is_ascii_digit()
+ && b[8].is_ascii_digit()
+ && b[9].is_ascii_digit()
+ && (b.len() == 10 || b[10] == b'T' || b[10] == b't')
+ }
+ }
+ }
+ Value::Bool(_) | Value::Object(_) => false,
+ }
+}
+
+fn infer_field_type(val: &Value, date_detection: bool) -> FieldType {
match val {
Value::String(s) => {
- if looks_like_date_string(s) {
+ if date_detection && looks_like_date_string(s) {
FieldType::Date
} else {
FieldType::Text
@@ -18409,7 +18568,7 @@ fn infer_field_type(val: &Value) -> FieldType {
// Detect type from first non-null element in the array.
arr.iter()
.find(|v| !v.is_null())
- .map(infer_field_type)
+ .map(|v| infer_field_type(v, date_detection))
.unwrap_or(FieldType::Text)
}
Value::Object(_) => FieldType::Object,
@@ -25338,6 +25497,81 @@ fn build_registry_from_settings(settings: &Value) -> AnalyzerRegistry {
registry
}
+#[cfg(test)]
+mod date_detection_tests {
+ use super::*;
+
+ /// Pins the accept/reject envelope of dynamic date inference.
+ #[test]
+ fn looks_like_date_string_envelope() {
+ for good in [
+ "2026-07-25",
+ "2026-07-25T10:30:00",
+ "2026-07-25T10:30:00.123",
+ "2026-07-25T10:30:00Z",
+ "2026-07-25T10:30:00+02:00",
+ ] {
+ assert!(looks_like_date_string(good), "{good} should detect as date");
+ }
+ for bad in [
+ "n/a",
+ "hello",
+ "2026", // conservative: bare year stays text
+ "2026-07", // conservative: year-month stays text
+ "2026/07/25", // slash format not in the detection set
+ "2026-invoice-1", // date-shaped prefix, garbage tail
+ "20260725",
+ "",
+ ] {
+ assert!(!looks_like_date_string(bad), "{bad} should stay text");
+ }
+ }
+
+ /// `date_detection: false` forces first-seen ISO strings to `text`.
+ #[test]
+ fn infer_respects_date_detection_toggle() {
+ let iso = Value::String("2026-07-25T10:30:00Z".into());
+ assert!(matches!(infer_field_type(&iso, true), FieldType::Date));
+ assert!(matches!(infer_field_type(&iso, false), FieldType::Text));
+ // Arrays follow the first non-null element under the same toggle.
+ let arr = serde_json::json!([null, "2026-07-25"]);
+ assert!(matches!(infer_field_type(&arr, true), FieldType::Date));
+ assert!(matches!(infer_field_type(&arr, false), FieldType::Text));
+ // Numbers are never date-detected.
+ let num = serde_json::json!(1721900000000i64);
+ assert!(matches!(infer_field_type(&num, true), FieldType::Long));
+ }
+
+ /// Ingest-time acceptance for default-format date fields: lenient on
+ /// everything ES could parse, rejecting only impossible shapes.
+ #[test]
+ fn default_format_date_value_envelope() {
+ for ok in [
+ serde_json::json!(null),
+ serde_json::json!(1721900000000i64),
+ serde_json::json!("1721900000000"),
+ serde_json::json!("2026"),
+ serde_json::json!("2026-07"),
+ serde_json::json!("2026-07-25"),
+ serde_json::json!("2026-07-25T10:30"), // partial time is valid ES
+ serde_json::json!("2026-07-25T10:30:00.123+02:00"),
+ serde_json::json!(["2026-07-25", null, 1721900000000i64]),
+ ] {
+ assert!(default_format_date_value_ok(&ok), "{ok} must be accepted");
+ }
+ for bad in [
+ serde_json::json!("n/a"),
+ serde_json::json!("hello"),
+ serde_json::json!("2026/07/25"),
+ serde_json::json!(true),
+ serde_json::json!({"nested": 1}),
+ serde_json::json!(["2026-07-25", "n/a"]),
+ ] {
+ assert!(!default_format_date_value_ok(&bad), "{bad} must be rejected");
+ }
+ }
+}
+
#[cfg(test)]
mod fts_projection_tests {
use super::*;