Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(query): resolve ES named date formats for ignore_malformed validation (#89)

Vinz2168573e32ad5942
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 1a138934f..f86bc91a8 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -27338,109 +27338,181 @@ fn filter_keyword_length(val: &Value, max: usize) -> (Value, bool) {
}
}
-/// Translate an ES date-format pattern to a chrono strftime string and
-/// check whether `v` parses under it.
+/// Check whether `v` parses under the declared ES date `fmt`.
+///
+/// Thin alias for [`xerj_query::dates::date_value_valid_with_format`] — the
+/// SINGLE ingest-time date predicate, shared with the `ignore_malformed:
+/// false` rejection path in `xerj_engine::bulk`. It must stay a delegation:
+/// when these two paths had separate implementations, the strict one was the
+/// laxer of the pair, so turning `ignore_malformed` off made the engine
+/// accept *more*.
+///
+/// The compiled format is memoised on the format string, so a bulk of N docs
+/// compiles a field's declared format once rather than N times.
fn is_date_value_valid_with_format(v: &Value, fmt: &str) -> bool {
- // Accept epoch numbers for epoch_millis / epoch_second formats.
- if fmt.contains("epoch_millis") {
- return matches!(v, Value::Number(_))
- || matches!(v, Value::String(s) if s.parse::<i64>().is_ok());
+ xerj_query::dates::date_value_valid_with_format(v, fmt)
+}
+
+#[cfg(test)]
+mod ignore_malformed_date_format_tests {
+ //! Regression coverage for a real bug found live against OpenSearch's
+ //! UBI sample dashboards: `opensearch_dashboards_sample_ubi_events` maps
+ //! `timestamp` as `{"type":"date","format":"strict_date_time",
+ //! "ignore_malformed":true}`. Every document with a valid ISO timestamp
+ //! string was wrongly marked `_ignored` (the named format was matched as
+ //! literal text instead of being resolved to its real pattern), while
+ //! every document with a bare epoch-millis number was wrongly accepted
+ //! (any format treated a JSON number as valid) — exactly backwards from
+ //! real behavior. Confirmed by replaying the same mapping and documents
+ //! against a real OpenSearch 2.11.1 node.
+ use super::*;
+
+ #[test]
+ fn valid_iso_string_under_named_strict_format_is_not_ignored() {
+ assert!(is_date_value_valid_with_format(
+ &json!("2024-12-10T00:50:30.466Z"),
+ "strict_date_time"
+ ));
}
- if fmt.contains("epoch_second") {
- return matches!(v, Value::Number(_))
- || matches!(v, Value::String(s) if s.parse::<i64>().is_ok());
+
+ #[test]
+ fn bare_epoch_millis_number_under_non_epoch_format_is_ignored() {
+ assert!(!is_date_value_valid_with_format(
+ &json!(1717527762025i64),
+ "strict_date_time"
+ ));
}
- let Some(s) = v.as_str() else {
- return matches!(v, Value::Number(_));
- };
- // Split combined format ("||") — ES allows multiple fallback formats.
- for single_fmt in fmt.split("||") {
- let pat = es_date_format_to_strftime(single_fmt.trim());
- // ES validates declared date formats STRICTLY: the input must match
- // the pattern exactly (e.g. `dd-MM-yyyy` requires a 4-digit year, so
- // "19-12-90" is malformed). chrono's parsers are lenient about field
- // widths, so we additionally require the parsed value to round-trip
- // back to the original string under the same pattern.
- if let Ok(dt) = chrono::NaiveDateTime::parse_from_str(s, &pat) {
- if dt.format(&pat).to_string() == s {
- return true;
- }
- }
- if let Ok(d) = chrono::NaiveDate::parse_from_str(s, &pat) {
- if d.format(&pat).to_string() == s {
- return true;
- }
- }
- // Timezone-aware formats: accept a successful parse (round-tripping
- // `%z`/`%:z` offset spellings is unreliable, so don't require it).
- if chrono::DateTime::parse_from_str(s, &pat).is_ok() {
- return true;
- }
+
+ #[test]
+ fn bare_epoch_millis_number_is_valid_when_format_allows_it() {
+ assert!(is_date_value_valid_with_format(
+ &json!(1717527762025i64),
+ "strict_date_time||epoch_millis"
+ ));
}
- false
-}
-/// Map common ES date-pattern tokens (yyyy/MM/dd/HH/mm/ss/SSS/Z) to chrono
-/// strftime directives. Preserves literal chars between tokens.
-fn es_date_format_to_strftime(es_fmt: &str) -> String {
- let mut out = String::with_capacity(es_fmt.len() + 8);
- let bytes = es_fmt.as_bytes();
- let mut i = 0;
- while i < bytes.len() {
- // Longest-match substitution for tokens we care about.
- let rest = &es_fmt[i..];
- if rest.starts_with("yyyy") {
- out.push_str("%Y");
- i += 4;
- } else if rest.starts_with("yy") {
- out.push_str("%y");
- i += 2;
- } else if rest.starts_with("MM") {
- out.push_str("%m");
- i += 2;
- } else if rest.starts_with("dd") {
- out.push_str("%d");
- i += 2;
- } else if rest.starts_with("HH") {
- out.push_str("%H");
- i += 2;
- } else if rest.starts_with("mm") {
- out.push_str("%M");
- i += 2;
- } else if rest.starts_with("ss") {
- out.push_str("%S");
- i += 2;
- } else if rest.starts_with("SSSSSS") {
- out.push_str("%6f");
- i += 6;
- } else if rest.starts_with("SSSSSSSSS") {
- out.push_str("%9f");
- i += 9;
- } else if rest.starts_with("SSS") {
- out.push_str("%3f");
- i += 3;
- } else if rest.starts_with("Z") {
- out.push_str("%z");
- i += 1;
- } else if rest.starts_with("XXX") {
- out.push_str("%:z");
- i += 3;
- } else if rest.starts_with("'") {
- // Literal quoted text until next quote.
- i += 1;
- while i < bytes.len() && bytes[i] != b'\'' {
- out.push(bytes[i] as char);
- i += 1;
- }
- if i < bytes.len() {
- i += 1;
- }
- } else {
- out.push(bytes[i] as char);
- i += 1;
+ #[test]
+ fn number_under_numeric_shaped_non_epoch_format_is_valid() {
+ // basic_date is `yyyyMMdd`; ES stringifies a JSON number before
+ // formatting, so 20240101 must be accepted here too — this used to
+ // be silently dropped by gating numeric input on epoch formats.
+ assert!(is_date_value_valid_with_format(
+ &json!(20240101),
+ "basic_date"
+ ));
+ }
+
+ #[test]
+ fn real_named_formats_are_not_in_the_fail_closed_bucket() {
+ // The regression this replaces: `ordinal_date` and 27 other REAL ES
+ // formats were unresolvable, so fail-closed dropped every one of
+ // their values under `ignore_malformed: true` — including JSON
+ // numbers, which the previous release accepted. They resolve now,
+ // and validate for real.
+ assert!(is_date_value_valid_with_format(
+ &json!("2024-001"),
+ "ordinal_date"
+ ));
+ assert!(!is_date_value_valid_with_format(
+ &json!("literally anything"),
+ "ordinal_date"
+ ));
+ // The number shape that main accepted and the fix dropped.
+ assert!(is_date_value_valid_with_format(
+ &json!(2024001),
+ "basic_ordinal_date"
+ ));
+ assert!(is_date_value_valid_with_format(&json!(12), "hour"));
+ assert!(is_date_value_valid_with_format(&json!(2024), "weekyear"));
+ assert!(is_date_value_valid_with_format(
+ &json!("121030.123Z"),
+ "basic_time"
+ ));
+ }
+
+ #[test]
+ fn a_format_elasticsearch_rejects_still_fails_closed() {
+ // Fail-closed survives for what it was actually for: a format that
+ // is neither a named ES format nor a valid java.time pattern (ES
+ // rejects `banana` with `Unknown pattern letter: b` at mapping
+ // creation, so no document can legitimately arrive under one).
+ assert!(!is_date_value_valid_with_format(
+ &json!("2024-01-01"),
+ "banana"
+ ));
+ assert!(!is_date_value_valid_with_format(
+ &json!(1717527762025i64),
+ "banana"
+ ));
+ }
+
+ #[test]
+ fn no_es_named_format_drops_data_under_ignore_malformed() {
+ // Field-level guard on the whole shipped ES format set: a name that
+ // stops resolving silently empties the field, so assert none of them
+ // reject a value they should accept by construction — every format
+ // accepts JSON null, and no format may reject everything.
+ for name in xerj_query::dates::ES_NAMED_FORMATS {
+ assert!(
+ is_date_value_valid_with_format(&Value::Null, name),
+ "`{name}` treats null as malformed"
+ );
+ assert!(
+ xerj_query::dates::compile_formats(name).is_ok(),
+ "`{name}` is unresolvable — every value of a field declaring \
+ it would be dropped from _source under ignore_malformed"
+ );
}
}
- out
+
+ #[tokio::test]
+ async fn apply_ignore_malformed_end_to_end_matches_ubi_events_mapping() {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let state = AppState::new(config, engine, metrics);
+
+ state.engine.index_mappings.insert(
+ "ubi-repro".to_string(),
+ json!({
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "strict_date_time",
+ "ignore_malformed": true
+ },
+ "action_name": {"type": "keyword"}
+ }
+ }),
+ );
+
+ // A "click" event: valid ISO timestamp string — must NOT be ignored.
+ let click_doc = apply_ignore_malformed(
+ &state,
+ "ubi-repro",
+ json!({"timestamp": ["2024-12-10T00:50:30.466Z"], "action_name": "click"}),
+ );
+ assert!(
+ click_doc.get("_ignored").is_none(),
+ "valid ISO timestamp wrongly ignored: {click_doc:?}"
+ );
+
+ // An "on_search" event: bare epoch-millis number — MUST be ignored,
+ // since strict_date_time alone doesn't accept numeric input.
+ let on_search_doc = apply_ignore_malformed(
+ &state,
+ "ubi-repro",
+ json!({"timestamp": 1717527762025i64, "action_name": "on_search"}),
+ );
+ assert_eq!(
+ on_search_doc.get("_ignored"),
+ Some(&json!(["timestamp"])),
+ "epoch-millis number under a non-epoch format should be ignored: {on_search_doc:?}"
+ );
+ }
}
fn is_field_value_valid(ftype: &str, v: &Value) -> bool {
diff --git a/engine/crates/xerj-engine/src/bulk.rs b/engine/crates/xerj-engine/src/bulk.rs
index 88fb99ee3..49fc43a16 100644
--- a/engine/crates/xerj-engine/src/bulk.rs
+++ b/engine/crates/xerj-engine/src/bulk.rs
@@ -2391,50 +2391,21 @@ fn find_bad_date_field(
None
}
-/// Does `val` parse against the ES date `fmt`? Handles the formats
-/// the YAML test suite exercises: `epoch_second`, `epoch_millis`,
-/// `strict_date_optional_time*`, plus any chrono-compatible pattern
-/// (via es_format_to_epoch_ms from index.rs). For the purposes of
-/// rejecting obvious garbage, a value is accepted if *any* of the
-/// declared formats (ES accepts `||`-separated patterns) matches.
+/// Does `val` parse against the ES date `fmt`?
+///
+/// This is the `ignore_malformed: false` path — a `false` here rejects the
+/// whole document with `document_parsing_exception`. It delegates to
+/// [`xerj_query::dates::date_value_valid_with_format`], which is also what
+/// the `ignore_malformed: true` path calls, so the two cannot disagree.
+///
+/// They used to, and in the wrong direction: this function hand-rolled its
+/// own rules and ended with a blanket `true` for "named or custom patterns",
+/// so under a declared format like `dd/MM/yyyy` the value `"literally
+/// anything"` was accepted here while the *lenient* path dropped it. Setting
+/// `ignore_malformed: false` — asking for stricter handling — made the engine
+/// accept strictly more. Keep this a delegation.
fn date_value_parses(val: &Value, fmt: &str) -> bool {
- // Multi-format: `a||b||c` — accept if any individual pattern parses.
- if fmt.contains("||") {
- return fmt.split("||").any(|f| date_value_parses(val, f.trim()));
- }
- match val {
- Value::Null => true, // null is always fine
- Value::Number(n) => {
- // epoch_* formats accept any number. Textual formats need
- // a string, so numeric values only parse for epoch formats.
- fmt.contains("epoch") || n.as_i64().is_some()
- }
- Value::String(s) => {
- let s = s.trim();
- if fmt == "epoch_second" || fmt == "epoch_millis" {
- return s.parse::<i64>().is_ok();
- }
- // strict_date_optional_time / date_optional_time accept
- // any ISO-8601-ish string (year, year-month, full
- // timestamp). We accept anything that chrono's flexible
- // ISO parsers understand or that starts with a plausible
- // year prefix.
- if fmt.contains("date_optional_time") || fmt == "date" || fmt == "year" {
- return chrono::DateTime::parse_from_rfc3339(s).is_ok()
- || s.parse::<i64>().is_ok()
- || chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").is_ok()
- || chrono::NaiveDate::parse_from_str(s, "%Y").is_ok()
- || chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S").is_ok();
- }
- // For named or custom patterns, be conservative — only
- // reject obviously-broken values (non-numeric under an
- // epoch format). Unknown format patterns pass through so
- // we don't over-reject locale-specific tests that our
- // translator doesn't yet handle.
- true
- }
- _ => false,
- }
+ xerj_query::dates::date_value_valid_with_format(val, fmt)
}
/// `*`-glob match (single wildcard segment, no `**`, no character
diff --git a/engine/crates/xerj-query/src/dates.rs b/engine/crates/xerj-query/src/dates.rs
index 413740038..53ed53257 100644
--- a/engine/crates/xerj-query/src/dates.rs
+++ b/engine/crates/xerj-query/src/dates.rs
@@ -25,14 +25,16 @@
//!
//! Everything here works on UTC epoch milliseconds (ES date resolution).
-use chrono::{Datelike, Duration, Months, NaiveDate, NaiveDateTime, Timelike};
+use chrono::{Datelike, Duration, Months, NaiveDate, NaiveDateTime, Timelike, Weekday};
+use std::collections::HashMap;
+use std::sync::{Arc, OnceLock, RwLock};
// ─────────────────────────────────────────────────────────────────────────────
// Errors
// ─────────────────────────────────────────────────────────────────────────────
/// Why a bound could not be resolved as a date.
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
pub enum DateResolveError {
/// The `format` parameter itself is invalid (unknown pattern letter).
/// Carries the offending letter; the caller formats the full ES message
@@ -60,6 +62,17 @@ struct DateParts {
year: Option<i32>,
month: Option<u32>,
day: Option<u32>,
+ /// Day-of-year (`D`/`DDD`), the ordinal-date formats' day component.
+ /// Takes precedence over `month`/`day` when present.
+ day_of_year: Option<u32>,
+ /// ISO week-based year (`x` in the built-in week-date formats). When
+ /// present the calendar date comes from (week_year, week, weekday) and
+ /// `year`/`month`/`day` are ignored.
+ week_year: Option<i32>,
+ /// ISO week-of-week-based-year, 1-53 (`w`).
+ week: Option<u32>,
+ /// ISO day-of-week, 1 = Monday … 7 = Sunday (`e`).
+ weekday: Option<u32>,
hour: Option<u32>,
minute: Option<u32>,
second: Option<u32>,
@@ -68,6 +81,20 @@ struct DateParts {
tz_secs: Option<i32>,
}
+/// Map ISO day-of-week 1-7 (Mon-Sun) to a chrono `Weekday`.
+fn iso_weekday(n: u32) -> Option<Weekday> {
+ Some(match n {
+ 1 => Weekday::Mon,
+ 2 => Weekday::Tue,
+ 3 => Weekday::Wed,
+ 4 => Weekday::Thu,
+ 5 => Weekday::Fri,
+ 6 => Weekday::Sat,
+ 7 => Weekday::Sun,
+ _ => return None,
+ })
+}
+
impl DateParts {
/// Resolve to epoch milliseconds. `round_up` selects the ES round-up
/// fill for missing fields (month/day → 1, time-of-day → max).
@@ -90,8 +117,31 @@ impl DateParts {
self.milli.unwrap_or(0),
)
};
- let dt = NaiveDate::from_ymd_opt(year, month, day)?
- .and_hms_milli_opt(hour, minute, second, milli)?;
+ // Three mutually exclusive ways to name a calendar day, in the
+ // precedence java.time resolves them: ISO week date (week-based
+ // year + week + day-of-week), ordinal date (year + day-of-year),
+ // then the ordinary year/month/day.
+ let date = if let Some(wy) = self.week_year {
+ let weekday = match self.weekday {
+ Some(d) => iso_weekday(d)?,
+ None if round_up => Weekday::Sun,
+ None => Weekday::Mon,
+ };
+ match self.week {
+ Some(w) => NaiveDate::from_isoywd_opt(wy, w, weekday)?,
+ // A bare `weekyear` (`xxxx`) covers the whole ISO year:
+ // round down to week 1, round up to its last week (53 when
+ // the year has one, else 52).
+ None if round_up => NaiveDate::from_isoywd_opt(wy, 53, weekday)
+ .or_else(|| NaiveDate::from_isoywd_opt(wy, 52, weekday))?,
+ None => NaiveDate::from_isoywd_opt(wy, 1, weekday)?,
+ }
+ } else if let Some(doy) = self.day_of_year {
+ NaiveDate::from_yo_opt(year, doy)?
+ } else {
+ NaiveDate::from_ymd_opt(year, month, day)?
+ };
+ let dt = date.and_hms_milli_opt(hour, minute, second, milli)?;
let ms = dt.and_utc().timestamp_millis();
Some(ms - i64::from(self.tz_secs.unwrap_or(0)) * 1000)
}
@@ -127,6 +177,19 @@ pub enum PatTok {
Day {
two_digit: bool,
},
+ /// Day-of-year (`D`/`DDD`); payload = minimum digit count. Drives the
+ /// ordinal-date formats (`ordinal_date` = `yyyy-DDD`).
+ DayOfYear(usize),
+ /// ISO week-based year (`x` inside the built-in week-date formats);
+ /// payload = exact digit count.
+ WeekYear(usize),
+ /// ISO week-of-week-based-year (`w`); payload = exact-width flag.
+ Week {
+ two_digit: bool,
+ },
+ /// Numeric ISO day-of-week, 1 = Monday … 7 = Sunday (`e`/`ee` inside the
+ /// built-in week-date formats).
+ DayOfWeekNum,
Hour {
two_digit: bool,
},
@@ -146,6 +209,9 @@ pub enum PatTok {
AmPm,
/// Verbatim text that must match exactly.
Literal(String),
+ /// A `[...]` optional section (java.time's `optionalStart`/`optionalEnd`).
+ /// Matches if the whole section matches; otherwise it is skipped.
+ Optional(Vec<PatTok>),
/// A *valid* Java pattern letter this engine does not implement (e.g.
/// week-of-year `w`). Compiling succeeds (ES accepts the format); any
/// value parsed against it fails, producing ES's `failed to parse date
@@ -163,39 +229,367 @@ pub fn compile_formats(format: &str) -> Result<Vec<DateFmt>, DateResolveError> {
format.split("||").map(compile_one_format).collect()
}
+/// How many distinct format strings the compile cache will hold. Format
+/// strings come from mappings and from range-query `format` parameters, so
+/// the key space is attacker-influenced; past this many entries the cache
+/// stops growing and later formats simply compile every time (correct, just
+/// not cached).
+const FORMAT_CACHE_CAP: usize = 1024;
+
+type FormatCacheEntry = Result<Arc<Vec<DateFmt>>, DateResolveError>;
+
+fn format_cache() -> &'static RwLock<HashMap<String, FormatCacheEntry>> {
+ static CACHE: OnceLock<RwLock<HashMap<String, FormatCacheEntry>>> = OnceLock::new();
+ CACHE.get_or_init(|| RwLock::new(HashMap::new()))
+}
+
+/// [`compile_formats`], memoised on the format string.
+///
+/// Ingest recompiles a field's declared `format` once per *value*: a bulk of
+/// 10k docs with one date field re-tokenises the same pattern 10k times. The
+/// format is a property of the mapping, not of the value, so the compiled
+/// result is cached and handed out behind an `Arc`. Compilation *failures*
+/// are cached too — a bad format is equally hot and equally deterministic.
+pub fn compile_formats_cached(format: &str) -> FormatCacheEntry {
+ if let Ok(cache) = format_cache().read() {
+ if let Some(hit) = cache.get(format) {
+ return hit.clone();
+ }
+ }
+ let compiled = compile_formats(format).map(Arc::new);
+ if let Ok(mut cache) = format_cache().write() {
+ if cache.len() < FORMAT_CACHE_CAP {
+ cache.insert(format.to_string(), compiled.clone());
+ }
+ }
+ compiled
+}
+
+/// Is `value` acceptable for a date field declaring `format`?
+///
+/// THE single ingest-time date-validation predicate. Both ingest paths must
+/// call it and nothing else:
+///
+/// * `ignore_malformed: true` — a value that fails is dropped from the doc
+/// and listed in `_ignored`.
+/// * `ignore_malformed: false` — a value that fails rejects the whole
+/// document with `document_parsing_exception`.
+///
+/// They used to be two independent implementations, and the strict one was
+/// the *laxer* of the two: it accepted any string under any named format, so
+/// turning `ignore_malformed` off — asking for stricter handling — made the
+/// engine accept more. Sharing one predicate is what keeps that from
+/// happening again; if you are about to write a second one, don't.
+///
+/// A format string that does not compile returns `false` (fail closed): an
+/// unresolvable format is not evidence the value is fine, only that we
+/// cannot tell. Real ES rejects such a format when the mapping is created,
+/// so no document should ever reach this with one.
+pub fn date_value_valid_with_format(value: &serde_json::Value, format: &str) -> bool {
+ match compile_formats_cached(format) {
+ Ok(formats) => date_value_matches_formats(value, &formats),
+ Err(_) => false,
+ }
+}
+
+/// Does `value` parse as a valid date under any of the compiled `formats`?
+/// This is the ingest-time validation check (`ignore_malformed` decides
+/// whether a field that fails this gets dropped or the whole doc rejected),
+/// so it must match ES's actual acceptance rules precisely:
+///
+/// * `null` is always valid — ES treats it as "no value", not malformed
+/// input, uniformly across field types.
+/// * A JSON number is stringified and matched the same way a string would
+/// be, against every member of a `||`-joined format list — mirroring
+/// `DateFieldMapper`, which reads the value via the parser's `text()`
+/// (stringifying a numeric token) before handing it to the declared
+/// formatter. A number is valid input for ANY format whose textual form
+/// it matches, not just `epoch_millis`/`epoch_second`.
+/// * A string is valid whenever ANY member of the format list parses it —
+/// reusing [`parse_with_format`], the same parser
+/// [`resolve_date_bound_str`] uses for range-query bounds, so named
+/// formats (`strict_date_time`, `date_optional_time`, ...) resolve to
+/// their real pattern instead of being matched as literal text. No
+/// whitespace trimming: ES hands the value to the formatter as-is.
+pub fn date_value_matches_formats(value: &serde_json::Value, formats: &[DateFmt]) -> bool {
+ match value {
+ // ES treats JSON `null` as "no value" uniformly across field types
+ // (not malformed input), so it's never recorded in `_ignored`.
+ serde_json::Value::Null => true,
+ // ES's DateFieldMapper reads the value via the JSON parser's
+ // `text()`, which stringifies a numeric token before handing it to
+ // the declared formatter — a number is valid input for ANY format
+ // whose textual form it can match, not just epoch_millis/
+ // epoch_second. `basic_date` (`yyyyMMdd`) + the number `20240101`
+ // stringifies to "20240101" and parses cleanly; gating numeric
+ // input on epoch formats specifically (as this used to) silently
+ // dropped exactly that shape.
+ serde_json::Value::Number(n) => {
+ let s = n.to_string();
+ formats.iter().any(|f| value_matches_format(f, &s))
+ }
+ // No trim: ES does not strip leading/trailing whitespace before
+ // handing the value to the declared formatter, so a padded value
+ // that would fail there must fail here too.
+ serde_json::Value::String(s) => formats.iter().any(|f| value_matches_format(f, s)),
+ _ => false,
+ }
+}
+
+/// Validation-only match of one value against one compiled format.
+///
+/// Strict parse first — the same one range bounds use. If that fails and the
+/// pattern contains a *locale-dependent* text field (`MMM` month name, `E`
+/// weekday name), retry with those fields matched as opaque text.
+///
+/// This engine only knows English month names, but ES honours the mapping's
+/// `locale`, so under `format: "E, d MMM yyyy HH:mm:ss Z"` + `locale: "fr"`
+/// the value `"mer., 6 déc. 2000 02:55:00 -0800"` is perfectly valid data.
+/// Failing it would drop (or reject) documents real ES indexes — the exact
+/// class of bug this whole predicate exists to prevent. So a month name we
+/// can't identify is treated as "unverified", NOT as "malformed": everything
+/// else in the pattern — the separators, the day, the year, the time, the
+/// offset — is still checked, so genuine garbage is still rejected.
+///
+/// Note this is deliberately different from an *unresolvable format*, which
+/// fails closed. There we know nothing about the value; here we have checked
+/// every part of it except which language a word is in.
+fn value_matches_format(fmt: &DateFmt, value: &str) -> bool {
+ if parse_with_format(fmt, value, false).is_some() {
+ return true;
+ }
+ match fmt {
+ DateFmt::Pattern(toks) if has_locale_text(toks) => {
+ parse_pattern_with(toks, value, TextMatch::LocaleTolerant)
+ .and_then(|p| p.to_epoch_ms(false))
+ .is_some()
+ }
+ _ => false,
+ }
+}
+
+/// Does this pattern contain a field whose text depends on the locale?
+fn has_locale_text(toks: &[PatTok]) -> bool {
+ toks.iter()
+ .any(|t| matches!(t, PatTok::MonthName | PatTok::WeekdayName))
+}
+
+/// Every date format name Elasticsearch ships, lenient and `strict_` alike.
+///
+/// Exported so tests in other crates can assert the whole set resolves: a
+/// name that silently stops resolving becomes a silent data-loss bug on the
+/// `ignore_malformed: true` path, and this list is what makes that loud.
+pub const ES_NAMED_FORMATS: &[&str] = &[
+ "epoch_millis",
+ "epoch_second",
+ "date_optional_time",
+ "strict_date_optional_time",
+ "date_optional_time_nanos",
+ "strict_date_optional_time_nanos",
+ "iso8601",
+ "rfc3339",
+ "basic_date",
+ "strict_basic_date",
+ "basic_date_time",
+ "strict_basic_date_time",
+ "basic_date_time_no_millis",
+ "strict_basic_date_time_no_millis",
+ "basic_ordinal_date",
+ "strict_basic_ordinal_date",
+ "basic_ordinal_date_time",
+ "strict_basic_ordinal_date_time",
+ "basic_ordinal_date_time_no_millis",
+ "strict_basic_ordinal_date_time_no_millis",
+ "basic_time",
+ "strict_basic_time",
+ "basic_time_no_millis",
+ "strict_basic_time_no_millis",
+ "basic_t_time",
+ "strict_basic_t_time",
+ "basic_t_time_no_millis",
+ "strict_basic_t_time_no_millis",
+ "basic_week_date",
+ "strict_basic_week_date",
+ "basic_week_date_time",
+ "strict_basic_week_date_time",
+ "basic_week_date_time_no_millis",
+ "strict_basic_week_date_time_no_millis",
+ "date",
+ "strict_date",
+ "date_hour",
+ "strict_date_hour",
+ "date_hour_minute",
+ "strict_date_hour_minute",
+ "date_hour_minute_second",
+ "strict_date_hour_minute_second",
+ "date_hour_minute_second_fraction",
+ "strict_date_hour_minute_second_fraction",
+ "date_hour_minute_second_millis",
+ "strict_date_hour_minute_second_millis",
+ "date_time",
+ "strict_date_time",
+ "date_time_no_millis",
+ "strict_date_time_no_millis",
+ "hour",
+ "strict_hour",
+ "hour_minute",
+ "strict_hour_minute",
+ "hour_minute_second",
+ "strict_hour_minute_second",
+ "hour_minute_second_fraction",
+ "strict_hour_minute_second_fraction",
+ "hour_minute_second_millis",
+ "strict_hour_minute_second_millis",
+ "ordinal_date",
+ "strict_ordinal_date",
+ "ordinal_date_time",
+ "strict_ordinal_date_time",
+ "ordinal_date_time_no_millis",
+ "strict_ordinal_date_time_no_millis",
+ "time",
+ "strict_time",
+ "time_no_millis",
+ "strict_time_no_millis",
+ "t_time",
+ "strict_t_time",
+ "t_time_no_millis",
+ "strict_t_time_no_millis",
+ "week_date",
+ "strict_week_date",
+ "week_date_time",
+ "strict_week_date_time",
+ "week_date_time_no_millis",
+ "strict_week_date_time_no_millis",
+ "weekyear",
+ "strict_weekyear",
+ "weekyear_week",
+ "strict_weekyear_week",
+ "weekyear_week_day",
+ "strict_weekyear_week_day",
+ "year",
+ "strict_year",
+ "year_month",
+ "strict_year_month",
+ "year_month_day",
+ "strict_year_month_day",
+];
+
+/// Resolve one ES built-in named date format to its pattern.
+///
+/// This is the complete list of names Elasticsearch ships (`DateFormatters`),
+/// in the Joda spellings ES's own documentation uses. Every name here MUST
+/// resolve: a name that falls through to being compiled as a literal pattern
+/// hits an invalid pattern letter (`b` in `basic_time`, `o` in `ordinal_date`,
+/// `t` in `time`, …) and the whole format errors out. Because callers treat
+/// an unresolvable format as "this engine cannot validate anything for this
+/// field", a missing entry here is not a cosmetic gap — under
+/// `ignore_malformed: true` it silently drops every value of the field.
+///
+/// `strict_` and lenient names map to the same pattern. Real ES only differs
+/// between them in how tolerant it is of missing zero-padding; this engine's
+/// pattern parser is strict for both, which is the pre-existing convention
+/// (`date` / `strict_date` already shared `yyyy-MM-dd`).
+fn builtin_format_pattern(name: &str) -> Option<&'static str> {
+ // Strip the `strict_` prefix — every strict variant maps to the same
+ // pattern as its lenient twin.
+ let base = name.strip_prefix("strict_").unwrap_or(name);
+ Some(match base {
+ // ── basic (no separators) ────────────────────────────────────────
+ // The `.SSS` fraction is bracketed wherever ES's own formatter
+ // builds it with `optionalStart()` — `strict_date_time` accepts
+ // `2021-05-01T07:10:00Z` with no millis at all. Requiring it here
+ // would reject values real ES indexes.
+ "basic_date" => "yyyyMMdd",
+ "basic_date_time" => "yyyyMMdd'T'HHmmss[.SSS]XX",
+ "basic_date_time_no_millis" => "yyyyMMdd'T'HHmmssXX",
+ "basic_ordinal_date" => "yyyyDDD",
+ "basic_ordinal_date_time" => "yyyyDDD'T'HHmmss[.SSS]XX",
+ "basic_ordinal_date_time_no_millis" => "yyyyDDD'T'HHmmssXX",
+ "basic_time" => "HHmmss[.SSS]XX",
+ "basic_time_no_millis" => "HHmmssXX",
+ "basic_t_time" => "'T'HHmmss[.SSS]XX",
+ "basic_t_time_no_millis" => "'T'HHmmssXX",
+ "basic_week_date" => "xxxx'W'wwe",
+ "basic_week_date_time" => "xxxx'W'wwe'T'HHmmss[.SSS]XX",
+ "basic_week_date_time_no_millis" => "xxxx'W'wwe'T'HHmmssXX",
+ // ── calendar date / date-time ────────────────────────────────────
+ "date" | "year_month_day" => "yyyy-MM-dd",
+ "date_hour" => "yyyy-MM-dd'T'HH",
+ "date_hour_minute" => "yyyy-MM-dd'T'HH:mm",
+ "date_hour_minute_second" => "yyyy-MM-dd'T'HH:mm:ss",
+ "date_hour_minute_second_millis" | "date_hour_minute_second_fraction" => {
+ "yyyy-MM-dd'T'HH:mm:ss.SSS"
+ }
+ "date_time" => "yyyy-MM-dd'T'HH:mm:ss[.SSS]XX",
+ "date_time_no_millis" => "yyyy-MM-dd'T'HH:mm:ssXX",
+ "year" => "yyyy",
+ "year_month" => "yyyy-MM",
+ // ── time of day ──────────────────────────────────────────────────
+ "hour" => "HH",
+ "hour_minute" => "HH:mm",
+ "hour_minute_second" => "HH:mm:ss",
+ "hour_minute_second_millis" | "hour_minute_second_fraction" => "HH:mm:ss.SSS",
+ "time" => "HH:mm:ss[.SSS]XX",
+ "time_no_millis" => "HH:mm:ssXX",
+ "t_time" => "'T'HH:mm:ss[.SSS]XX",
+ "t_time_no_millis" => "'T'HH:mm:ssXX",
+ // ── ordinal date (day-of-year) ───────────────────────────────────
+ "ordinal_date" => "yyyy-DDD",
+ "ordinal_date_time" => "yyyy-DDD'T'HH:mm:ss[.SSS]XX",
+ "ordinal_date_time_no_millis" => "yyyy-DDD'T'HH:mm:ssXX",
+ // ── ISO week date ────────────────────────────────────────────────
+ "week_date" | "weekyear_week_day" => "xxxx-'W'ww-e",
+ "week_date_time" => "xxxx-'W'ww-e'T'HH:mm:ss[.SSS]XX",
+ "week_date_time_no_millis" => "xxxx-'W'ww-e'T'HH:mm:ssXX",
+ "weekyear" => "xxxx",
+ "weekyear_week" => "xxxx-'W'ww",
+ _ => return None,
+ })
+}
+
fn compile_one_format(name: &str) -> Result<DateFmt, DateResolveError> {
// Named builtins first (both strict_ and lenient joda names).
- let pattern: &str = match name {
+ match name {
"epoch_millis" => return Ok(DateFmt::EpochMillis),
"epoch_second" => return Ok(DateFmt::EpochSecond),
"strict_date_optional_time"
| "date_optional_time"
| "strict_date_optional_time_nanos"
| "date_optional_time_nanos"
- | "iso8601" => return Ok(DateFmt::IsoOptionalTime),
- "basic_date" => "yyyyMMdd",
- "basic_date_time" => "yyyyMMdd'T'HHmmss.SSSXX",
- "basic_date_time_no_millis" => "yyyyMMdd'T'HHmmssXX",
- "date" | "strict_date" | "year_month_day" | "strict_year_month_day" => "yyyy-MM-dd",
- "year" | "strict_year" => "yyyy",
- "year_month" | "strict_year_month" => "yyyy-MM",
- "date_time" | "strict_date_time" => "yyyy-MM-dd'T'HH:mm:ss.SSSXX",
- "date_time_no_millis" | "strict_date_time_no_millis" => "yyyy-MM-dd'T'HH:mm:ssXX",
- "date_hour_minute_second" | "strict_date_hour_minute_second" => "yyyy-MM-dd'T'HH:mm:ss",
- "date_hour_minute_second_millis"
- | "strict_date_hour_minute_second_millis"
- | "date_hour_minute_second_fraction"
- | "strict_date_hour_minute_second_fraction" => "yyyy-MM-dd'T'HH:mm:ss.SSS",
- "date_hour_minute" | "strict_date_hour_minute" => "yyyy-MM-dd'T'HH:mm",
- "hour_minute_second" | "strict_hour_minute_second" => "HH:mm:ss",
- "rfc3339" | "rfc3339_lenient" => return Ok(DateFmt::IsoOptionalTime),
- other => other,
- };
- compile_pattern(pattern).map(DateFmt::Pattern)
+ | "iso8601"
+ | "rfc3339"
+ | "rfc3339_lenient" => return Ok(DateFmt::IsoOptionalTime),
+ _ => {}
+ }
+ if let Some(pattern) = builtin_format_pattern(name) {
+ return compile_pattern_in(pattern, PatternDialect::Builtin).map(DateFmt::Pattern);
+ }
+ // Not a built-in name — treat the string as a user-supplied java.time
+ // pattern, exactly as ES does. An invalid pattern letter errors here
+ // (ES: `Invalid format: [banana]: Unknown pattern letter: b`).
+ compile_pattern_in(name, PatternDialect::User).map(DateFmt::Pattern)
+}
+
+/// Which pattern-letter dialect a pattern string is written in.
+#[derive(Clone, Copy, PartialEq)]
+enum PatternDialect {
+ /// A user-supplied `java.time` pattern from a mapping or a range
+ /// query's `format`. Letters keep their `java.time` meanings (`x` is a
+ /// zone offset, `e` is a day-of-week *name*).
+ User,
+ /// One of the built-in names expanded by [`builtin_format_pattern`].
+ /// Those are written in the Joda spellings ES's docs use, where `x` is
+ /// the week-based year, `w` the week-of-week-based-year and `e` the
+ /// numeric day-of-week. Confining these meanings to the built-in
+ /// dialect keeps user patterns behaving exactly as they did.
+ Builtin,
}
/// Compile a Java-style date pattern into tokens.
-fn compile_pattern(pattern: &str) -> Result<Vec<PatTok>, DateResolveError> {
+fn compile_pattern_in(
+ pattern: &str,
+ dialect: PatternDialect,
+) -> Result<Vec<PatTok>, DateResolveError> {
let chars: Vec<char> = pattern.chars().collect();
let mut toks: Vec<PatTok> = Vec::new();
let mut i = 0usize;
@@ -227,6 +621,26 @@ fn compile_pattern(pattern: &str) -> Result<Vec<PatTok>, DateResolveError> {
push_literal(&mut toks, &lit);
continue;
}
+ // `[...]` optional section, java.time's optionalStart/optionalEnd.
+ // Only the built-in patterns use it; user patterns keep `[` as a
+ // literal, exactly as before.
+ if c == '[' && dialect == PatternDialect::Builtin {
+ let mut depth = 1usize;
+ let start = i + 1;
+ let mut j = start;
+ while j < chars.len() && depth > 0 {
+ match chars[j] {
+ '[' => depth += 1,
+ ']' => depth -= 1,
+ _ => {}
+ }
+ j += 1;
+ }
+ let inner: String = chars[start..j.saturating_sub(1)].iter().collect();
+ toks.push(PatTok::Optional(compile_pattern_in(&inner, dialect)?));
+ i = j;
+ continue;
+ }
if c.is_ascii_alphabetic() {
if !VALID_JAVA_PATTERN_LETTERS.contains(c) {
return Err(DateResolveError::UnknownPatternLetter(c));
@@ -250,6 +664,17 @@ fn compile_pattern(pattern: &str) -> Result<Vec<PatTok>, DateResolveError> {
'd' => PatTok::Day {
two_digit: run >= 2,
},
+ // Day-of-year: `DDD` (ordinal dates) is exactly 3 digits,
+ // `D` accepts 1-3. Previously unimplemented, so every
+ // ordinal-date value failed to parse.
+ 'D' => PatTok::DayOfYear(run.min(3)),
+ // Joda `x` = week-based year in the built-in week-date
+ // formats; `java.time` `x` = zone offset everywhere else.
+ 'x' if dialect == PatternDialect::Builtin => PatTok::WeekYear(run.min(4)),
+ 'w' if dialect == PatternDialect::Builtin => PatTok::Week {
+ two_digit: run >= 2,
+ },
+ 'e' if dialect == PatternDialect::Builtin => PatTok::DayOfWeekNum,
'H' | 'k' => PatTok::Hour {
two_digit: run >= 2,
},
@@ -388,13 +813,62 @@ fn parse_iso_partial(s: &str) -> Option<DateParts> {
Some(parts)
}
+/// How `MMM`/`E` text fields are matched.
+#[derive(Clone, Copy, PartialEq)]
+enum TextMatch {
+ /// Month names must be English (the only locale this engine knows).
+ /// Used for range-bound resolution, which needs a real month number.
+ English,
+ /// Any alphabetic run (plus an optional trailing `.`) matches, and an
+ /// unrecognised month resolves to January. Validation only — see
+ /// [`value_matches_format`].
+ LocaleTolerant,
+}
+
/// Parse a value against compiled pattern tokens.
fn parse_pattern(toks: &[PatTok], value: &str) -> Option<DateParts> {
+ parse_pattern_with(toks, value, TextMatch::English)
+}
+
+fn parse_pattern_with(toks: &[PatTok], value: &str, text: TextMatch) -> Option<DateParts> {
+ let mut st = PatState {
+ i: 0,
+ parts: DateParts::default(),
+ pm: false,
+ has_ampm: false,
+ };
+ consume_tokens(toks, value, text, &mut st)?;
+ if st.i != value.len() {
+ return None; // ES: "unparsed text found at index N"
+ }
+ if st.has_ampm && st.pm {
+ if let Some(h) = st.parts.hour {
+ if h < 12 {
+ st.parts.hour = Some(h + 12);
+ }
+ }
+ }
+ Some(st.parts)
+}
+
+/// Parser position plus the fields decoded so far. Cloned to trial-parse an
+/// optional section and rolled back if it doesn't match.
+#[derive(Clone)]
+struct PatState {
+ i: usize,
+ parts: DateParts,
+ pm: bool,
+ has_ampm: bool,
+}
+
+/// Consume `toks` from `st.i`. Returns `None` (leaving `st` unusable) if any
+/// token fails to match; callers that need to recover clone `st` first.
+fn consume_tokens(toks: &[PatTok], value: &str, text: TextMatch, st: &mut PatState) -> Option<()> {
let b = value.as_bytes();
- let mut i = 0usize;
- let mut parts = DateParts::default();
- let mut pm = false;
- let mut has_ampm = false;
+ let mut i = st.i;
+ let mut parts = st.parts.clone();
+ let mut pm = st.pm;
+ let mut has_ampm = st.has_ampm;
for (t_idx, tok) in toks.iter().enumerate() {
match tok {
@@ -413,6 +887,9 @@ fn parse_pattern(toks: &[PatTok], value: &str) -> Option<DateParts> {
Some(
PatTok::Month { .. }
| PatTok::Day { .. }
+ | PatTok::DayOfYear(_)
+ | PatTok::Week { .. }
+ | PatTok::DayOfWeekNum
| PatTok::Hour { .. }
| PatTok::Minute { .. }
| PatTok::Second { .. }
@@ -432,13 +909,60 @@ fn parse_pattern(toks: &[PatTok], value: &str) -> Option<DateParts> {
parts.month = Some(d.parse().ok()?);
}
PatTok::MonthName => {
- let name = take_alpha(b, &mut i)?;
- parts.month = Some(month_from_name(&name)?);
+ if text == TextMatch::LocaleTolerant {
+ // Consume the word; if it isn't an English month we
+ // can't say which month it is, only that a word is
+ // where a month name belongs. January stands in so the
+ // value still has to be a resolvable date overall.
+ let name = take_locale_text(value, &mut i)?;
+ parts.month = Some(month_from_name(&name).unwrap_or(1));
+ } else {
+ let name = take_alpha(b, &mut i)?;
+ parts.month = Some(month_from_name(&name)?);
+ }
}
PatTok::Day { two_digit } => {
let d = take_digits(b, &mut i, if *two_digit { 2 } else { 1 }, 2)?;
parts.day = Some(d.parse().ok()?);
}
+ PatTok::DayOfYear(min_digits) => {
+ let d = take_digits(b, &mut i, *min_digits, 3)?;
+ let doy: u32 = d.parse().ok()?;
+ if !(1..=366).contains(&doy) {
+ return None;
+ }
+ parts.day_of_year = Some(doy);
+ }
+ PatTok::WeekYear(digits) => {
+ let neg = if b.get(i) == Some(&b'-') {
+ i += 1;
+ true
+ } else {
+ false
+ };
+ let d = take_digits(b, &mut i, *digits, *digits)?;
+ let mut wy: i32 = d.parse().ok()?;
+ if neg {
+ wy = -wy;
+ }
+ parts.week_year = Some(wy);
+ }
+ PatTok::Week { two_digit } => {
+ let d = take_digits(b, &mut i, if *two_digit { 2 } else { 1 }, 2)?;
+ let w: u32 = d.parse().ok()?;
+ if !(1..=53).contains(&w) {
+ return None;
+ }
+ parts.week = Some(w);
+ }
+ PatTok::DayOfWeekNum => {
+ let d = take_digits(b, &mut i, 1, 1)?;
+ let dow: u32 = d.parse().ok()?;
+ if !(1..=7).contains(&dow) {
+ return None;
+ }
+ parts.weekday = Some(dow);
+ }
PatTok::Hour { two_digit } => {
let d = take_digits(b, &mut i, if *two_digit { 2 } else { 1 }, 2)?;
parts.hour = Some(d.parse().ok()?);
@@ -461,7 +985,11 @@ fn parse_pattern(toks: &[PatTok], value: &str) -> Option<DateParts> {
i += used;
}
PatTok::WeekdayName => {
- take_alpha(b, &mut i)?;
+ if text == TextMatch::LocaleTolerant {
+ take_locale_text(value, &mut i)?;
+ } else {
+ take_alpha(b, &mut i)?;
+ }
}
PatTok::AmPm => {
let a = take_alpha(b, &mut i)?;
@@ -479,20 +1007,32 @@ fn parse_pattern(toks: &[PatTok], value: &str) -> Option<DateParts> {
}
i += lb.len();
}
- PatTok::Unsupported => return None,
- }
- }
- if i != b.len() {
- return None; // ES: "unparsed text found at index N"
- }
- if has_ampm && pm {
- if let Some(h) = parts.hour {
- if h < 12 {
- parts.hour = Some(h + 12);
+ // `[...]` — try the section, and simply skip it if it does not
+ // match. This is what makes a trailing `.SSS` optional in the
+ // `*_date_time` formats, where ES's own formatter builds the
+ // fraction with `optionalStart()`.
+ PatTok::Optional(inner) => {
+ let mut trial = PatState {
+ i,
+ parts: parts.clone(),
+ pm,
+ has_ampm,
+ };
+ if consume_tokens(inner, value, text, &mut trial).is_some() {
+ i = trial.i;
+ parts = trial.parts;
+ pm = trial.pm;
+ has_ampm = trial.has_ampm;
+ }
}
+ PatTok::Unsupported => return None,
}
}
- Some(parts)
+ st.i = i;
+ st.parts = parts;
+ st.pm = pm;
+ st.has_ampm = has_ampm;
+ Some(())
}
fn take_digits(b: &[u8], i: &mut usize, min: usize, max: usize) -> Option<String> {
@@ -506,6 +1046,29 @@ fn take_digits(b: &[u8], i: &mut usize, min: usize, max: usize) -> Option<String
Some(String::from_utf8_lossy(&b[start..*i]).into_owned())
}
+/// Consume a locale-agnostic text field: a run of alphabetic characters in
+/// any script, plus an optional trailing `.`. Both are needed for real
+/// non-English month/weekday abbreviations — French December is `déc.`, dot
+/// included, and `é` is not ASCII.
+fn take_locale_text(value: &str, i: &mut usize) -> Option<String> {
+ let start = *i;
+ for ch in value[*i..].chars() {
+ if ch.is_alphabetic() {
+ *i += ch.len_utf8();
+ } else {
+ break;
+ }
+ }
+ if *i == start {
+ return None;
+ }
+ let word = value[start..*i].to_string();
+ if value[*i..].starts_with('.') {
+ *i += 1;
+ }
+ Some(word)
+}
+
fn take_alpha(b: &[u8], i: &mut usize) -> Option<String> {
let start = *i;
while *i < b.len() && b[*i].is_ascii_alphabetic() {
@@ -832,6 +1395,130 @@ mod tests {
resolve_date_bound_str(v, up, None).unwrap().unwrap()
}
+ // ── date_value_matches_formats (ingest-time ignore_malformed check) ──────
+ // Regression coverage for a real bug found live against OpenSearch's UBI
+ // sample dashboards: a named strict format like `strict_date_time` was
+ // matched as literal text (never resolving to its actual pattern), so
+ // every valid ISO string was wrongly rejected, while a bare epoch-millis
+ // number was wrongly ACCEPTED under any format, non-epoch included —
+ // exactly backwards from real Elasticsearch/OpenSearch behavior
+ // (confirmed by replaying the same mapping + documents against a real
+ // OpenSearch 2.11.1 node).
+
+ #[test]
+ fn named_strict_format_accepts_matching_iso_string() {
+ let formats = compile_formats("strict_date_time").unwrap();
+ let v = serde_json::json!("2024-01-01T00:00:00.000Z");
+ assert!(date_value_matches_formats(&v, &formats));
+ }
+
+ #[test]
+ fn named_strict_format_rejects_bare_number() {
+ // strict_date_time has no epoch_millis in its format list, so a raw
+ // number must NOT be accepted — matching real OpenSearch, which
+ // ignores this exact value under this exact mapping.
+ let formats = compile_formats("strict_date_time").unwrap();
+ let v = serde_json::json!(1717527762025i64);
+ assert!(!date_value_matches_formats(&v, &formats));
+ }
+
+ #[test]
+ fn numeric_shaped_format_accepts_a_matching_number_not_just_epoch() {
+ // Real ES's DateFieldMapper stringifies a JSON number before
+ // handing it to the declared formatter, so a number is valid input
+ // for ANY format whose textual form it matches — not just
+ // epoch_millis/epoch_second. `basic_date` is `yyyyMMdd`, so the
+ // number 20240101 must be accepted (previously silently dropped:
+ // gating numeric input on epoch formats specifically rejected it).
+ let formats = compile_formats("basic_date").unwrap();
+ assert!(date_value_matches_formats(
+ &serde_json::json!(20240101),
+ &formats
+ ));
+ }
+
+ #[test]
+ fn non_matching_number_under_numeric_shaped_format_is_still_rejected() {
+ let formats = compile_formats("basic_date").unwrap();
+ // Not 8 digits — doesn't fit yyyyMMdd.
+ assert!(!date_value_matches_formats(
+ &serde_json::json!(123),
+ &formats
+ ));
+ }
+
+ #[test]
+ fn only_genuinely_invalid_formats_fail_closed() {
+ // Fail-closed is right, but it must apply to formats ES itself
+ // rejects — NOT to real ES named formats. `ordinal_date` used to
+ // land in the unresolvable bucket (it isn't a java.time pattern:
+ // 'o' is not a pattern letter), which silently dropped every value
+ // of any field declaring it. It must resolve now.
+ for name in ES_NAMED_FORMATS {
+ assert!(
+ compile_formats(name).is_ok(),
+ "real ES named format `{name}` does not resolve — every value \
+ of a field declaring it would be dropped under \
+ ignore_malformed"
+ );
+ }
+ // A format ES also rejects (`Unknown pattern letter: b`) still fails
+ // closed.
+ assert!(compile_formats("banana").is_err());
+ assert!(compile_formats("strict_date_time||banana").is_err());
+ }
+
+ #[test]
+ fn whitespace_padded_value_is_not_trimmed() {
+ // ES hands the value to the formatter as-is; a value that would
+ // only match after trimming must be rejected, matching ES.
+ let formats = compile_formats("strict_date_time").unwrap();
+ let v = serde_json::json!(" 2024-01-01T00:00:00.000Z ");
+ assert!(!date_value_matches_formats(&v, &formats));
+ }
+
+ #[test]
+ fn epoch_millis_format_accepts_number_and_numeric_string() {
+ let formats = compile_formats("epoch_millis").unwrap();
+ assert!(date_value_matches_formats(
+ &serde_json::json!(1717527762025i64),
+ &formats
+ ));
+ assert!(date_value_matches_formats(
+ &serde_json::json!("1717527762025"),
+ &formats
+ ));
+ }
+
+ #[test]
+ fn combined_format_accepts_either_half() {
+ let formats = compile_formats("strict_date_time||epoch_millis").unwrap();
+ assert!(date_value_matches_formats(
+ &serde_json::json!("2024-01-01T00:00:00.000Z"),
+ &formats
+ ));
+ assert!(date_value_matches_formats(
+ &serde_json::json!(1717527762025i64),
+ &formats
+ ));
+ }
+
+ #[test]
+ fn named_strict_format_rejects_malformed_string() {
+ let formats = compile_formats("strict_date_time").unwrap();
+ let v = serde_json::json!("not-a-date");
+ assert!(!date_value_matches_formats(&v, &formats));
+ }
+
+ #[test]
+ fn null_value_always_matches() {
+ let formats = compile_formats("strict_date_time").unwrap();
+ assert!(date_value_matches_formats(
+ &serde_json::Value::Null,
+ &formats
+ ));
+ }
+
#[test]
fn partial_month_fills() {
// gte/lt (round down): first ms of the parsed fields, min fill.
@@ -1009,3 +1696,698 @@ mod tests {
assert_eq!(resolve_date_bound_str("2026-2-15", false, None), Ok(None));
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Named-format truth table
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod named_format_truth_table {
+ //! One row per Elasticsearch named date format: a string ES accepts, a
+ //! JSON number, and whether ES accepts that number.
+ //!
+ //! This table exists because of a specific data-loss regression. An
+ //! earlier fix made unresolvable formats fail closed — right instinct —
+ //! but 28 REAL ES formats were unresolvable, because they are not
+ //! java.time patterns (`basic_time` trips on `b`, `ordinal_date` on `o`,
+ //! `time` on `t`) and nothing mapped them to their real pattern. Under
+ //! `ignore_malformed: true` that silently dropped every value of any
+ //! field declaring one of them. The formats below are now implemented,
+ //! so they validate instead of vanishing.
+ //!
+ //! The `number_ok` column is the load-bearing one. ES's `DateFieldMapper`
+ //! stringifies a JSON number and feeds it to the declared formatter, so a
+ //! number is accepted exactly when its digits match the format's textual
+ //! shape — true for the all-digit formats (`yyyyDDD`, `HH`, `xxxx`, …),
+ //! false as soon as the format requires a separator, a `T`, or a zone
+ //! offset.
+
+ use super::*;
+
+ struct Row {
+ format: &'static str,
+ /// A value real Elasticsearch accepts under `format`.
+ good: &'static str,
+ /// A JSON number offered under `format`.
+ number: i64,
+ /// Does ES accept that number? (i.e. do its digits match the
+ /// format's textual shape)
+ number_ok: bool,
+ }
+
+ const fn row(format: &'static str, good: &'static str, number: i64, number_ok: bool) -> Row {
+ Row {
+ format,
+ good,
+ number,
+ number_ok,
+ }
+ }
+
+ /// Every named format, with a real value for each.
+ const TRUTH_TABLE: &[Row] = &[
+ // ── epoch ────────────────────────────────────────────────────────
+ row("epoch_millis", "1717527762025", 1717527762025, true),
+ row("epoch_second", "1717527762", 1717527762, true),
+ // ── the default, and its aliases ─────────────────────────────────
+ row(
+ "strict_date_optional_time",
+ "2024-01-01T00:00:00.000Z",
+ 1717527762025,
+ false,
+ ),
+ row("date_optional_time", "2024-01-01", 1717527762025, false),
+ row("iso8601", "2024-01-01T12:10:30Z", 1717527762025, false),
+ // ── basic (separator-free) ───────────────────────────────────────
+ row("basic_date", "20240101", 20240101, true),
+ row("basic_date_time", "20240101T121030.123Z", 20240101, false),
+ row(
+ "basic_date_time_no_millis",
+ "20240101T121030Z",
+ 20240101,
+ false,
+ ),
+ // NEWLY SUPPORTED from here down — each of these used to fail to
+ // compile, which under ignore_malformed dropped every value.
+ row("basic_ordinal_date", "2024001", 2024001, true),
+ row(
+ "basic_ordinal_date_time",
+ "2024001T121030.123Z",
+ 2024001,
+ false,
+ ),
+ row(
+ "basic_ordinal_date_time_no_millis",
+ "2024001T121030Z",
+ 2024001,
+ false,
+ ),
+ row("basic_time", "121030.123Z", 121030, false),
+ row("basic_time_no_millis", "121030Z", 121030, false),
+ row("basic_t_time", "T121030.123Z", 121030, false),
+ row("basic_t_time_no_millis", "T121030Z", 121030, false),
+ row("basic_week_date", "2024W011", 2024011, false),
+ row(
+ "basic_week_date_time",
+ "2024W011T121030.123Z",
+ 2024011,
+ false,
+ ),
+ row(
+ "basic_week_date_time_no_millis",
+ "2024W011T121030Z",
+ 2024011,
+ false,
+ ),
+ // ── calendar date / date-time ────────────────────────────────────
+ row("date", "2024-01-01", 20240101, false),
+ row("year_month_day", "2024-01-01", 20240101, false),
+ row("date_hour", "2024-01-01T12", 20240101, false),
+ row("date_hour_minute", "2024-01-01T12:10", 20240101, false),
+ row(
+ "date_hour_minute_second",
+ "2024-01-01T12:10:30",
+ 20240101,
+ false,
+ ),
+ row(
+ "date_hour_minute_second_millis",
+ "2024-01-01T12:10:30.123",
+ 20240101,
+ false,
+ ),
+ row(
+ "date_hour_minute_second_fraction",
+ "2024-01-01T12:10:30.123",
+ 20240101,
+ false,
+ ),
+ row(
+ "date_time",
+ "2024-01-01T12:10:30.123Z",
+ 1717527762025,
+ false,
+ ),
+ row(
+ "date_time_no_millis",
+ "2024-01-01T12:10:30Z",
+ 1717527762025,
+ false,
+ ),
+ row("year", "2024", 2024, true),
+ row("year_month", "2024-01", 202401, false),
+ // ── time of day ──────────────────────────────────────────────────
+ row("hour", "12", 12, true),
+ row("hour_minute", "12:10", 1210, false),
+ row("hour_minute_second", "12:10:30", 121030, false),
+ row("hour_minute_second_millis", "12:10:30.123", 121030, false),
+ row("hour_minute_second_fraction", "12:10:30.123", 121030, false),
+ row("time", "12:10:30.123Z", 121030, false),
+ row("time_no_millis", "12:10:30Z", 121030, false),
+ row("t_time", "T12:10:30.123Z", 121030, false),
+ row("t_time_no_millis", "T12:10:30Z", 121030, false),
+ // ── ordinal date ─────────────────────────────────────────────────
+ row("ordinal_date", "2024-001", 2024001, false),
+ row(
+ "ordinal_date_time",
+ "2024-001T12:10:30.123Z",
+ 2024001,
+ false,
+ ),
+ row(
+ "ordinal_date_time_no_millis",
+ "2024-001T12:10:30Z",
+ 2024001,
+ false,
+ ),
+ // ── ISO week date ────────────────────────────────────────────────
+ row("week_date", "2024-W01-1", 2024011, false),
+ row("week_date_time", "2024-W01-1T12:10:30.123Z", 2024011, false),
+ row(
+ "week_date_time_no_millis",
+ "2024-W01-1T12:10:30Z",
+ 2024011,
+ false,
+ ),
+ row("weekyear", "2024", 2024, true),
+ row("weekyear_week", "2024-W01", 202401, false),
+ row("weekyear_week_day", "2024-W01-1", 2024011, false),
+ ];
+
+ /// Values no format in the table may accept.
+ const GARBAGE: &[&str] = &[
+ "not-a-date",
+ "",
+ "banana",
+ "2024-99-99T99:99:99.999Z",
+ " ",
+ "null",
+ "2024-01-01T00:00:00.000Z extra",
+ ];
+
+ fn accepts(format: &str, value: &serde_json::Value) -> bool {
+ date_value_valid_with_format(value, format)
+ }
+
+ #[test]
+ fn every_named_format_resolves() {
+ // THE regression check. A format that does not compile is treated
+ // as "cannot validate", which under ignore_malformed: true drops
+ // every value of the field. Any name here that stops resolving is
+ // silent data loss, so this asserts the whole shipped ES set — not
+ // just the ones the table happens to exercise.
+ for name in ES_NAMED_FORMATS {
+ assert!(
+ compile_formats(name).is_ok(),
+ "ES named format `{name}` does not resolve: every value of a \
+ field declaring it would be silently dropped"
+ );
+ }
+ }
+
+ #[test]
+ fn strings_elasticsearch_accepts_are_accepted() {
+ for r in TRUTH_TABLE {
+ assert!(
+ accepts(r.format, &serde_json::json!(r.good)),
+ "format `{}` rejected `{}`, which real ES accepts",
+ r.format,
+ r.good
+ );
+ }
+ }
+
+ #[test]
+ fn numbers_match_elasticsearch_stringify_then_parse() {
+ for r in TRUTH_TABLE {
+ let got = accepts(r.format, &serde_json::json!(r.number));
+ assert_eq!(
+ got, r.number_ok,
+ "format `{}` with the number {}: expected accept={}, got {}",
+ r.format, r.number, r.number_ok, got
+ );
+ }
+ }
+
+ #[test]
+ fn genuine_garbage_is_rejected_under_every_format() {
+ for r in TRUTH_TABLE {
+ for g in GARBAGE {
+ assert!(
+ !accepts(r.format, &serde_json::json!(g)),
+ "format `{}` accepted garbage `{}`",
+ r.format,
+ g
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn a_format_elasticsearch_itself_rejects_still_fails_closed() {
+ // Fail-closed is preserved for what it was meant for: a format
+ // string that is not a named format AND not a valid java.time
+ // pattern. ES rejects these at mapping-creation time
+ // (`Unknown pattern letter: b`), so no document can legitimately
+ // arrive under one.
+ assert!(compile_formats("banana").is_err());
+ for g in GARBAGE {
+ assert!(!accepts("banana", &serde_json::json!(g)));
+ }
+ assert!(!accepts("banana", &serde_json::json!(1717527762025i64)));
+ assert!(!accepts("banana", &serde_json::json!("2024-01-01")));
+ }
+
+ #[test]
+ fn every_newly_supported_format_would_have_failed_closed_before() {
+ // Pins the exact scope of the regression that was fixed: these are
+ // the names that are NOT java.time patterns, so before they were
+ // added to the builtin table they compiled as literal patterns and
+ // errored on their first invalid pattern letter. Each must now both
+ // resolve AND validate its canonical value.
+ for r in TRUTH_TABLE {
+ let is_builtin_only = builtin_format_pattern(r.format).is_some()
+ && compile_pattern_in(r.format, PatternDialect::User).is_err();
+ if !is_builtin_only {
+ continue;
+ }
+ assert!(
+ accepts(r.format, &serde_json::json!(r.good)),
+ "`{}` resolves only via the builtin table and must validate \
+ its canonical value `{}`",
+ r.format,
+ r.good
+ );
+ }
+ }
+
+ #[test]
+ fn combined_formats_accept_either_side() {
+ // The shape real mappings use: a named format OR epoch millis.
+ for r in TRUTH_TABLE {
+ let combined = format!("{}||epoch_millis", r.format);
+ assert!(
+ accepts(&combined, &serde_json::json!(r.good)),
+ "`{combined}` rejected `{}`",
+ r.good
+ );
+ assert!(
+ accepts(&combined, &serde_json::json!(1717527762025i64)),
+ "`{combined}` rejected an epoch-millis number"
+ );
+ }
+ }
+
+ #[test]
+ fn null_is_never_malformed() {
+ for r in TRUTH_TABLE {
+ assert!(accepts(r.format, &serde_json::Value::Null));
+ }
+ }
+
+ #[test]
+ fn week_and_ordinal_dates_resolve_to_the_right_instant() {
+ // Resolving is not the same as resolving CORRECTLY: check the two
+ // new calendar systems land on the day they name.
+ // 2024-001 is 1 Jan 2024; ISO week 2024-W01-1 is Mon 1 Jan 2024.
+ let jan1 = resolve_date_bound_str("2024-01-01", false, None)
+ .unwrap()
+ .unwrap();
+ let ordinal_fmt = compile_formats("ordinal_date").unwrap();
+ let week_fmt = compile_formats("week_date").unwrap();
+ let ordinal = resolve_date_bound_str("2024-001", false, Some(&ordinal_fmt))
+ .unwrap()
+ .unwrap();
+ let week = resolve_date_bound_str("2024-W01-1", false, Some(&week_fmt))
+ .unwrap()
+ .unwrap();
+ assert_eq!(ordinal, jan1, "ordinal_date 2024-001 should be 2024-01-01");
+ assert_eq!(week, jan1, "week_date 2024-W01-1 should be 2024-01-01");
+
+ // 2024 is a leap year: day 366 exists and is 31 Dec.
+ let dec31 = resolve_date_bound_str("2024-12-31", false, None)
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ resolve_date_bound_str("2024-366", false, Some(&ordinal_fmt)),
+ Ok(Some(dec31))
+ );
+ // 2023 is not: day 366 must not resolve.
+ assert!(!accepts("ordinal_date", &serde_json::json!("2023-366")));
+ // Week 53 does not exist in 2024 (a 52-week ISO year).
+ assert!(!accepts("week_date", &serde_json::json!("2024-W53-1")));
+ // ...but it does in 2020.
+ assert!(accepts("week_date", &serde_json::json!("2020-W53-1")));
+ }
+
+ #[test]
+ fn compiled_formats_are_cached_and_the_cache_is_transparent() {
+ // The cache must not change any answer, only avoid the recompile.
+ for r in TRUTH_TABLE {
+ let uncached = compile_formats(r.format)
+ .map(|f| date_value_matches_formats(&serde_json::json!(r.good), &f))
+ .unwrap_or(false);
+ assert_eq!(
+ uncached,
+ accepts(r.format, &serde_json::json!(r.good)),
+ "cached and uncached disagree for `{}`",
+ r.format
+ );
+ }
+ // Same string twice hands back the same compiled Arc.
+ let a = compile_formats_cached("strict_date_time||epoch_millis").unwrap();
+ let b = compile_formats_cached("strict_date_time||epoch_millis").unwrap();
+ assert!(
+ Arc::ptr_eq(&a, &b),
+ "second compile of an identical format string should be a cache hit"
+ );
+ }
+}
+
+#[cfg(test)]
+mod locale_dependent_text_fields {
+ //! A pattern with a textual month/weekday is locale-dependent, and this
+ //! engine only knows English month names. ES honours the mapping's
+ //! `locale`, so rejecting a French month name would drop documents real
+ //! ES indexes — and it would do so on BOTH ingest paths at once (dropped
+ //! under `ignore_malformed: true`, whole document rejected under
+ //! `false`). The structure around the word is still validated.
+
+ use super::*;
+
+ /// The mapping from the `180_locale_dependent_mapping` conformance test.
+ const FR_FORMAT: &str = "E, d MMM yyyy HH:mm:ss Z";
+
+ fn accepts(format: &str, value: &str) -> bool {
+ date_value_valid_with_format(&serde_json::json!(value), format)
+ }
+
+ #[test]
+ fn french_month_names_are_accepted_not_dropped() {
+ assert!(accepts(FR_FORMAT, "mer., 6 déc. 2000 02:55:00 -0800"));
+ assert!(accepts(FR_FORMAT, "jeu., 7 déc. 2000 02:55:00 -0800"));
+ }
+
+ #[test]
+ fn english_month_names_still_take_the_strict_path() {
+ assert!(accepts(FR_FORMAT, "Wed, 6 Dec 2000 02:55:00 -0800"));
+ }
+
+ #[test]
+ fn locale_tolerance_only_covers_the_word_not_the_structure() {
+ // Everything except which language the month is in is still checked.
+ for bad in [
+ "not a date at all",
+ "mer., 6 déc. 2000 02:55:00", // missing the offset
+ "mer., 6 déc. 2000 99:99:99 -0800", // impossible time
+ "mer., 6 déc. 20 02:55:00 -0800", // 2-digit year
+ "mer., déc. 2000 02:55:00 -0800", // missing the day
+ ", 6 déc. 2000 02:55:00 -0800", // missing the weekday
+ "mer., 6 déc. 2000 02:55:00 -0800!", // trailing garbage
+ ] {
+ assert!(
+ !accepts(FR_FORMAT, bad),
+ "locale tolerance wrongly accepted `{bad}`"
+ );
+ }
+ }
+
+ #[test]
+ fn locale_tolerance_does_not_leak_into_range_bound_resolution() {
+ // Bound resolution needs a real month number, so it stays English-
+ // only: a French month resolves to nothing rather than silently
+ // becoming January.
+ let fmt = compile_formats(FR_FORMAT).unwrap();
+ assert_eq!(
+ resolve_date_bound_str("mer., 6 déc. 2000 02:55:00 -0800", false, Some(&fmt)),
+ Err(DateResolveError::UnparseableValue(
+ "mer., 6 déc. 2000 02:55:00 -0800".to_string()
+ ))
+ );
+ // The English spelling of the same instant does resolve.
+ assert!(matches!(
+ resolve_date_bound_str("Wed, 6 Dec 2000 02:55:00 -0800", false, Some(&fmt)),
+ Ok(Some(_))
+ ));
+ }
+
+ #[test]
+ fn a_format_without_text_fields_gets_no_tolerance() {
+ // The carve-out is scoped to patterns that actually have a textual
+ // month/weekday — nothing else becomes laxer.
+ assert!(!accepts("yyyy-MM-dd", "banana"));
+ assert!(!accepts("strict_date_time", "mer., 6 déc. 2000"));
+ }
+}
+
+#[cfg(test)]
+mod strict_and_lenient_agree {
+ //! `ignore_malformed: false` (reject the document) and
+ //! `ignore_malformed: true` (drop the field) must make the SAME
+ //! malformed/not-malformed call. They didn't: the strict path had its
+ //! own implementation that ended in a blanket `true` for named and
+ //! custom patterns, so asking for stricter handling made the engine
+ //! accept strictly MORE. Both now call
+ //! [`date_value_valid_with_format`]; this pins the values that used to
+ //! show the contradiction.
+
+ use super::*;
+
+ /// Values the old strict path waved through under a declared format.
+ const WAS_WRONGLY_ACCEPTED_WHEN_STRICT: &[(&str, &str)] = &[
+ ("dd/MM/yyyy", "literally anything"),
+ ("dd/MM/yyyy", "2024-01-01"),
+ ("strict_date_time", "not-a-date"),
+ ("basic_date", "nonsense"),
+ ("yyyy-MM-dd HH:mm:ss", "whenever"),
+ ("ordinal_date", "banana"),
+ ];
+
+ #[test]
+ fn strict_no_longer_accepts_what_lenient_rejects() {
+ for (fmt, value) in WAS_WRONGLY_ACCEPTED_WHEN_STRICT {
+ assert!(
+ !date_value_valid_with_format(&serde_json::json!(value), fmt),
+ "`{value}` under `{fmt}` is malformed and must be rejected by \
+ both ingest paths"
+ );
+ }
+ }
+
+ #[test]
+ fn the_two_paths_share_one_predicate() {
+ // Whatever the answer is, it is the same answer — the property that
+ // was violated. Exercised across the whole named-format set with a
+ // spread of value shapes.
+ for name in ES_NAMED_FORMATS {
+ for v in [
+ serde_json::json!("2024-01-01T00:00:00.000Z"),
+ serde_json::json!("garbage"),
+ serde_json::json!(1717527762025i64),
+ serde_json::json!(2024),
+ serde_json::Value::Null,
+ ] {
+ let a = date_value_valid_with_format(&v, name);
+ let b = compile_formats(name)
+ .map(|f| date_value_matches_formats(&v, &f))
+ .unwrap_or(false);
+ assert_eq!(a, b, "format `{name}` disagreed on {v:?}");
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod conformance_corpus_values {
+ //! Every `(declared format, indexed value)` pair that appears in the
+ //! ES-compat YAML corpus under `engine/tests/es-compat-yaml/yaml`.
+ //!
+ //! These matter more than they look. The `ignore_malformed: false` path
+ //! used to accept every string under a named or custom pattern, so it
+ //! never rejected any of these. Now that both ingest paths share one real
+ //! parser, a value here that fails to parse stops being indexed and
+ //! starts being a `document_parsing_exception` — a conformance failure
+ //! and, in production, a rejected document. Two were found this way:
+ //! `strict_date_time` was requiring the `.SSS` that ES makes optional,
+ //! and the French `MMM` month names had no locale tolerance.
+
+ use super::*;
+
+ const CORPUS: &[(&str, &str)] = &[
+ // aggregations/range.yml — the pair that exposed the mandatory-.SSS
+ // bug: real ES accepts these under strict_date_time with no millis.
+ ("strict_date_time||strict_date", "2021-05-01T07:10:00Z"),
+ ("strict_date_time||strict_date", "2021-05-02T08:34:00Z"),
+ ("strict_date_time||strict_date", "2021-05-03T08:36:00Z"),
+ ("strict_date_time||strict_date", "2021-05-04T09:05:00Z"),
+ ("strict_date_time||strict_date", "2021-05-06T09:22:00Z"),
+ ("strict_date_time||strict_date", "2015-01-01"),
+ // aggregations/composite.yml, date_histogram.yml,
+ // ignored_metadata_field.yml
+ ("yyyy-MM-dd HH:mm:ss", "2021-05-01 20:00:00"),
+ ("yyyy-MM-dd HH:mm:ss", "2021-05-01 21:20:00"),
+ ("yyyy-MM-dd HH:mm:ss", "2021-05-01 23:54:00"),
+ // aggregations/date_range.yml — epoch_second, given as strings.
+ ("epoch_second", "28800000000"),
+ ("epoch_second", "315561600000"),
+ ("epoch_second", "631180800000"),
+ // aggregations/range_timezone_bug.yml — 9-digit fraction + offset.
+ (
+ "uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSSZZZZZ",
+ "2021-08-12T01:00:00.000000000+02:00",
+ ),
+ // search/500_date_range.yml — bare `uuuu` years, including 5 digits.
+ ("uuuu", "1900"),
+ ("uuuu", "2022"),
+ ("uuuu", "1500"),
+ ("uuuu", "10000"),
+ // search/390_doc_values_search.yml
+ ("yyyy/MM/dd", "2017/01/01"),
+ ("yyyy/MM/dd", "2017/01/02"),
+ // search/530_ignore_above_stored_source.yml
+ ("yyyy-MM-dd'T'HH:mm:ss", "2017-10-20T03:08:45"),
+ ("yyyy-MM-dd'T'HH:mm:ss", "2017-10-21T07:00:00"),
+ ("yyyy-MM-dd'T'HH:mm:ss", "2017-10-22T01:00:00"),
+ // search/90_search_after.yml, search/630_format_sort_missing_dates.yml
+ ("yyyy-MM-dd HH:mm:ss.SSS", "2019-10-21 00:30:04.828"),
+ ("yyyy-MM-dd HH:mm:ss.SSS", "2021-02-11 08:30:04.828"),
+ ("yyyy-MM-dd HH:mm:ss.SSS", "2021-10-13 00:30:04.828"),
+ ("yyyy-MM-dd HH:mm:ss.SSSSSS", "2019-10-21 00:30:04.828740"),
+ ("yyyy-MM-dd HH:mm:ss.SSSSSS", "2021-06-11 04:30:04.828456"),
+ ("yyyy-MM-dd HH:mm:ss.SSSSSS", "2021-10-13 00:30:04.828123"),
+ ("dd/MM/yyyy HH:mm:ss.SSS", "15/04/2021 06:30:04.821"),
+ ("dd/MM/yyyy HH:mm:ss.SSS", "20/05/2021 05:30:04.832"),
+ ("dd/MM/yyyy HH:mm:ss.SSS", "21/08/2021 03:30:04.732"),
+ // search/180_locale_dependent_mapping.yml — locale: fr
+ (
+ "E, d MMM yyyy HH:mm:ss Z",
+ "mer., 6 déc. 2000 02:55:00 -0800",
+ ),
+ (
+ "E, d MMM yyyy HH:mm:ss Z",
+ "jeu., 7 déc. 2000 02:55:00 -0800",
+ ),
+ // search/140_pre_filter_search_shards.yml
+ ("yyyy-MM-dd", "2015-01-01"),
+ ("yyyy-MM-dd", "2016-02-01"),
+ ];
+
+ #[test]
+ fn no_corpus_value_is_rejected() {
+ for (fmt, value) in CORPUS {
+ assert!(
+ date_value_valid_with_format(&serde_json::json!(value), fmt),
+ "`{value}` under declared format `{fmt}` would now be dropped \
+ (ignore_malformed: true) or rejected outright \
+ (ignore_malformed: false) — real ES indexes it"
+ );
+ }
+ }
+
+ #[test]
+ fn strict_date_time_millis_are_optional_both_ways() {
+ // ES builds strict_date_time's fraction with optionalStart(), so
+ // both spellings are valid; requiring `.SSS` rejected real data.
+ for v in [
+ "2021-05-01T07:10:00Z",
+ "2021-05-01T07:10:00.123Z",
+ "2021-05-01T07:10:00+02:00",
+ "2021-05-01T07:10:00.123+02:00",
+ ] {
+ assert!(
+ date_value_valid_with_format(&serde_json::json!(v), "strict_date_time"),
+ "strict_date_time rejected `{v}`"
+ );
+ }
+ // The offset is still required, and the seconds still are too.
+ assert!(!date_value_valid_with_format(
+ &serde_json::json!("2021-05-01T07:10:00"),
+ "strict_date_time"
+ ));
+ assert!(!date_value_valid_with_format(
+ &serde_json::json!("2021-05-01T07:10Z"),
+ "strict_date_time"
+ ));
+ }
+
+ #[test]
+ fn optional_sections_are_builtin_only() {
+ // `[` keeps its old literal meaning in a user-supplied pattern.
+ let toks = compile_pattern_in("yyyy[MM", PatternDialect::User).unwrap();
+ assert!(toks
+ .iter()
+ .any(|t| matches!(t, PatTok::Literal(l) if l == "[")));
+ assert!(date_value_valid_with_format(
+ &serde_json::json!("2024[01"),
+ "yyyy[MM"
+ ));
+ }
+}
+
+#[cfg(test)]
+mod ignored_metadata_field_oracle {
+ //! `aggregations/ignored_metadata_field.yml` is an ES fixture that
+ //! asserts an exact `_ignored` count, so it states, per value, whether
+ //! real Elasticsearch considers it malformed. That makes it a ready-made
+ //! truth table for the `ignore_malformed: true` path — the one the
+ //! regression under repair silently emptied.
+
+ use super::*;
+
+ fn valid(fmt: &str, v: &str) -> bool {
+ date_value_valid_with_format(&serde_json::json!(v), fmt)
+ }
+
+ /// `date_of_birth`, declared `format: "dd-MM-yyyy"`.
+ #[test]
+ fn date_of_birth_dd_mm_yyyy() {
+ for good in [
+ "12-03-1990",
+ "15-05-1991",
+ "01-09-1994",
+ "05-06-1989",
+ "16-11-1990",
+ "18-12-1992",
+ ] {
+ assert!(valid("dd-MM-yyyy", good), "`{good}` should be indexed");
+ }
+ for bad in [
+ "12-03-1990 12:30:45", // trailing time
+ "19-12-90", // 2-digit year
+ "20/03/1992", // wrong separator
+ "02311988", // no separators
+ "17.15.1990", // wrong separator + month 15
+ ] {
+ assert!(!valid("dd-MM-yyyy", bad), "`{bad}` should be _ignored");
+ }
+ }
+
+ /// `order_datetime`, declared `format: "yyyy-MM-dd HH:mm:ss"`.
+ #[test]
+ fn order_datetime_yyyy_mm_dd_hh_mm_ss() {
+ for good in [
+ "2021-05-01 20:01:37",
+ "2021-05-03 19:38:22",
+ "2021-05-01 20:05:37",
+ ] {
+ assert!(
+ valid("yyyy-MM-dd HH:mm:ss", good),
+ "`{good}` should be indexed"
+ );
+ }
+ for bad in [
+ "2021-05-02", // date only
+ "2021-05-01-20:01:37", // '-' where ' ' belongs
+ "20210501 20:01:37", // no date separators
+ "2021-05-01 20:01:37.123", // trailing millis
+ "2021-05-03 20:01", // no seconds
+ "2021-05-03 20-01-55", // '-' time separators
+ "2021-05-01T20:02:00", // 'T' where ' ' belongs
+ ] {
+ assert!(
+ !valid("yyyy-MM-dd HH:mm:ss", bad),
+ "`{bad}` should be _ignored"
+ );
+ }
+ }
+}