AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): map RFC 3164 syslog ts as a date (#1691)
xerj-orgc9a3a10d46e3
diff --git a/engine/crates/xerj-autoindex/src/coerce.rs b/engine/crates/xerj-autoindex/src/coerce.rsindex ab640b997..689ddeff0 100644--- a/engine/crates/xerj-autoindex/src/coerce.rs+++ b/engine/crates/xerj-autoindex/src/coerce.rs@@ -26,6 +26,7 @@ pub fn enc_from_str(s: &str) -> Option<DateEnc> {DateEnc::Rfc2822,DateEnc::EpochMillis,DateEnc::EpochSeconds,+ DateEnc::BsdSyslog,].into_iter().find(|e| e.as_str() == s)diff --git a/engine/crates/xerj-autoindex/src/infer/dates.rs b/engine/crates/xerj-autoindex/src/infer/dates.rsindex 197e70c7b..5e4ec4168 100644--- a/engine/crates/xerj-autoindex/src/infer/dates.rs+++ b/engine/crates/xerj-autoindex/src/infer/dates.rs@@ -4,7 +4,7 @@//! (The engine's dynamic mapping has NO date detection — verified — so//! autoindex is the date layer.)-use chrono::{DateTime, NaiveDate, NaiveDateTime, SecondsFormat, TimeZone, Utc};+use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, SecondsFormat, TimeZone, Utc};/// Epoch windows: 1990-01-01 .. 2100-01-01pub const EPOCH_MS_MIN: i64 = 631_152_000_000;@@ -22,6 +22,10 @@ pub enum DateEnc {Rfc2822, // Tue, 01 Jul 2025 10:00:00 +0000EpochMillis, // numericEpochSeconds, // numeric+ /// RFC 3164 timestamp: `Oct 17 22:26:01` / `Oct 5 09:06:58`. No year and+ /// no zone. Appended last so a tie still prefers every encoding above,+ /// which all carry a year.+ BsdSyslog,}impl DateEnc {@@ -35,12 +39,25 @@ impl DateEnc {DateEnc::Rfc2822 => "rfc2822",DateEnc::EpochMillis => "epoch-millis",DateEnc::EpochSeconds => "epoch-seconds",+ // The year is not in the stamp. It is the year of `now` unless+ // that reading is more than 24h in the future, in which case it+ // is the previous year (a December line read in January).+ DateEnc::BsdSyslog => {+ "bsd-syslog (year assumed: current UTC year, or the previous year when the stamp is more than 24h ahead)"+ }}}}-/// Try all string encodings in priority order.+/// Try all string encodings in priority order. A year-less BSD syslog+/// stamp takes its year from [`Utc::now`] — see [`DateEnc::BsdSyslog`].pub fn parse_date_str(s: &str) -> Option<(DateTime<Utc>, DateEnc)> {+ parse_date_str_at(s, Utc::now())+}++/// [`parse_date_str`] with the clock the BSD-syslog year rule reads.+/// Every other encoding ignores `now`.+pub fn parse_date_str_at(s: &str, now: DateTime<Utc>) -> Option<(DateTime<Utc>, DateEnc)> {let t = s.trim();if t.len() < 8 || t.len() > 40 {return None;@@ -88,10 +105,66 @@ pub fn parse_date_str(s: &str) -> Option<(DateTime<Utc>, DateEnc)> {if let Ok(dt) = DateTime::parse_from_rfc2822(t) {return Some((dt.with_timezone(&Utc), DateEnc::Rfc2822));}+ if let Some(dt) = parse_bsd_syslog(t, now) {+ return Some((dt, DateEnc::BsdSyslog));+ }}None}+/// English month abbreviations. Chrono's `%b` follows the process locale, so+/// a stamp of `Oct` must not depend on `LC_TIME`.+fn english_month(name: &str) -> Option<u32> {+ Some(match name {+ "Jan" => 1,+ "Feb" => 2,+ "Mar" => 3,+ "Apr" => 4,+ "May" => 5,+ "Jun" => 6,+ "Jul" => 7,+ "Aug" => 8,+ "Sep" => 9,+ "Oct" => 10,+ "Nov" => 11,+ "Dec" => 12,+ _ => return None,+ })+}++/// `Mmm dd hh:mm:ss` (day space-padded), the whole string and nothing else.+/// Year: `now`'s UTC year, unless that instant is more than 24 hours ahead+/// of `now`, in which case the previous year. A Feb 29 that is not a real+/// day in the chosen year does not parse — inventing a different day would+/// put the line on the wrong date.+fn parse_bsd_syslog(t: &str, now: DateTime<Utc>) -> Option<DateTime<Utc>> {+ let mut parts = t.split_whitespace();+ let month = english_month(parts.next()?)?;+ let day: u32 = parts.next()?.parse().ok()?;+ let time = parts.next()?;+ if parts.next().is_some() || !(1..=31).contains(&day) || time.len() != 8 {+ return None;+ }+ let mut hm = time.split(':');+ let hour: u32 = hm.next()?.parse().ok()?;+ let minute: u32 = hm.next()?.parse().ok()?;+ let second: u32 = hm.next()?.parse().ok()?;+ if hm.next().is_some() {+ return None;+ }+ let at = |year: i32| {+ NaiveDate::from_ymd_opt(year, month, day)?+ .and_hms_opt(hour, minute, second)+ .map(|ndt| Utc.from_utc_datetime(&ndt))+ };+ let year = now.year();+ let dt = at(year)?;+ match now.checked_add_signed(chrono::Duration::hours(24)) {+ Some(limit) if dt > limit => at(year - 1),+ _ => Some(dt),+ }+}+/// Interpret an integer as epoch millis/seconds if within the window.pub fn parse_epoch(n: i64) -> Option<(DateTime<Utc>, DateEnc)> {if (EPOCH_MS_MIN..=EPOCH_MS_MAX).contains(&n) {@@ -181,4 +254,43 @@ mod tests {let (dt, _) = parse_date_str("01/Mar/2026:00:00:03 +0000").unwrap();assert_eq!(to_rfc3339_millis(&dt), "2026-03-01T00:00:03.000Z");}++ fn at(y: i32, mo: u32, d: u32, h: u32, mi: u32, s: u32) -> DateTime<Utc> {+ Utc.with_ymd_and_hms(y, mo, d, h, mi, s).unwrap()+ }++ /// RFC 3164 has no year. Leaving the stamp as keyword makes a time-range+ /// filter miss every line. The year is the clock's year unless that+ /// reading is more than a day ahead (a December line indexed in January).+ #[test]+ fn bsd_syslog_stamps_are_dates_with_an_assumed_year() {+ let now = at(2026, 10, 17, 23, 0, 0);+ let (dt, enc) = parse_date_str_at("Oct 17 22:26:01", now).unwrap();+ assert_eq!(enc, DateEnc::BsdSyslog);+ assert_eq!(to_rfc3339_millis(&dt), "2026-10-17T22:26:01.000Z");++ let (padded, _) = parse_date_str_at("Oct 5 09:06:58", now).unwrap();+ assert_eq!(to_rfc3339_millis(&padded), "2026-10-05T09:06:58.000Z");++ // Read in January: December is more than 24h ahead if taken as+ // this year, so the stamp is last year.+ let jan = at(2026, 1, 2, 0, 0, 0);+ let (rolled, _) = parse_date_str_at("Dec 31 23:59:59", jan).unwrap();+ assert_eq!(to_rfc3339_millis(&rolled), "2025-12-31T23:59:59.000Z");++ // A few minutes of clock skew stays in the current year.+ let early = at(2026, 10, 17, 22, 0, 0);+ let (skew, _) = parse_date_str_at("Oct 17 22:26:01", early).unwrap();+ assert_eq!(to_rfc3339_millis(&skew), "2026-10-17T22:26:01.000Z");++ assert!(parse_date_str_at("Not a date", now).is_none());+ assert!(parse_date_str_at("Oct 17 22:26:01 trailing", now).is_none());+ assert!(parse_date_str_at("Oct 32 00:00:00", now).is_none());+ assert!(+ parse_date_str_at("Tue, 01 Jul 2025 10:00:00 +0000", now)+ .unwrap()+ .1+ != DateEnc::BsdSyslog+ );+ }}diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rsindex 610f1a8b1..8aa8d57d0 100644--- a/engine/crates/xerj-autoindex/src/infer/mod.rs+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs@@ -885,6 +885,28 @@ mod tests {}}+ /// RFC 3164 `ts` (`Oct 17 22:26:01`) has no year. Before it was recognized+ /// the column missed the 95% date threshold and mapped `keyword`, so a+ /// time-range filter matched nothing.+ #[test]+ fn a_bsd_syslog_timestamp_column_maps_as_date() {+ let mut acc = FieldAcc::default();+ for day in 1..=20 {+ acc.add(&Value::String(format!("Oct {day} 09:06:58")));+ }+ assert_eq!(acc.date_hits.get(&DateEnc::BsdSyslog).copied(), Some(20));+ let mut fields = HashMap::new();+ fields.insert("ts".to_string(), acc);+ let specs = infer_fields(&fields, 20, true);+ assert_eq!(specs.len(), 1);+ assert_eq!(specs[0].es_type, "date", "{specs:?}");+ assert_eq!(+ specs[0].date_enc.as_deref(),+ Some(DateEnc::BsdSyslog.as_str())+ );+ assert!(specs[0].date_min.is_some() && specs[0].date_max.is_some());+ }+#[test]fn a_tied_entity_election_elects_the_most_specific_entity_every_run() {for _ in 0..500 {