AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): make the log-template, date-encoding and entity elections total
xerj-org4d54e7ff7ebf
diff --git a/engine/crates/xerj-autoindex/src/extract/logs.rs b/engine/crates/xerj-autoindex/src/extract/logs.rsindex bf769b022..a07045f91 100644--- a/engine/crates/xerj-autoindex/src/extract/logs.rs+++ b/engine/crates/xerj-autoindex/src/extract/logs.rs@@ -46,6 +46,45 @@ enum Kind {Syslog,}+impl Kind {+ /// Tie-break rank for the file-level election, lowest wins. Deliberately+ /// the order `parse_kind` tries the templates in, so an ambiguous FILE+ /// resolves the way an ambiguous LINE does — most specific first. The app+ /// template demands an ISO timestamp *and* a level keyword; CLF demands a+ /// quoted request line *and* a 3-digit status; the syslog pattern is by+ /// far the loosest (`Mon dd HH:MM:SS host prog: rest`) and will claim+ /// lines the other two describe better. Every variant has a distinct+ /// rank, which is what makes the ordering total.+ fn specificity(self) -> u8 {+ match self {+ Kind::App => 0,+ Kind::Clf => 1,+ Kind::Syslog => 2,+ }+ }+}++/// Elect the one template the whole file is parsed with.+///+/// `votes` is a `HashMap`, so this comparator has to be TOTAL: anything left+/// tied would be settled by the map's per-instance random hash seed, and the+/// same file would parse differently on every run. That is not a cosmetic+/// difference here — every line that does not match the elected template is+/// demoted to a continuation and glued onto the previous record's `message`,+/// so a flipped election changes which lines are records at all.+///+/// Most matching lines wins; a tie goes to the most specific template, since+/// electing the loose one drops the fields the specific one would have named.+fn elect_kind(votes: &std::collections::HashMap<Kind, usize>) -> Option<Kind> {+ votes+ .iter()+ .min_by(|(a_kind, a_n), (b_kind, b_n)| {+ b_n.cmp(a_n)+ .then_with(|| a_kind.specificity().cmp(&b_kind.specificity()))+ })+ .map(|(k, _)| *k)+}+/// Used by the sniffer: does this line look like any known log template?pub fn probe_line(line: &str) -> bool {parse_kind(line).is_some()@@ -209,7 +248,7 @@ pub fn extract(}seen += 1;if seen >= 200 || votes.values().sum::<usize>() >= 120 {- elected = votes.iter().max_by_key(|(_, v)| **v).map(|(k, _)| *k);+ elected = elect_kind(&votes);}pk.or(elected)}@@ -275,4 +314,94 @@ mod tests {assert_eq!(m["tenant"], serde_json::json!("t-n"));assert_eq!(m["message"], serde_json::json!("Segment merge."));}++ const APP_LINE: &str = r#"2026-03-17 00:00:13,529 INFO [ingest-svc] tenant=t-n user=u-1153 latency_ms=221 msg="Segment merge.""#;+ const CLF_LINE: &str = r#"198.192.155.112 - - [01/Mar/2026:00:00:03 +0000] "GET /api/v1/x HTTP/1.1" 200 20854 "-" "Mozilla/5.0""#;++ /// Every election builds its own `HashMap`, and every fresh map gets its+ /// own random hash seed — so 500 in-process elections over one unchanged+ /// tally sample 500 different iteration orders. `max_by_key` settled a tie+ /// by whichever of them came last.+ #[test]+ fn a_tied_template_election_elects_the_most_specific_template_every_run() {+ for _ in 0..500 {+ let votes: std::collections::HashMap<Kind, usize> =+ [(Kind::Clf, 7), (Kind::App, 7), (Kind::Syslog, 7)]+ .into_iter()+ .collect();+ assert_eq!(+ elect_kind(&votes),+ Some(Kind::App),+ "a three-way tie goes to the app template, the most constrained one"+ );++ let votes: std::collections::HashMap<Kind, usize> =+ [(Kind::Syslog, 5), (Kind::Clf, 5)].into_iter().collect();+ assert_eq!(+ elect_kind(&votes),+ Some(Kind::Clf),+ "syslog is the loosest template and never wins a tie"+ );++ let votes: std::collections::HashMap<Kind, usize> =+ [(Kind::Syslog, 6), (Kind::App, 5), (Kind::Clf, 5)]+ .into_iter()+ .collect();+ assert_eq!(+ elect_kind(&votes),+ Some(Kind::Syslog),+ "specificity is only the tie-break; a real majority still wins"+ );++ assert_eq!(elect_kind(&Default::default()), None);+ }+ }++ /// The end-to-end symptom: a file whose first 120 parsed lines split 60/60+ /// between two templates. Whichever kind is elected, every later line that+ /// does not match it is demoted to a continuation, so the flip changed the+ /// RECORD COUNT of an unchanged file from run to run.+ #[test]+ fn a_file_tied_between_two_templates_parses_identically_every_run() {+ let dir = tempfile::tempdir().unwrap();+ let path = dir.path().join("tied.log");+ let mut text = String::new();+ for i in 0..120 {+ text.push_str(if i % 2 == 0 { APP_LINE } else { CLF_LINE });+ text.push('\n');+ }+ // Read after the election has been settled by the 120-line tie.+ for _ in 0..10 {+ text.push_str(CLF_LINE);+ text.push('\n');+ }+ std::fs::write(&path, &text).unwrap();++ for _ in 0..200 {+ let mut recs = Vec::new();+ let stats = extract(&path, false, None, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ assert_eq!(+ stats.records, 120,+ "the app template is elected, so the 10 trailing CLF lines are \+ continuations of record 120, not 10 records of their own"+ );+ assert_eq!(recs[0].fields["level"], serde_json::json!("INFO"));+ assert_eq!(+ recs[119].fields["ip"],+ serde_json::json!("198.192.155.112"),+ "lines parsed before the election keep their own template"+ );+ let tail = recs[119].fields["message"].as_str().unwrap();+ assert_eq!(+ tail.lines().count(),+ 11,+ "a CLF record has no message of its own, so the field is the \+ empty seed plus the 10 absorbed continuation lines"+ );+ }+ }}diff --git a/engine/crates/xerj-autoindex/src/infer/entities.rs b/engine/crates/xerj-autoindex/src/infer/entities.rsindex 8729c11db..db7d68489 100644--- a/engine/crates/xerj-autoindex/src/infer/entities.rs+++ b/engine/crates/xerj-autoindex/src/infer/entities.rs@@ -21,6 +21,21 @@ impl Entity {Entity::Uuid => "uuid",}}++ /// Tie-break rank for a field-level election, lowest wins. Kept in the+ /// order `classify` below tries the tests in — most specific first — so an+ /// ambiguous FIELD resolves the way an ambiguous VALUE does: an exact+ /// `IpAddr` parse, then the fixed 36-char uuid shape, then email, then the+ /// loosest test, url. Distinct per variant, which is what makes an+ /// ordering built on it total. Change this only alongside `classify`.+ pub fn precedence(&self) -> u8 {+ match self {+ Entity::Ip => 0,+ Entity::Uuid => 1,+ Entity::Email => 2,+ Entity::Url => 3,+ }+ }}struct Res {diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rsindex 4f22ca52d..6491db9e1 100644--- a/engine/crates/xerj-autoindex/src/infer/mod.rs+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs@@ -334,6 +334,52 @@ fn p95(samples: &[u32]) -> u32 {v[((v.len() - 1) * 95) / 100]}+/// Elect the date encoding recorded in the field's mapping.+///+/// `date_hits` is a `HashMap`, so this comparator has to be TOTAL: anything+/// left tied would be settled by the map's per-instance random hash seed, and+/// the same sample would write a different `date_enc` on every run — which+/// then reaches the catalog and the coercion plan (`coerce::plan_from_specs`),+/// where the elected encoding decides how bare integers are read.+///+/// Most evidence wins; a tie goes to the lowest `DateEnc` in declaration+/// order. That order is not arbitrary: it is the order `parse_date_str` tries+/// the encodings in, richest first — rfc3339 carries an explicit zone,+/// iso-naive and space-naive carry a time we have to assume is UTC, date-only+/// carries no time at all. On a genuine 50/50 split (a field mixing+/// `2026-03-17T00:00:13` with `2026-03-17 00:00:13`) we name the encoding that+/// concedes the least. It is also the order `date_evidence` is sorted in, so+/// the elected encoding is always the first of the tied ones listed there.+fn elect_date_enc(date_hits: &HashMap<DateEnc, u64>) -> Option<DateEnc> {+ date_hits+ .iter()+ .min_by(|(a_enc, a_n), (b_enc, b_n)| b_n.cmp(a_n).then_with(|| a_enc.cmp(b_enc)))+ .map(|(k, _)| *k)+}++/// Elect the field's semantic entity tag (keyword + `semantic`).+///+/// `entity` is a `HashMap`, so the comparator has to be TOTAL for the same+/// reason as above. Note that a tie cannot change TODAY's verdict: the caller+/// demands the winner hold ≥90% of the field's values, and two entities each+/// holding ≥90% of the same sample is arithmetically impossible. That safety+/// is an accident of one constant, not a property of this election — relax the+/// supermajority to a plurality and the hash seed picks the mapping. Ordering+/// it totally costs nothing and does not wait for that edit.+///+/// Most values wins; a tie goes to the most specific classifier, so a field+/// split between shapes is tagged the way `entities::classify` tags a value+/// that could be read as either.+fn elect_entity(entity: &HashMap<Entity, u64>) -> Option<Entity> {+ entity+ .iter()+ .min_by(|(a_ent, a_n), (b_ent, b_n)| {+ b_n.cmp(a_n)+ .then_with(|| a_ent.precedence().cmp(&b_ent.precedence()))+ })+ .map(|(k, _)| *k)+}+/// Infer the full field spec list for a dataset. `records` = sampled record/// count (for null ratios). Two passes so epoch candidates can corroborate/// against elected date fields.@@ -382,12 +428,7 @@ pub fn infer_fields(// string dateslet date_total: u64 = acc.date_hits.values().sum();if acc.str_n > 0 && th95(date_total) && date_total > 0 {- let elected = acc- .date_hits- .iter()- .max_by_key(|(_, v)| **v)- .map(|(k, _)| *k)- .unwrap();+ let elected = elect_date_enc(&acc.date_hits).unwrap();spec.es_type = "date".into();spec.date_enc = Some(elected.as_str().into());let mut ev: Vec<(DateEnc, u64)> = acc.date_hits.iter().map(|(k, v)| (*k, *v)).collect();@@ -445,12 +486,7 @@ pub fn infer_fields(continue;}// strings → entity / keyword / text- let ent = acc- .entity- .iter()- .max_by_key(|(_, v)| **v)- .filter(|(_, v)| **v * 10 >= n * 9 && n >= 20)- .map(|(k, _)| *k);+ let ent = elect_entity(&acc.entity).filter(|e| acc.entity[e] * 10 >= n * 9 && n >= 20);if let Some(e) = ent {spec.es_type = "keyword".into();spec.semantic = Some(e.as_str().into());@@ -574,3 +610,135 @@ pub fn elect_time_field(specs: &[FieldSpec]) -> Option<String> {}).map(|s| s.name.clone())}++#[cfg(test)]+mod tests {+ use super::*;++ /// Every election runs over a freshly built `HashMap`, and every fresh map+ /// gets its own random hash seed — so 500 in-process elections sample 500+ /// different iteration orders. `max_by_key` settled a tie by whichever of+ /// them came last.+ #[test]+ fn a_tied_date_encoding_election_elects_the_richest_encoding_every_run() {+ for _ in 0..500 {+ let hits: HashMap<DateEnc, u64> = [+ (DateEnc::SpaceNaive, 10),+ (DateEnc::IsoNaive, 10),+ (DateEnc::DateOnly, 10),+ ]+ .into_iter()+ .collect();+ assert_eq!(+ elect_date_enc(&hits),+ Some(DateEnc::IsoNaive),+ "of the tied encodings, iso-naive concedes the least"+ );++ let hits: HashMap<DateEnc, u64> = [(DateEnc::Clf, 4), (DateEnc::Rfc2822, 4)]+ .into_iter()+ .collect();+ assert_eq!(elect_date_enc(&hits), Some(DateEnc::Clf));++ let hits: HashMap<DateEnc, u64> = [(DateEnc::DateOnly, 9), (DateEnc::Rfc3339, 8)]+ .into_iter()+ .collect();+ assert_eq!(+ elect_date_enc(&hits),+ Some(DateEnc::DateOnly),+ "declaration order is only the tie-break; more evidence still wins"+ );++ assert_eq!(elect_date_enc(&HashMap::new()), None);+ }+ }++ /// The end-to-end symptom: one unchanged sample, split 50/50 between two+ /// string encodings, must always write the same `date_enc` into the spec —+ /// that string reaches the catalog and `coerce::plan_from_specs`.+ #[test]+ fn a_field_tied_between_two_date_encodings_maps_identically_every_run() {+ for _ in 0..200 {+ let mut acc = FieldAcc::default();+ for i in 0..10 {+ acc.add(&Value::String(format!("2026-03-{:02}T00:00:13", i + 1)));+ acc.add(&Value::String(format!("2026-03-{:02} 00:00:13", i + 1)));+ }+ assert_eq!(acc.date_hits[&DateEnc::IsoNaive], 10);+ assert_eq!(acc.date_hits[&DateEnc::SpaceNaive], 10);++ let mut fields = HashMap::new();+ fields.insert("ts".to_string(), acc);+ let specs = infer_fields(&fields, 20, true);+ assert_eq!(specs.len(), 1);+ assert_eq!(specs[0].es_type, "date");+ assert_eq!(+ specs[0].date_enc.as_deref(),+ Some(DateEnc::IsoNaive.as_str())+ );+ assert_eq!(+ specs[0].date_evidence,+ [+ "iso-naive (assumed UTC): 10".to_string(),+ "yyyy-mm-dd hh:mm:ss (assumed UTC): 10".to_string(),+ ],+ "the elected encoding is the first of the tied ones listed"+ );+ }+ }++ #[test]+ fn a_tied_entity_election_elects_the_most_specific_entity_every_run() {+ for _ in 0..500 {+ let votes: HashMap<Entity, u64> = [+ (Entity::Url, 6),+ (Entity::Uuid, 6),+ (Entity::Ip, 6),+ (Entity::Email, 6),+ ]+ .into_iter()+ .collect();+ assert_eq!(+ elect_entity(&votes),+ Some(Entity::Ip),+ "an exact IpAddr parse is the most specific classifier"+ );++ let votes: HashMap<Entity, u64> =+ [(Entity::Url, 3), (Entity::Email, 3)].into_iter().collect();+ assert_eq!(elect_entity(&votes), Some(Entity::Email));++ let votes: HashMap<Entity, u64> =+ [(Entity::Url, 7), (Entity::Ip, 6)].into_iter().collect();+ assert_eq!(+ elect_entity(&votes),+ Some(Entity::Url),+ "precedence is only the tie-break; more values still wins"+ );++ assert_eq!(elect_entity(&HashMap::new()), None);+ }+ }++ /// Pins the reason the entity tie is latent rather than live: the ≥90%+ /// supermajority gate rejects any tied field, so the winner never reaches+ /// the spec. If this test starts failing because the gate was relaxed, the+ /// total ordering in `elect_entity` is the only thing keeping the mapping+ /// stable — do not remove it.+ #[test]+ fn a_tied_entity_field_is_rejected_by_the_supermajority_gate() {+ let mut acc = FieldAcc::default();+ for i in 0..15u32 {+ acc.add(&Value::String(format!("10.0.0.{i}")));+ acc.add(&Value::String(format!(+ "741e7b6b-dbd2-4a7f-93a9-4ba50fb561{i:02}"+ )));+ }+ assert_eq!(acc.entity[&Entity::Ip], 15);+ assert_eq!(acc.entity[&Entity::Uuid], 15);+ let mut fields = HashMap::new();+ fields.insert("addr".to_string(), acc);+ let specs = infer_fields(&fields, 30, true);+ assert_eq!(specs[0].semantic, None, "neither side holds 90% of 30");+ }+}