Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): make the log-template, date-encoding and entity elections total

xerj-org4d54e7ff7ebf
diff --git a/engine/crates/xerj-autoindex/src/extract/logs.rs b/engine/crates/xerj-autoindex/src/extract/logs.rs
index bf769b022..a07045f91 100644
--- a/engine/crates/xerj-autoindex/src/extract/logs.rs
+++ b/engine/crates/xerj-autoindex/src/extract/logs.rs
@@ -46,6 +46,45 @@ enum Kind {
Syslog,
}
+impl Kind {
+ /// Tie-break rank for the file-level election, lowest wins. Deliberately
+ /// the order `parse_kind` tries the templates in, so an ambiguous FILE
+ /// resolves the way an ambiguous LINE does — most specific first. The app
+ /// template demands an ISO timestamp *and* a level keyword; CLF demands a
+ /// quoted request line *and* a 3-digit status; the syslog pattern is by
+ /// far the loosest (`Mon dd HH:MM:SS host prog: rest`) and will claim
+ /// lines the other two describe better. Every variant has a distinct
+ /// rank, which is what makes the ordering total.
+ fn specificity(self) -> u8 {
+ match self {
+ Kind::App => 0,
+ Kind::Clf => 1,
+ Kind::Syslog => 2,
+ }
+ }
+}
+
+/// Elect the one template the whole file is parsed with.
+///
+/// `votes` is a `HashMap`, so this comparator has to be TOTAL: anything left
+/// tied would be settled by the map's per-instance random hash seed, and the
+/// same file would parse differently on every run. That is not a cosmetic
+/// difference here — every line that does not match the elected template is
+/// demoted to a continuation and glued onto the previous record's `message`,
+/// so a flipped election changes which lines are records at all.
+///
+/// Most matching lines wins; a tie goes to the most specific template, since
+/// electing the loose one drops the fields the specific one would have named.
+fn elect_kind(votes: &std::collections::HashMap<Kind, usize>) -> Option<Kind> {
+ votes
+ .iter()
+ .min_by(|(a_kind, a_n), (b_kind, b_n)| {
+ b_n.cmp(a_n)
+ .then_with(|| a_kind.specificity().cmp(&b_kind.specificity()))
+ })
+ .map(|(k, _)| *k)
+}
+
/// Used by the sniffer: does this line look like any known log template?
pub fn probe_line(line: &str) -> bool {
parse_kind(line).is_some()
@@ -209,7 +248,7 @@ pub fn extract(
}
seen += 1;
if seen >= 200 || votes.values().sum::<usize>() >= 120 {
- elected = votes.iter().max_by_key(|(_, v)| **v).map(|(k, _)| *k);
+ elected = elect_kind(&votes);
}
pk.or(elected)
}
@@ -275,4 +314,94 @@ mod tests {
assert_eq!(m["tenant"], serde_json::json!("t-n"));
assert_eq!(m["message"], serde_json::json!("Segment merge."));
}
+
+ const APP_LINE: &str = r#"2026-03-17 00:00:13,529 INFO [ingest-svc] tenant=t-n user=u-1153 latency_ms=221 msg="Segment merge.""#;
+ const CLF_LINE: &str = r#"198.192.155.112 - - [01/Mar/2026:00:00:03 +0000] "GET /api/v1/x HTTP/1.1" 200 20854 "-" "Mozilla/5.0""#;
+
+ /// Every election builds its own `HashMap`, and every fresh map gets its
+ /// own random hash seed — so 500 in-process elections over one unchanged
+ /// tally sample 500 different iteration orders. `max_by_key` settled a tie
+ /// by whichever of them came last.
+ #[test]
+ fn a_tied_template_election_elects_the_most_specific_template_every_run() {
+ for _ in 0..500 {
+ let votes: std::collections::HashMap<Kind, usize> =
+ [(Kind::Clf, 7), (Kind::App, 7), (Kind::Syslog, 7)]
+ .into_iter()
+ .collect();
+ assert_eq!(
+ elect_kind(&votes),
+ Some(Kind::App),
+ "a three-way tie goes to the app template, the most constrained one"
+ );
+
+ let votes: std::collections::HashMap<Kind, usize> =
+ [(Kind::Syslog, 5), (Kind::Clf, 5)].into_iter().collect();
+ assert_eq!(
+ elect_kind(&votes),
+ Some(Kind::Clf),
+ "syslog is the loosest template and never wins a tie"
+ );
+
+ let votes: std::collections::HashMap<Kind, usize> =
+ [(Kind::Syslog, 6), (Kind::App, 5), (Kind::Clf, 5)]
+ .into_iter()
+ .collect();
+ assert_eq!(
+ elect_kind(&votes),
+ Some(Kind::Syslog),
+ "specificity is only the tie-break; a real majority still wins"
+ );
+
+ assert_eq!(elect_kind(&Default::default()), None);
+ }
+ }
+
+ /// The end-to-end symptom: a file whose first 120 parsed lines split 60/60
+ /// between two templates. Whichever kind is elected, every later line that
+ /// does not match it is demoted to a continuation, so the flip changed the
+ /// RECORD COUNT of an unchanged file from run to run.
+ #[test]
+ fn a_file_tied_between_two_templates_parses_identically_every_run() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("tied.log");
+ let mut text = String::new();
+ for i in 0..120 {
+ text.push_str(if i % 2 == 0 { APP_LINE } else { CLF_LINE });
+ text.push('\n');
+ }
+ // Read after the election has been settled by the 120-line tie.
+ for _ in 0..10 {
+ text.push_str(CLF_LINE);
+ text.push('\n');
+ }
+ std::fs::write(&path, &text).unwrap();
+
+ for _ in 0..200 {
+ let mut recs = Vec::new();
+ let stats = extract(&path, false, None, &mut |r| {
+ recs.push(r);
+ true
+ })
+ .unwrap();
+ assert_eq!(
+ stats.records, 120,
+ "the app template is elected, so the 10 trailing CLF lines are \
+ continuations of record 120, not 10 records of their own"
+ );
+ assert_eq!(recs[0].fields["level"], serde_json::json!("INFO"));
+ assert_eq!(
+ recs[119].fields["ip"],
+ serde_json::json!("198.192.155.112"),
+ "lines parsed before the election keep their own template"
+ );
+ let tail = recs[119].fields["message"].as_str().unwrap();
+ assert_eq!(
+ tail.lines().count(),
+ 11,
+ "a CLF record has no message of its own, so the field is the \
+ empty seed plus the 10 absorbed continuation lines"
+ );
+ }
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/infer/entities.rs b/engine/crates/xerj-autoindex/src/infer/entities.rs
index 8729c11db..db7d68489 100644
--- a/engine/crates/xerj-autoindex/src/infer/entities.rs
+++ b/engine/crates/xerj-autoindex/src/infer/entities.rs
@@ -21,6 +21,21 @@ impl Entity {
Entity::Uuid => "uuid",
}
}
+
+ /// Tie-break rank for a field-level election, lowest wins. Kept in the
+ /// order `classify` below tries the tests in — most specific first — so an
+ /// ambiguous FIELD resolves the way an ambiguous VALUE does: an exact
+ /// `IpAddr` parse, then the fixed 36-char uuid shape, then email, then the
+ /// loosest test, url. Distinct per variant, which is what makes an
+ /// ordering built on it total. Change this only alongside `classify`.
+ pub fn precedence(&self) -> u8 {
+ match self {
+ Entity::Ip => 0,
+ Entity::Uuid => 1,
+ Entity::Email => 2,
+ Entity::Url => 3,
+ }
+ }
}
struct Res {
diff --git a/engine/crates/xerj-autoindex/src/infer/mod.rs b/engine/crates/xerj-autoindex/src/infer/mod.rs
index 4f22ca52d..6491db9e1 100644
--- a/engine/crates/xerj-autoindex/src/infer/mod.rs
+++ b/engine/crates/xerj-autoindex/src/infer/mod.rs
@@ -334,6 +334,52 @@ fn p95(samples: &[u32]) -> u32 {
v[((v.len() - 1) * 95) / 100]
}
+/// Elect the date encoding recorded in the field's mapping.
+///
+/// `date_hits` is a `HashMap`, so this comparator has to be TOTAL: anything
+/// left tied would be settled by the map's per-instance random hash seed, and
+/// the same sample would write a different `date_enc` on every run — which
+/// then reaches the catalog and the coercion plan (`coerce::plan_from_specs`),
+/// where the elected encoding decides how bare integers are read.
+///
+/// Most evidence wins; a tie goes to the lowest `DateEnc` in declaration
+/// order. That order is not arbitrary: it is the order `parse_date_str` tries
+/// the encodings in, richest first — rfc3339 carries an explicit zone,
+/// iso-naive and space-naive carry a time we have to assume is UTC, date-only
+/// carries no time at all. On a genuine 50/50 split (a field mixing
+/// `2026-03-17T00:00:13` with `2026-03-17 00:00:13`) we name the encoding that
+/// concedes the least. It is also the order `date_evidence` is sorted in, so
+/// the elected encoding is always the first of the tied ones listed there.
+fn elect_date_enc(date_hits: &HashMap<DateEnc, u64>) -> Option<DateEnc> {
+ date_hits
+ .iter()
+ .min_by(|(a_enc, a_n), (b_enc, b_n)| b_n.cmp(a_n).then_with(|| a_enc.cmp(b_enc)))
+ .map(|(k, _)| *k)
+}
+
+/// Elect the field's semantic entity tag (keyword + `semantic`).
+///
+/// `entity` is a `HashMap`, so the comparator has to be TOTAL for the same
+/// reason as above. Note that a tie cannot change TODAY's verdict: the caller
+/// demands the winner hold ≥90% of the field's values, and two entities each
+/// holding ≥90% of the same sample is arithmetically impossible. That safety
+/// is an accident of one constant, not a property of this election — relax the
+/// supermajority to a plurality and the hash seed picks the mapping. Ordering
+/// it totally costs nothing and does not wait for that edit.
+///
+/// Most values wins; a tie goes to the most specific classifier, so a field
+/// split between shapes is tagged the way `entities::classify` tags a value
+/// that could be read as either.
+fn elect_entity(entity: &HashMap<Entity, u64>) -> Option<Entity> {
+ entity
+ .iter()
+ .min_by(|(a_ent, a_n), (b_ent, b_n)| {
+ b_n.cmp(a_n)
+ .then_with(|| a_ent.precedence().cmp(&b_ent.precedence()))
+ })
+ .map(|(k, _)| *k)
+}
+
/// Infer the full field spec list for a dataset. `records` = sampled record
/// count (for null ratios). Two passes so epoch candidates can corroborate
/// against elected date fields.
@@ -382,12 +428,7 @@ pub fn infer_fields(
// string dates
let date_total: u64 = acc.date_hits.values().sum();
if acc.str_n > 0 && th95(date_total) && date_total > 0 {
- let elected = acc
- .date_hits
- .iter()
- .max_by_key(|(_, v)| **v)
- .map(|(k, _)| *k)
- .unwrap();
+ let elected = elect_date_enc(&acc.date_hits).unwrap();
spec.es_type = "date".into();
spec.date_enc = Some(elected.as_str().into());
let mut ev: Vec<(DateEnc, u64)> = acc.date_hits.iter().map(|(k, v)| (*k, *v)).collect();
@@ -445,12 +486,7 @@ pub fn infer_fields(
continue;
}
// strings → entity / keyword / text
- let ent = acc
- .entity
- .iter()
- .max_by_key(|(_, v)| **v)
- .filter(|(_, v)| **v * 10 >= n * 9 && n >= 20)
- .map(|(k, _)| *k);
+ let ent = elect_entity(&acc.entity).filter(|e| acc.entity[e] * 10 >= n * 9 && n >= 20);
if let Some(e) = ent {
spec.es_type = "keyword".into();
spec.semantic = Some(e.as_str().into());
@@ -574,3 +610,135 @@ pub fn elect_time_field(specs: &[FieldSpec]) -> Option<String> {
})
.map(|s| s.name.clone())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// Every election runs over a freshly built `HashMap`, and every fresh map
+ /// gets its own random hash seed — so 500 in-process elections sample 500
+ /// different iteration orders. `max_by_key` settled a tie by whichever of
+ /// them came last.
+ #[test]
+ fn a_tied_date_encoding_election_elects_the_richest_encoding_every_run() {
+ for _ in 0..500 {
+ let hits: HashMap<DateEnc, u64> = [
+ (DateEnc::SpaceNaive, 10),
+ (DateEnc::IsoNaive, 10),
+ (DateEnc::DateOnly, 10),
+ ]
+ .into_iter()
+ .collect();
+ assert_eq!(
+ elect_date_enc(&hits),
+ Some(DateEnc::IsoNaive),
+ "of the tied encodings, iso-naive concedes the least"
+ );
+
+ let hits: HashMap<DateEnc, u64> = [(DateEnc::Clf, 4), (DateEnc::Rfc2822, 4)]
+ .into_iter()
+ .collect();
+ assert_eq!(elect_date_enc(&hits), Some(DateEnc::Clf));
+
+ let hits: HashMap<DateEnc, u64> = [(DateEnc::DateOnly, 9), (DateEnc::Rfc3339, 8)]
+ .into_iter()
+ .collect();
+ assert_eq!(
+ elect_date_enc(&hits),
+ Some(DateEnc::DateOnly),
+ "declaration order is only the tie-break; more evidence still wins"
+ );
+
+ assert_eq!(elect_date_enc(&HashMap::new()), None);
+ }
+ }
+
+ /// The end-to-end symptom: one unchanged sample, split 50/50 between two
+ /// string encodings, must always write the same `date_enc` into the spec —
+ /// that string reaches the catalog and `coerce::plan_from_specs`.
+ #[test]
+ fn a_field_tied_between_two_date_encodings_maps_identically_every_run() {
+ for _ in 0..200 {
+ let mut acc = FieldAcc::default();
+ for i in 0..10 {
+ acc.add(&Value::String(format!("2026-03-{:02}T00:00:13", i + 1)));
+ acc.add(&Value::String(format!("2026-03-{:02} 00:00:13", i + 1)));
+ }
+ assert_eq!(acc.date_hits[&DateEnc::IsoNaive], 10);
+ assert_eq!(acc.date_hits[&DateEnc::SpaceNaive], 10);
+
+ let mut fields = HashMap::new();
+ fields.insert("ts".to_string(), acc);
+ let specs = infer_fields(&fields, 20, true);
+ assert_eq!(specs.len(), 1);
+ assert_eq!(specs[0].es_type, "date");
+ assert_eq!(
+ specs[0].date_enc.as_deref(),
+ Some(DateEnc::IsoNaive.as_str())
+ );
+ assert_eq!(
+ specs[0].date_evidence,
+ [
+ "iso-naive (assumed UTC): 10".to_string(),
+ "yyyy-mm-dd hh:mm:ss (assumed UTC): 10".to_string(),
+ ],
+ "the elected encoding is the first of the tied ones listed"
+ );
+ }
+ }
+
+ #[test]
+ fn a_tied_entity_election_elects_the_most_specific_entity_every_run() {
+ for _ in 0..500 {
+ let votes: HashMap<Entity, u64> = [
+ (Entity::Url, 6),
+ (Entity::Uuid, 6),
+ (Entity::Ip, 6),
+ (Entity::Email, 6),
+ ]
+ .into_iter()
+ .collect();
+ assert_eq!(
+ elect_entity(&votes),
+ Some(Entity::Ip),
+ "an exact IpAddr parse is the most specific classifier"
+ );
+
+ let votes: HashMap<Entity, u64> =
+ [(Entity::Url, 3), (Entity::Email, 3)].into_iter().collect();
+ assert_eq!(elect_entity(&votes), Some(Entity::Email));
+
+ let votes: HashMap<Entity, u64> =
+ [(Entity::Url, 7), (Entity::Ip, 6)].into_iter().collect();
+ assert_eq!(
+ elect_entity(&votes),
+ Some(Entity::Url),
+ "precedence is only the tie-break; more values still wins"
+ );
+
+ assert_eq!(elect_entity(&HashMap::new()), None);
+ }
+ }
+
+ /// Pins the reason the entity tie is latent rather than live: the ≥90%
+ /// supermajority gate rejects any tied field, so the winner never reaches
+ /// the spec. If this test starts failing because the gate was relaxed, the
+ /// total ordering in `elect_entity` is the only thing keeping the mapping
+ /// stable — do not remove it.
+ #[test]
+ fn a_tied_entity_field_is_rejected_by_the_supermajority_gate() {
+ let mut acc = FieldAcc::default();
+ for i in 0..15u32 {
+ acc.add(&Value::String(format!("10.0.0.{i}")));
+ acc.add(&Value::String(format!(
+ "741e7b6b-dbd2-4a7f-93a9-4ba50fb561{i:02}"
+ )));
+ }
+ assert_eq!(acc.entity[&Entity::Ip], 15);
+ assert_eq!(acc.entity[&Entity::Uuid], 15);
+ let mut fields = HashMap::new();
+ fields.insert("addr".to_string(), acc);
+ let specs = infer_fields(&fields, 30, true);
+ assert_eq!(specs[0].semantic, None, "neither side holds 90% of 30");
+ }
+}