Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(aggs): pin filter match/term/match_phrase analyzer parity (#1745) (#1748)

Gyanu Mayank8855b85474ee
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 1ea3a5ce5..ad585e02c 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -15873,6 +15873,131 @@ mod wildcard_regexp_filter_tests {
}
}
+/// #1745: filter / filters aggs must apply the same analyzer token rules as
+/// `_count` for match / term / match_phrase. Cases from #1748
+/// (@gyanu2507), run through the mapping-aware path from #1752: only clauses
+/// on declared `text` fields are analyzed, while `keyword` values are matched
+/// whole, so a multi-word keyword value never matches one of its words.
+#[cfg(test)]
+mod match_term_phrase_filter_tests {
+ use super::run_aggs_with_schema;
+ use serde_json::{json, Value};
+ use xerj_common::types::{FieldConfig, FieldType, Schema};
+
+ fn schema() -> Schema {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("title", FieldType::Text));
+ schema
+ .fields
+ .push(FieldConfig::new("tag", FieldType::Keyword));
+ schema
+ }
+
+ fn books() -> Vec<Value> {
+ vec![
+ json!({"title": "lazy dog sleeps", "tag": "new york"}),
+ json!({"title": "Quick dogs run", "tag": "york"}),
+ json!({"title": "the fox and the dog", "tag": "New York"}),
+ ]
+ }
+
+ /// `doc_count` of a single `filter` agg over `books()`.
+ fn filter_count(clause: Value) -> Option<u64> {
+ let docs = books();
+ let out = run_aggs_with_schema(&json!({"f": {"filter": clause}}), &docs, &docs, &schema());
+ out["f"]["doc_count"].as_u64()
+ }
+
+ #[test]
+ fn match_on_text_is_token_equality_not_substring() {
+ // "dogs" is its own token: `dog` must not match it.
+ assert_eq!(filter_count(json!({"match": {"title": "dog"}})), Some(2));
+ assert_eq!(filter_count(json!({"match": {"title": "DOG"}})), Some(2));
+ }
+
+ #[test]
+ fn match_multi_term_defaults_to_or_and_honours_operator_and() {
+ assert_eq!(
+ filter_count(json!({"match": {"title": "quick fox"}})),
+ Some(2)
+ );
+ assert_eq!(
+ filter_count(json!({"match": {"title": {"query": "quick fox", "operator": "and"}}})),
+ Some(0)
+ );
+ assert_eq!(
+ filter_count(json!({"match": {"title": {"query": "fox dog", "operator": "and"}}})),
+ Some(1)
+ );
+ }
+
+ #[test]
+ fn match_phrase_requires_adjacency_and_honours_slop() {
+ assert_eq!(
+ filter_count(json!({"match_phrase": {"title": "fox dog"}})),
+ Some(0),
+ "fox … and the … dog is not adjacent"
+ );
+ assert_eq!(
+ filter_count(json!({"match_phrase": {"title": "fox and"}})),
+ Some(1)
+ );
+ assert_eq!(
+ filter_count(json!({"match_phrase": {"title": "the fox and the dog"}})),
+ Some(1)
+ );
+ // fox is position 1 and dog position 4: two moves are needed.
+ assert_eq!(
+ filter_count(json!({"match_phrase": {"title": {"query": "fox dog", "slop": 1}}})),
+ Some(0)
+ );
+ assert_eq!(
+ filter_count(json!({"match_phrase": {"title": {"query": "fox dog", "slop": 2}}})),
+ Some(1)
+ );
+ }
+
+ #[test]
+ fn term_on_keyword_matches_the_whole_value_only() {
+ // A multi-word keyword value is one term: `york` must not match
+ // "new york", and the comparison is case-sensitive.
+ assert_eq!(filter_count(json!({"term": {"tag": "york"}})), Some(1));
+ assert_eq!(filter_count(json!({"term": {"tag": "new york"}})), Some(1));
+ assert_eq!(filter_count(json!({"term": {"tag": "New York"}})), Some(1));
+ assert_eq!(filter_count(json!({"term": {"tag": "new"}})), Some(0));
+ assert_eq!(
+ filter_count(json!({"terms": {"tag": ["york", "new york"]}})),
+ Some(2)
+ );
+ }
+
+ #[test]
+ fn filters_agg_buckets_and_other_bucket_match_issue_1745_counts() {
+ let docs = books();
+ let out = run_aggs_with_schema(
+ &json!({"f": {"filters": {
+ "other_bucket_key": "rest",
+ "filters": {
+ "dog": {"match": {"title": "dog"}},
+ "phrase": {"match_phrase": {"title": "fox dog"}},
+ "ny": {"term": {"tag": "new york"}},
+ }
+ }}}),
+ &docs,
+ &docs,
+ &schema(),
+ );
+ let buckets = &out["f"]["buckets"];
+ assert_eq!(buckets["dog"]["doc_count"].as_u64(), Some(2), "{out}");
+ assert_eq!(buckets["phrase"]["doc_count"].as_u64(), Some(0), "{out}");
+ assert_eq!(buckets["ny"]["doc_count"].as_u64(), Some(1), "{out}");
+ // Only "Quick dogs run" (tag `york`) matches none of the three.
+ assert_eq!(buckets["rest"]["doc_count"].as_u64(), Some(1), "{out}");
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Capability-manifest drift guard (issue #211)
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.yml b/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.yml
new file mode 100644
index 000000000..285a1d89d
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.yml
@@ -0,0 +1,145 @@
+# #1745 / #1748: `filter` and `filters` aggregations analyze `match` and
+# `match_phrase` on `text` fields like the query does, while a `keyword`
+# value is a single term, so `york` never matches inside "new york".
+# Every count below is what Elasticsearch returns for the same request.
+setup:
+ - do:
+ indices.create:
+ index: places
+ body:
+ mappings:
+ properties:
+ title:
+ type: text
+ tag:
+ type: keyword
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - index:
+ _index: places
+ _id: "1"
+ - title: New York pizza
+ tag: new york
+ - index:
+ _index: places
+ _id: "2"
+ - title: york minster tour
+ tag: york
+ - index:
+ _index: places
+ _id: "3"
+ - title: new jersey diner
+ tag: new jersey
+ - index:
+ _index: places
+ _id: "4"
+ - title: the big apple
+ tag: New York
+
+---
+"keyword values match whole, text values match by token":
+
+ - do:
+ search:
+ index: places
+ body:
+ size: 0
+ aggs:
+ kw_word:
+ filter:
+ match:
+ tag: york
+ kw_phrase:
+ filter:
+ match:
+ tag: new york
+ text_word:
+ filter:
+ match:
+ title: york
+ text_or:
+ filter:
+ match:
+ title: new york
+ text_and:
+ filter:
+ match:
+ title:
+ query: new york
+ operator: and
+ - match: { hits.total.value: 4 }
+ - match: { aggregations.kw_word.doc_count: 1 }
+ - match: { aggregations.kw_phrase.doc_count: 1 }
+ - match: { aggregations.text_word.doc_count: 2 }
+ - match: { aggregations.text_or.doc_count: 3 }
+ - match: { aggregations.text_and.doc_count: 1 }
+
+---
+"term on a keyword value that contains a space":
+
+ - do:
+ search:
+ index: places
+ body:
+ size: 0
+ aggs:
+ f:
+ filters:
+ other_bucket_key: rest
+ filters:
+ exact:
+ term:
+ tag: new york
+ other_case:
+ term:
+ tag: New York
+ one_word:
+ term:
+ tag: new
+ either:
+ terms:
+ tag: [york, new jersey]
+ - match: { aggregations.f.buckets.exact.doc_count: 1 }
+ - match: { aggregations.f.buckets.other_case.doc_count: 1 }
+ - match: { aggregations.f.buckets.one_word.doc_count: 0 }
+ - match: { aggregations.f.buckets.either.doc_count: 2 }
+ - match: { aggregations.f.buckets.rest.doc_count: 0 }
+
+---
+"multi-word match_phrase on a text field":
+
+ - do:
+ search:
+ index: places
+ body:
+ size: 0
+ aggs:
+ two_words:
+ filter:
+ match_phrase:
+ title: new york
+ reversed:
+ filter:
+ match_phrase:
+ title: york new
+ three_words:
+ filter:
+ match_phrase:
+ title: New York Pizza
+ gap:
+ filter:
+ match_phrase:
+ title: new pizza
+ gap_slop:
+ filter:
+ match_phrase:
+ title:
+ query: new pizza
+ slop: 1
+ - match: { aggregations.two_words.doc_count: 1 }
+ - match: { aggregations.reversed.doc_count: 0 }
+ - match: { aggregations.three_words.doc_count: 1 }
+ - match: { aggregations.gap.doc_count: 0 }
+ - match: { aggregations.gap_slop.doc_count: 1 }