AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(aggs): pin filter match/term/match_phrase analyzer parity (#1745) (#1748)
Gyanu Mayank8855b85474ee
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 1ea3a5ce5..ad585e02c 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -15873,6 +15873,131 @@ mod wildcard_regexp_filter_tests {}}+/// #1745: filter / filters aggs must apply the same analyzer token rules as+/// `_count` for match / term / match_phrase. Cases from #1748+/// (@gyanu2507), run through the mapping-aware path from #1752: only clauses+/// on declared `text` fields are analyzed, while `keyword` values are matched+/// whole, so a multi-word keyword value never matches one of its words.+#[cfg(test)]+mod match_term_phrase_filter_tests {+ use super::run_aggs_with_schema;+ use serde_json::{json, Value};+ use xerj_common::types::{FieldConfig, FieldType, Schema};++ fn schema() -> Schema {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("title", FieldType::Text));+ schema+ .fields+ .push(FieldConfig::new("tag", FieldType::Keyword));+ schema+ }++ fn books() -> Vec<Value> {+ vec![+ json!({"title": "lazy dog sleeps", "tag": "new york"}),+ json!({"title": "Quick dogs run", "tag": "york"}),+ json!({"title": "the fox and the dog", "tag": "New York"}),+ ]+ }++ /// `doc_count` of a single `filter` agg over `books()`.+ fn filter_count(clause: Value) -> Option<u64> {+ let docs = books();+ let out = run_aggs_with_schema(&json!({"f": {"filter": clause}}), &docs, &docs, &schema());+ out["f"]["doc_count"].as_u64()+ }++ #[test]+ fn match_on_text_is_token_equality_not_substring() {+ // "dogs" is its own token: `dog` must not match it.+ assert_eq!(filter_count(json!({"match": {"title": "dog"}})), Some(2));+ assert_eq!(filter_count(json!({"match": {"title": "DOG"}})), Some(2));+ }++ #[test]+ fn match_multi_term_defaults_to_or_and_honours_operator_and() {+ assert_eq!(+ filter_count(json!({"match": {"title": "quick fox"}})),+ Some(2)+ );+ assert_eq!(+ filter_count(json!({"match": {"title": {"query": "quick fox", "operator": "and"}}})),+ Some(0)+ );+ assert_eq!(+ filter_count(json!({"match": {"title": {"query": "fox dog", "operator": "and"}}})),+ Some(1)+ );+ }++ #[test]+ fn match_phrase_requires_adjacency_and_honours_slop() {+ assert_eq!(+ filter_count(json!({"match_phrase": {"title": "fox dog"}})),+ Some(0),+ "fox … and the … dog is not adjacent"+ );+ assert_eq!(+ filter_count(json!({"match_phrase": {"title": "fox and"}})),+ Some(1)+ );+ assert_eq!(+ filter_count(json!({"match_phrase": {"title": "the fox and the dog"}})),+ Some(1)+ );+ // fox is position 1 and dog position 4: two moves are needed.+ assert_eq!(+ filter_count(json!({"match_phrase": {"title": {"query": "fox dog", "slop": 1}}})),+ Some(0)+ );+ assert_eq!(+ filter_count(json!({"match_phrase": {"title": {"query": "fox dog", "slop": 2}}})),+ Some(1)+ );+ }++ #[test]+ fn term_on_keyword_matches_the_whole_value_only() {+ // A multi-word keyword value is one term: `york` must not match+ // "new york", and the comparison is case-sensitive.+ assert_eq!(filter_count(json!({"term": {"tag": "york"}})), Some(1));+ assert_eq!(filter_count(json!({"term": {"tag": "new york"}})), Some(1));+ assert_eq!(filter_count(json!({"term": {"tag": "New York"}})), Some(1));+ assert_eq!(filter_count(json!({"term": {"tag": "new"}})), Some(0));+ assert_eq!(+ filter_count(json!({"terms": {"tag": ["york", "new york"]}})),+ Some(2)+ );+ }++ #[test]+ fn filters_agg_buckets_and_other_bucket_match_issue_1745_counts() {+ let docs = books();+ let out = run_aggs_with_schema(+ &json!({"f": {"filters": {+ "other_bucket_key": "rest",+ "filters": {+ "dog": {"match": {"title": "dog"}},+ "phrase": {"match_phrase": {"title": "fox dog"}},+ "ny": {"term": {"tag": "new york"}},+ }+ }}}),+ &docs,+ &docs,+ &schema(),+ );+ let buckets = &out["f"]["buckets"];+ assert_eq!(buckets["dog"]["doc_count"].as_u64(), Some(2), "{out}");+ assert_eq!(buckets["phrase"]["doc_count"].as_u64(), Some(0), "{out}");+ assert_eq!(buckets["ny"]["doc_count"].as_u64(), Some(1), "{out}");+ // Only "Quick dogs run" (tag `york`) matches none of the three.+ assert_eq!(buckets["rest"]["doc_count"].as_u64(), Some(1), "{out}");+ }+}+// ─────────────────────────────────────────────────────────────────────────────// Capability-manifest drift guard (issue #211)// ─────────────────────────────────────────────────────────────────────────────diff --git a/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.yml b/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.ymlnew file mode 100644index 000000000..285a1d89d--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/aggregations/filters_text_keyword_boundary.yml@@ -0,0 +1,145 @@+# #1745 / #1748: `filter` and `filters` aggregations analyze `match` and+# `match_phrase` on `text` fields like the query does, while a `keyword`+# value is a single term, so `york` never matches inside "new york".+# Every count below is what Elasticsearch returns for the same request.+setup:+ - do:+ indices.create:+ index: places+ body:+ mappings:+ properties:+ title:+ type: text+ tag:+ type: keyword+ - do:+ bulk:+ refresh: true+ body:+ - index:+ _index: places+ _id: "1"+ - title: New York pizza+ tag: new york+ - index:+ _index: places+ _id: "2"+ - title: york minster tour+ tag: york+ - index:+ _index: places+ _id: "3"+ - title: new jersey diner+ tag: new jersey+ - index:+ _index: places+ _id: "4"+ - title: the big apple+ tag: New York++---+"keyword values match whole, text values match by token":++ - do:+ search:+ index: places+ body:+ size: 0+ aggs:+ kw_word:+ filter:+ match:+ tag: york+ kw_phrase:+ filter:+ match:+ tag: new york+ text_word:+ filter:+ match:+ title: york+ text_or:+ filter:+ match:+ title: new york+ text_and:+ filter:+ match:+ title:+ query: new york+ operator: and+ - match: { hits.total.value: 4 }+ - match: { aggregations.kw_word.doc_count: 1 }+ - match: { aggregations.kw_phrase.doc_count: 1 }+ - match: { aggregations.text_word.doc_count: 2 }+ - match: { aggregations.text_or.doc_count: 3 }+ - match: { aggregations.text_and.doc_count: 1 }++---+"term on a keyword value that contains a space":++ - do:+ search:+ index: places+ body:+ size: 0+ aggs:+ f:+ filters:+ other_bucket_key: rest+ filters:+ exact:+ term:+ tag: new york+ other_case:+ term:+ tag: New York+ one_word:+ term:+ tag: new+ either:+ terms:+ tag: [york, new jersey]+ - match: { aggregations.f.buckets.exact.doc_count: 1 }+ - match: { aggregations.f.buckets.other_case.doc_count: 1 }+ - match: { aggregations.f.buckets.one_word.doc_count: 0 }+ - match: { aggregations.f.buckets.either.doc_count: 2 }+ - match: { aggregations.f.buckets.rest.doc_count: 0 }++---+"multi-word match_phrase on a text field":++ - do:+ search:+ index: places+ body:+ size: 0+ aggs:+ two_words:+ filter:+ match_phrase:+ title: new york+ reversed:+ filter:+ match_phrase:+ title: york new+ three_words:+ filter:+ match_phrase:+ title: New York Pizza+ gap:+ filter:+ match_phrase:+ title: new pizza+ gap_slop:+ filter:+ match_phrase:+ title:+ query: new pizza+ slop: 1+ - match: { aggregations.two_words.doc_count: 1 }+ - match: { aggregations.reversed.doc_count: 0 }+ - match: { aggregations.three_words.doc_count: 1 }+ - match: { aggregations.gap.doc_count: 0 }+ - match: { aggregations.gap_slop.doc_count: 1 }