AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(aggs): #837 terms include/exclude regex is fully-anchored (was substring) (#838)
xerj-orgca377964093e
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex a8daf239a..9457e1e4b 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -4609,7 +4609,14 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {/// `^` and `$` aren't automatically added — ES matches if the regex/// finds the term *anywhere* unless the pattern starts/ends explicitly.fn term_matches_regex(term: &str, pattern: &str) -> bool {- match regex::Regex::new(pattern) {+ // ES `terms` include/exclude use a FULLY-ANCHORED Lucene regexp: the+ // pattern must match the ENTIRE term, not a substring (#837). Anchoring+ // with `^(?:…)$` makes `include:"a.*"` keep only terms starting with `a`+ // and `include:"ppl"` keep only the exact term `ppl` — matching ES.+ // Without the anchors `is_match` did a substring find, so `a.*` wrongly+ // matched every term containing an `a`.+ let anchored = format!("^(?:{pattern})$");+ match regex::Regex::new(&anchored) {Ok(re) => re.is_match(term),Err(_) => false,}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex c1edbdaa7..0b4994c25 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1898,6 +1898,78 @@ mod flush_publication_recovery_tests {}}+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]+ async fn terms_agg_include_exclude_is_fully_anchored() {+ // #837: terms include/exclude regex must match the WHOLE term (ES+ // fully-anchored Lucene regexp), not a substring. Before the fix+ // `include:"a.*"` matched every term containing an `a`.+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;+ let dir = tempfile::tempdir().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ let engine = crate::Engine::new(config).unwrap();+ let mut schema = Schema::empty();+ schema+ .add_field(FieldConfig::new("g", FieldType::Keyword))+ .unwrap();+ engine.create_index("ie837", schema).unwrap();+ let idx = engine.get_index("ie837").unwrap();+ idx.abort_background_tasks();+ for (id, g) in [+ ("1", "apple"),+ ("2", "apricot"),+ ("3", "banana"),+ ("4", "avocado"),+ ] {+ idx.index_document(Some(id.into()), json!({ "g": g }))+ .await+ .unwrap();+ }+ let run = |idx: std::sync::Arc<crate::Index>, terms: serde_json::Value| async move {+ let req = xerj_query::parse_request(+ &json!({"query":{"match_all":{}},"size":0,"aggs":{"byg":{"terms":terms}}}),+ )+ .unwrap();+ let r = idx.search(&req).await.unwrap();+ let mut ks: Vec<String> = r+ .aggs+ .as_ref()+ .and_then(|a| a["byg"]["buckets"].as_array())+ .map(|bs| {+ bs.iter()+ .filter_map(|b| b["key"].as_str().map(String::from))+ .collect()+ })+ .unwrap_or_default();+ ks.sort();+ ks+ };+ for phase in ["mem", "seg"] {+ // include "a.*": only terms STARTING with a (anchored).+ assert_eq!(+ run(idx.clone(), json!({"field":"g","size":10,"include":"a.*"})).await,+ vec!["apple", "apricot", "avocado"],+ "include a.* ({phase})"+ );+ // exclude "a.*": only banana survives.+ assert_eq!(+ run(idx.clone(), json!({"field":"g","size":10,"exclude":"a.*"})).await,+ vec!["banana"],+ "exclude a.* ({phase})"+ );+ // "ppl" is a substring of apple but not the whole term => no bucket.+ assert!(+ run(idx.clone(), json!({"field":"g","size":10,"include":"ppl"}))+ .await+ .is_empty(),+ "include ppl must be empty ({phase})"+ );+ if phase == "mem" {+ idx.flush().await.unwrap();+ }+ }+ }+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]async fn mustnot_only_prefix_keeps_nonmatching_docs_after_flush() {// #797: a bool with ONLY a must_not (no must/should/filter) containing a