Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(aggs): #837 terms include/exclude regex is fully-anchored (was substring) (#838)

xerj-orgca377964093e
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index a8daf239a..9457e1e4b 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -4609,7 +4609,14 @@ fn run_top_metrics(params: &Value, docs: &[Value]) -> Value {
/// `^` and `$` aren't automatically added — ES matches if the regex
/// finds the term *anywhere* unless the pattern starts/ends explicitly.
fn term_matches_regex(term: &str, pattern: &str) -> bool {
- match regex::Regex::new(pattern) {
+ // ES `terms` include/exclude use a FULLY-ANCHORED Lucene regexp: the
+ // pattern must match the ENTIRE term, not a substring (#837). Anchoring
+ // with `^(?:…)$` makes `include:"a.*"` keep only terms starting with `a`
+ // and `include:"ppl"` keep only the exact term `ppl` — matching ES.
+ // Without the anchors `is_match` did a substring find, so `a.*` wrongly
+ // matched every term containing an `a`.
+ let anchored = format!("^(?:{pattern})$");
+ match regex::Regex::new(&anchored) {
Ok(re) => re.is_match(term),
Err(_) => false,
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index c1edbdaa7..0b4994c25 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1898,6 +1898,78 @@ mod flush_publication_recovery_tests {
}
}
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+ async fn terms_agg_include_exclude_is_fully_anchored() {
+ // #837: terms include/exclude regex must match the WHOLE term (ES
+ // fully-anchored Lucene regexp), not a substring. Before the fix
+ // `include:"a.*"` matched every term containing an `a`.
+ let _fault_test = FLUSH_FAULT_TEST_LOCK.lock().await;
+ let dir = tempfile::tempdir().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ let engine = crate::Engine::new(config).unwrap();
+ let mut schema = Schema::empty();
+ schema
+ .add_field(FieldConfig::new("g", FieldType::Keyword))
+ .unwrap();
+ engine.create_index("ie837", schema).unwrap();
+ let idx = engine.get_index("ie837").unwrap();
+ idx.abort_background_tasks();
+ for (id, g) in [
+ ("1", "apple"),
+ ("2", "apricot"),
+ ("3", "banana"),
+ ("4", "avocado"),
+ ] {
+ idx.index_document(Some(id.into()), json!({ "g": g }))
+ .await
+ .unwrap();
+ }
+ let run = |idx: std::sync::Arc<crate::Index>, terms: serde_json::Value| async move {
+ let req = xerj_query::parse_request(
+ &json!({"query":{"match_all":{}},"size":0,"aggs":{"byg":{"terms":terms}}}),
+ )
+ .unwrap();
+ let r = idx.search(&req).await.unwrap();
+ let mut ks: Vec<String> = r
+ .aggs
+ .as_ref()
+ .and_then(|a| a["byg"]["buckets"].as_array())
+ .map(|bs| {
+ bs.iter()
+ .filter_map(|b| b["key"].as_str().map(String::from))
+ .collect()
+ })
+ .unwrap_or_default();
+ ks.sort();
+ ks
+ };
+ for phase in ["mem", "seg"] {
+ // include "a.*": only terms STARTING with a (anchored).
+ assert_eq!(
+ run(idx.clone(), json!({"field":"g","size":10,"include":"a.*"})).await,
+ vec!["apple", "apricot", "avocado"],
+ "include a.* ({phase})"
+ );
+ // exclude "a.*": only banana survives.
+ assert_eq!(
+ run(idx.clone(), json!({"field":"g","size":10,"exclude":"a.*"})).await,
+ vec!["banana"],
+ "exclude a.* ({phase})"
+ );
+ // "ppl" is a substring of apple but not the whole term => no bucket.
+ assert!(
+ run(idx.clone(), json!({"field":"g","size":10,"include":"ppl"}))
+ .await
+ .is_empty(),
+ "include ppl must be empty ({phase})"
+ );
+ if phase == "mem" {
+ idx.flush().await.unwrap();
+ }
+ }
+ }
+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn mustnot_only_prefix_keeps_nonmatching_docs_after_flush() {
// #797: a bool with ONLY a must_not (no must/should/filter) containing a