AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): memtable match/match_phrase tokenize with the analyzer the segment uses (#1280)
Tom Villani92fdec61228d
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex a61f96e5b..2c996559f 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -44831,8 +44831,9 @@ fn geo_coord(v: &Value) -> Option<f64> {/// that issue #230 names as the standing invariant; and the pre-flush answer/// was also the wrong one (ES's standard analyzer keeps `don't` whole).///-/// Callers pass field text that upstream has already lowercased; the-/// pipeline's lowercase filter is idempotent, so that is harmless.+/// `match` (both sides) and `match_phrase` (query side) tokenize with it too+/// (#1280), so they pass raw text; some callers pass text that upstream has+/// already lowercased, which the idempotent lowercase filter makes harmless.fn phrase_tokens(text: &str) -> Vec<String> {static STANDARD: std::sync::OnceLock<std::sync::Arc<xerj_fts::analyzer::AnalyzerPipeline>> =std::sync::OnceLock::new();@@ -45565,9 +45566,14 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo// `whitespace` analyzer splits only on ASCII whitespace.query.split_whitespace().map(str::to_string).collect()} else {- query- .to_lowercase()- .split(|c: char| !c.is_alphanumeric())+ // The standard analyzer, as on the segment side (#1280). A+ // `!is_alphanumeric()` split is not UAX#29: it keeps a run of+ // Han ideographs as ONE token where the analyzer emits one per+ // character, so a CJK `match` scanned here matched nothing+ // until `_refresh`; and it splits `foo_bar`, `don't` and+ // `3.14`, so `foo` matched here and not after `_refresh`.+ phrase_tokens(query)+ .into_iter().filter(|w| {w.len() >= 2|| w.chars()@@ -45575,7 +45581,6 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo.map(|c| c.is_ascii_digit()).unwrap_or(false)})- .map(|w| w.to_string()).collect()};if q_tokens.is_empty() {@@ -45583,15 +45588,19 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo}// Token-level match (ES standard-analyzer semantics): tokenize- // the field value on non-alphanumeric boundaries, then check- // token equality — NOT substring. Using substring would make- // `match(jump)` match "jumparound" which ES does not.+ // the field value with the same analyzer as the query, then+ // check token equality — NOT substring. Using substring would+ // make `match(jump)` match "jumparound" which ES does not.let tokenize = |s: &str| -> Vec<String> {- fold(s)- .split(|c: char| !c.is_alphanumeric())- .filter(|t| !t.is_empty())- .map(str::to_string)- .collect()+ if preserve_case {+ fold(s)+ .split(|c: char| !c.is_alphanumeric())+ .filter(|t| !t.is_empty())+ .map(str::to_string)+ .collect()+ } else {+ phrase_tokens(s)+ }};// How many of the query tokens must be present. `operator:and`// requires all; `operator:or` requires 1 by default, or@@ -45841,12 +45850,10 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> boQueryNode::MatchPhrase {field, query, slop, ..} => {- let query_tokens: Vec<String> = query- .to_lowercase()- .split(|c: char| !c.is_alphanumeric())- .filter(|t| !t.is_empty())- .map(str::to_string)- .collect();+ // The same analyzer as the field side below (#1280): a split here+ // made `报表` one query term against the field's `报`,`表`, and+ // `3.14` two terms against the field's one.+ let query_tokens: Vec<String> = phrase_tokens(query);// Real ES tokenizes whatever the field holds — a// `boolean`/numeric field's value is just its string form// ("true"/"false", "42") for phrase-matching purposes. Pre-fixdiff --git a/engine/crates/xerj-engine/tests/match_analyzer_flush_parity.rs b/engine/crates/xerj-engine/tests/match_analyzer_flush_parity.rsnew file mode 100644index 000000000..4ec96cdda--- /dev/null+++ b/engine/crates/xerj-engine/tests/match_analyzer_flush_parity.rs@@ -0,0 +1,188 @@+//! Regression tests for issue #1280: a `match` / `match_phrase` that the+//! memtable answers by scanning documents must tokenize with the analyzer the+//! segment uses, so the hit set does not change at `flush()`.+//!+//! Pre-fix the scan split text on `!char::is_alphanumeric()`. That is not the+//! standard tokenizer (UAX#29): a run of Han ideographs stayed ONE token where+//! the analyzer emits one per character, so a Chinese `match` inside a+//! filtered `bool` (which is scan-routed) found nothing until `_refresh`; and+//! `foo_bar`, `don't` and `3.14` were split, so `foo`, `don` and `3` matched+//! before the flush and not after. `match_phrase` already analyzed the field+//! side (#230) but still split the query side.+//!+//! Each test runs every case against the memtable, flushes once, re-runs it+//! against the segment, and asserts the expected hit set in both states.++use std::collections::BTreeSet;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++async fn ids(idx: &Index, q: &Value) -> BTreeSet<String> {+ let req = parse_request(&json!({ "query": q, "size": 50 })).expect("parse_request");+ idx.search(&req)+ .await+ .unwrap()+ .hits+ .iter()+ .map(|h| h.id.clone())+ .collect()+}++async fn assert_flush_parity(idx: &std::sync::Arc<Index>, cases: &[(Value, &[&str], &str)]) {+ let expected: Vec<BTreeSet<String>> = cases+ .iter()+ .map(|(_, exp, _)| exp.iter().map(|s| s.to_string()).collect())+ .collect();+ for ((q, _, label), exp) in cases.iter().zip(&expected) {+ let pre = ids(idx, q).await;+ assert_eq!(+ &pre, exp,+ "{label}: PRE-flush (memtable) hit set wrong for {q}"+ );+ }+ idx.flush().await.unwrap();+ for ((q, _, label), exp) in cases.iter().zip(&expected) {+ let post = ids(idx, q).await;+ assert_eq!(+ &post, exp,+ "{label}: POST-flush (segment) hit set wrong for {q}"+ );+ }+}++async fn seed(engine: &Engine, name: &str) -> std::sync::Arc<Index> {+ engine.create_index(name, Schema::empty()).unwrap();+ let idx = engine.get_index(name).unwrap();+ for (id, text) in [+ ("zh", "普通报表保留七日。"),+ ("ascii", "ERR_SCOPE_MISMATCH means outside scope."),+ ("us", "the foo_bar baz"),+ ("apos", "we don't stop now"),+ ("dot", "notes 3.14 here"),+ ] {+ idx.index_document(Some(id.into()), json!({"text": text, "scope": "synthetic"}))+ .await+ .unwrap();+ }+ idx+}++/// `bool` with a `filter` keeps the clause out of the single-clause unwrap,+/// so the memtable answers it by scanning documents.+fn filtered(clause: Value) -> Value {+ json!({"bool": {"must": [clause], "filter": [{"term": {"scope": "synthetic"}}]}})+}++/// Issue #1280's reproducer and its narrowing: CJK text through the scan.+#[tokio::test]+async fn cjk_match_hit_set_stable_across_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "cjk_parity").await;++ assert_flush_parity(+ &idx,+ &[+ (+ filtered(+ json!({"match": {"text": {"query": "普通报表保留多久?", "operator": "or"}}}),+ ),+ &["zh"],+ "issue #1280 filtered bool",+ ),+ (+ filtered(json!({"match": {"text": "报"}})),+ &["zh"],+ "one ideograph",+ ),+ (+ json!({"match": {"text": {"query": "普通报表", "operator": "and"}}}),+ &["zh"],+ "operator:and (scan-routed without a bool)",+ ),+ (+ json!({"match": {"text": {"query": "普通报表多久", "operator": "and"}}}),+ &[],+ "operator:and with an absent ideograph",+ ),+ (+ filtered(json!({"match_phrase": {"text": "报表"}})),+ &["zh"],+ "phrase, adjacent",+ ),+ (+ filtered(json!({"match_phrase": {"text": "报保"}})),+ &[],+ "phrase, not adjacent",+ ),+ (+ filtered(json!({"match": {"text": "ERR_SCOPE_MISMATCH"}})),+ &["ascii"],+ "ASCII control",+ ),+ ],+ )+ .await;+}++/// The same split also over-matched ASCII: UAX#29 keeps `_`, `'` and an+/// intra-number `.` inside one word.+#[tokio::test]+async fn word_boundary_match_hit_set_stable_across_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = seed(&engine, "uax29_parity").await;++ assert_flush_parity(+ &idx,+ &[+ (+ filtered(json!({"match": {"text": "foo"}})),+ &[],+ "foo vs foo_bar",+ ),+ (+ filtered(json!({"match": {"text": "foo_bar"}})),+ &["us"],+ "foo_bar",+ ),+ (+ filtered(json!({"match": {"text": "don"}})),+ &[],+ "don vs don't",+ ),+ (+ filtered(json!({"match": {"text": "don't"}})),+ &["apos"],+ "don't",+ ),+ (+ filtered(json!({"match": {"text": "release 3"}})),+ &[],+ "3 vs 3.14",+ ),+ (+ filtered(json!({"match": {"text": "3.14"}})),+ &["dot"],+ "3.14",+ ),+ (+ filtered(json!({"match_phrase": {"text": "notes 3.14"}})),+ &["dot"],+ "phrase 3.14",+ ),+ ],+ )+ .await;+}