AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): #423/#406 gate keyword-fuzzy sub-token fallback behind case_insensitive on the DV path
xerj-org6b9c24d60bc5
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 37b0a8131..e23402bb7 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -22091,12 +22091,15 @@ impl Index {// ── KeywordFuzzy global expansion stats (LOSS_BATTLE_PLAN B4) ────// term → (folded Damerau dist, Σ df) over ALL segments'// dictionaries, plus the blended df (max Σdf across expansions —- // ES TopTermsBlendedFreqScoringRewrite). The match predicate is- // the same folded whole-value-or-sub-token test as- // `xerj_fts::search::expand_fuzzy` / the doc-scan fuzzy arm.- // Sub-token-only matches (an intentional xerj leniency ES lacks)- // can carry a whole-value dist > max_edits; their sim clamps to- // ≥ 0 so the extra hits score deterministically.+ // ES TopTermsBlendedFreqScoringRewrite). The match predicate+ // mirrors `xerj_fts::search::expand_fuzzy` / the doc-scan fuzzy+ // arm EXACTLY: whole-value edit distance always, plus the+ // sub-token fallback ONLY when folding (case_insensitive) — the+ // case-sensitive keyword default is whole-term-only on every path+ // (#423/#406). Sub-token-only matches (an intentional xerj+ // leniency ES lacks, fold path only) can carry a whole-value+ // dist > max_edits; their sim clamps to ≥ 0 so the extra hits+ // score deterministically.let fuzzy_stats: Option<(HashMap<String, (usize, u64)>, u64)> = match plan {ScoredPlan::KeywordFuzzy {field,@@ -22117,6 +22120,19 @@ impl Index {if levenshtein_distance(cmp, &q_ref) <= *max_edits {return true;}+ // #423/#406: the sub-token fallback is a FOLD-path leniency+ // ONLY. When case-sensitive (the ES keyword default), the+ // memtable Fuzzy arm (`token_match`) and the FST+ // `expand_fuzzy` both match the whole un-analysed term and+ // nothing else — so this DV path must too, or a+ // separator-containing keyword ("alpha-beta") answers a+ // sub-token query ("beta") after flush but not before+ // (flush-invariance; caught by the PR #675 correctness+ // skeptic). The post-flush sub-token hit was also non-ES:+ // keyword fuzzy compares the whole term.+ if !ci {+ return false;+ }cmp.split(|c: char| !c.is_alphanumeric()).filter(|t| !t.is_empty()).any(|tok| levenshtein_distance(tok, &q_ref) <= *max_edits)diff --git a/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs b/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rsindex 8d54af22b..5df0a2088 100644--- a/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs+++ b/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs@@ -81,3 +81,67 @@ async fn fuzzy_keyword_is_case_sensitive_but_ci_flag_still_folds() {"#423/#406: fuzzy with case_insensitive true still folds");}++#[tokio::test]+async fn fuzzy_keyword_separator_matches_whole_term_only_when_case_sensitive() {+ // #423/#406 (PR #675 correctness skeptic): a case-SENSITIVE keyword fuzzy+ // matches the whole un-analysed term ONLY — the sub-token fallback is a+ // FOLD-path leniency. A separator-containing keyword "alpha-beta" must NOT+ // fuzzy-match the sub-token query "beta" (fuzziness 0) case-sensitively,+ // and — the load-bearing part — must answer the SAME before and after a+ // flush. The DV `ScoredPlan::KeywordFuzzy` expansion used to run the+ // sub-token fallback UNCONDITIONALLY, so it matched only after flush+ // (memtable=0, segment=1). The explicit case_insensitive:true still folds,+ // where the sub-token leniency applies and is itself flush-invariant.+ let dir = TempDir::new().unwrap();+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ let engine = Engine::new(config).expect("engine");++ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("name", FieldType::Keyword));+ engine.create_index("docs", schema).expect("create");+ let idx = engine.get_index("docs").expect("get");+ idx.index_document(Some("d0".to_string()), json!({ "name": "alpha-beta" }))+ .await+ .expect("index");++ let fz = |v: &str| json!({ "fuzzy": { "name": { "value": v, "fuzziness": 0 } } });+ let fz_ci = json!({ "fuzzy": { "name": { "value": "beta", "fuzziness": 0, "case_insensitive": true } } });++ let pre_sub = hits(&engine, fz("beta")).await;+ let pre_whole = hits(&engine, fz("alpha-beta")).await;+ let pre_ci = hits(&engine, fz_ci.clone()).await;+ idx.refresh().await.expect("refresh");+ let post_sub = hits(&engine, fz("beta")).await;+ let post_whole = hits(&engine, fz("alpha-beta")).await;+ let post_ci = hits(&engine, fz_ci.clone()).await;++ assert_eq!(+ pre_sub, post_sub,+ "#423/#406: case-sensitive keyword sub-token fuzzy must be flush-invariant"+ );+ assert_eq!(+ pre_whole, post_whole,+ "#423/#406: case-sensitive keyword whole-term fuzzy must be flush-invariant"+ );+ assert_eq!(+ pre_ci, post_ci,+ "#423/#406: fuzzy case_insensitive must be flush-invariant"+ );++ assert_eq!(+ post_sub, 0,+ "#423/#406: case-sensitive keyword fuzzy matches the whole term only (no sub-token)"+ );+ assert_eq!(+ post_whole, 1,+ "the whole-term case-sensitive fuzzy matches exactly"+ );+ assert_eq!(+ post_ci, 1,+ "#423/#406: case_insensitive folds and keeps the sub-token leniency"+ );+}