Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): #423/#406 gate keyword-fuzzy sub-token fallback behind case_insensitive on the DV path

xerj-org6b9c24d60bc5
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 37b0a8131..e23402bb7 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -22091,12 +22091,15 @@ impl Index {
// ── KeywordFuzzy global expansion stats (LOSS_BATTLE_PLAN B4) ────
// term → (folded Damerau dist, Σ df) over ALL segments'
// dictionaries, plus the blended df (max Σdf across expansions —
- // ES TopTermsBlendedFreqScoringRewrite). The match predicate is
- // the same folded whole-value-or-sub-token test as
- // `xerj_fts::search::expand_fuzzy` / the doc-scan fuzzy arm.
- // Sub-token-only matches (an intentional xerj leniency ES lacks)
- // can carry a whole-value dist > max_edits; their sim clamps to
- // ≥ 0 so the extra hits score deterministically.
+ // ES TopTermsBlendedFreqScoringRewrite). The match predicate
+ // mirrors `xerj_fts::search::expand_fuzzy` / the doc-scan fuzzy
+ // arm EXACTLY: whole-value edit distance always, plus the
+ // sub-token fallback ONLY when folding (case_insensitive) — the
+ // case-sensitive keyword default is whole-term-only on every path
+ // (#423/#406). Sub-token-only matches (an intentional xerj
+ // leniency ES lacks, fold path only) can carry a whole-value
+ // dist > max_edits; their sim clamps to ≥ 0 so the extra hits
+ // score deterministically.
let fuzzy_stats: Option<(HashMap<String, (usize, u64)>, u64)> = match plan {
ScoredPlan::KeywordFuzzy {
field,
@@ -22117,6 +22120,19 @@ impl Index {
if levenshtein_distance(cmp, &q_ref) <= *max_edits {
return true;
}
+ // #423/#406: the sub-token fallback is a FOLD-path leniency
+ // ONLY. When case-sensitive (the ES keyword default), the
+ // memtable Fuzzy arm (`token_match`) and the FST
+ // `expand_fuzzy` both match the whole un-analysed term and
+ // nothing else — so this DV path must too, or a
+ // separator-containing keyword ("alpha-beta") answers a
+ // sub-token query ("beta") after flush but not before
+ // (flush-invariance; caught by the PR #675 correctness
+ // skeptic). The post-flush sub-token hit was also non-ES:
+ // keyword fuzzy compares the whole term.
+ if !ci {
+ return false;
+ }
cmp.split(|c: char| !c.is_alphanumeric())
.filter(|t| !t.is_empty())
.any(|tok| levenshtein_distance(tok, &q_ref) <= *max_edits)
diff --git a/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs b/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs
index 8d54af22b..5df0a2088 100644
--- a/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs
+++ b/engine/crates/xerj-engine/tests/fuzzy_keyword_case_sensitive.rs
@@ -81,3 +81,67 @@ async fn fuzzy_keyword_is_case_sensitive_but_ci_flag_still_folds() {
"#423/#406: fuzzy with case_insensitive true still folds"
);
}
+
+#[tokio::test]
+async fn fuzzy_keyword_separator_matches_whole_term_only_when_case_sensitive() {
+ // #423/#406 (PR #675 correctness skeptic): a case-SENSITIVE keyword fuzzy
+ // matches the whole un-analysed term ONLY — the sub-token fallback is a
+ // FOLD-path leniency. A separator-containing keyword "alpha-beta" must NOT
+ // fuzzy-match the sub-token query "beta" (fuzziness 0) case-sensitively,
+ // and — the load-bearing part — must answer the SAME before and after a
+ // flush. The DV `ScoredPlan::KeywordFuzzy` expansion used to run the
+ // sub-token fallback UNCONDITIONALLY, so it matched only after flush
+ // (memtable=0, segment=1). The explicit case_insensitive:true still folds,
+ // where the sub-token leniency applies and is itself flush-invariant.
+ let dir = TempDir::new().unwrap();
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ let engine = Engine::new(config).expect("engine");
+
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("name", FieldType::Keyword));
+ engine.create_index("docs", schema).expect("create");
+ let idx = engine.get_index("docs").expect("get");
+ idx.index_document(Some("d0".to_string()), json!({ "name": "alpha-beta" }))
+ .await
+ .expect("index");
+
+ let fz = |v: &str| json!({ "fuzzy": { "name": { "value": v, "fuzziness": 0 } } });
+ let fz_ci = json!({ "fuzzy": { "name": { "value": "beta", "fuzziness": 0, "case_insensitive": true } } });
+
+ let pre_sub = hits(&engine, fz("beta")).await;
+ let pre_whole = hits(&engine, fz("alpha-beta")).await;
+ let pre_ci = hits(&engine, fz_ci.clone()).await;
+ idx.refresh().await.expect("refresh");
+ let post_sub = hits(&engine, fz("beta")).await;
+ let post_whole = hits(&engine, fz("alpha-beta")).await;
+ let post_ci = hits(&engine, fz_ci.clone()).await;
+
+ assert_eq!(
+ pre_sub, post_sub,
+ "#423/#406: case-sensitive keyword sub-token fuzzy must be flush-invariant"
+ );
+ assert_eq!(
+ pre_whole, post_whole,
+ "#423/#406: case-sensitive keyword whole-term fuzzy must be flush-invariant"
+ );
+ assert_eq!(
+ pre_ci, post_ci,
+ "#423/#406: fuzzy case_insensitive must be flush-invariant"
+ );
+
+ assert_eq!(
+ post_sub, 0,
+ "#423/#406: case-sensitive keyword fuzzy matches the whole term only (no sub-token)"
+ );
+ assert_eq!(
+ post_whole, 1,
+ "the whole-term case-sensitive fuzzy matches exactly"
+ );
+ assert_eq!(
+ post_ci, 1,
+ "#423/#406: case_insensitive folds and keeps the sub-token leniency"
+ );
+}