AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(engine): #668 keyword wildcard honors case_insensitive on all matcher paths
xerj-orgc9eee4a6be20
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 1e5265ac3..caf2757c3 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -22547,28 +22547,34 @@ impl Index {ords: (lo as u32..hi as u32).collect(),});}- ScoredFilterLeaf::KeywordWildcard { field, pattern } => {+ ScoredFilterLeaf::KeywordWildcard {+ field,+ pattern,+ case_insensitive,+ } => {let Some(Column::Keyword(k)) = cols.get(field.as_str()) else {return None;};- // SAME predicate as the brute path (`doc_matches_query`)- // and the FST route (`term_matches_wildcard`): fold BOTH- // sides, whole-value OR sub-token. The old range+match- // here was case-SENSITIVE, so a cased pattern (`K*1`,- // case_insensitive) found nothing in a lowercase- // dictionary — DV-only keyword wildcards returned 0 hits- // from segments while the memtable matched. A folded- // pattern cannot bound a byte-ordered prefix range, so- // walk the whole (distinct-term) dictionary — the same- // cost class as the FST expansion.+ // #668: honor the node's case flag — the SAME predicate+ // the brute path (`doc_matches_query`) and the FST route+ // now apply. `term{case_insensitive:true}` / query_string+ // rewrites fold BOTH sides (whole-value OR sub-token); a+ // standalone `{wildcard}` matches the RAW term+ // case-sensitively (ES default). A folded pattern cannot+ // bound a byte-ordered prefix range, so walk the whole+ // (distinct-term) dictionary — same cost class as the FST.let pat_lc = pattern.to_lowercase();let ords: Vec<u32> = (0..k.terms.len()).filter(|&o| {- let lc = k.terms[o].to_lowercase();- wildcard_match(&lc, &pat_lc)- || lc- .split(|c: char| !c.is_alphanumeric())- .any(|tok| !tok.is_empty() && wildcard_match(tok, &pat_lc))+ if *case_insensitive {+ let lc = k.terms[o].to_lowercase();+ wildcard_match(&lc, &pat_lc)+ || lc.split(|c: char| !c.is_alphanumeric()).any(|tok| {+ !tok.is_empty() && wildcard_match(tok, &pat_lc)+ })+ } else {+ wildcard_match(&k.terms[o], pattern.as_str())+ }}).map(|o| o as u32).collect();@@ -35737,30 +35743,40 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> boQueryNode::Wildcard {field,value: pattern,+ case_insensitive,..} => {- // #396: `wildcard` must mirror its segment path per field type+ // #396/#668: `wildcard` must mirror its segment path per field type// (ES never analyses the pattern):// - analysed TEXT: the segment matches the RAW pattern against the// lower-cased token stream (case_insensitive:false), so an- // uppercase pattern matches nothing — do the same here (this is- // the flush-divergence the KEYWORD-only earlier attempt missed).- // - KEYWORD: the segment doc-values keyword-wildcard path folds- // both sides (case-insensitive), so fold here to stay in- // agreement. ES-correct case-sensitive keyword wildcard needs the- // segment path changed too — tracked in #668.- // - unknown field (schemaless shim caller): the pre-#423 fold-both- // heuristic, byte-for-byte unchanged.- let is_text = matches!(- declared_field(schema, field).map(|f| &f.field_type),- Some(FieldType::Text)- );+ // uppercase pattern matches nothing — do the same here.+ // - KEYWORD, case-sensitive (standalone `{wildcard}`, the ES+ // default): match the RAW value with `wildcard_match` (#668).+ // - KEYWORD, case-insensitive (`term{case_insensitive:true}` /+ // query_string lowerings set the flag): fold both sides.+ // - field absent from the schema (the `EMPTY_SCHEMA` shim callers,+ // or a genuinely-undeclared field): the pre-#423 fold-both+ // heuristic. NB memtable search passes the real, dynamically-grown+ // schema, where an undeclared string is already mapped to `text`+ // and takes the branch above — so this arm is effectively the+ // schemaless-shim path, byte-for-byte unchanged.+ let ft = declared_field(schema, field).map(|f| &f.field_type);+ let is_text = matches!(ft, Some(FieldType::Text));+ // Case-sensitive only for a declared keyword whose node did NOT ask+ // to fold. Unknown fields (ft == None) keep folding (shim parity).+ let keyword_case_sensitive =+ matches!(ft, Some(FieldType::Keyword)) && !*case_insensitive;let pat_lc = pattern.to_lowercase();let matches_str = |s: &str| -> bool {if is_text {s.split(|c: char| !c.is_alphanumeric()).any(|tok| {!tok.is_empty() && wildcard_match(&tok.to_lowercase(), pattern.as_str())})+ } else if keyword_case_sensitive {+ // A keyword is a single un-analysed term: match the whole+ // raw value, case-sensitively (mirrors the segment FST).+ wildcard_match(s, pattern.as_str())} else {let lc = s.to_lowercase();wildcard_match(&lc, &pat_lc)@@ -41052,7 +41068,7 @@ fn query_node_to_fts_with_keyword_fields(value,boost,constant_score,- case_insensitive: _,+ case_insensitive,} => {// `wildcard` on a KEYWORD field: expand the keyword FST term// dictionary to every term matching the pattern (`*`=0+ chars,@@ -41063,16 +41079,17 @@ fn query_node_to_fts_with_keyword_fields(// (≪ docs) instead of a per-doc O(N) scan (~150 ms/100k → ~1 ms).//// Case-insensitivity is REQUIRED, not a shortcut: XERJ's parser- // drops `case_insensitive` and rewrites `term{case_insensitive:true}`- // to a Wildcard, both relying on the matcher folding case — so a- // case-sensitive FST route would break those (passing) paths.+ // rewrites `term{case_insensitive:true}` to a Wildcard, which sets+ // the node's `case_insensitive` flag so this route keeps folding for+ // that (passing) path. A standalone `{wildcard}` sets the flag+ // `false` → the FST route is case-SENSITIVE (ES-correct, #668).// TEXT fields keep the doc-scan (None): analyzed-token semantics.if exact_fields.contains(field.as_str()) {return Some(FtsQuery::Wildcard(xerj_fts::search::WildcardQuery {field: field.clone(),pattern: value.clone(),boost: boost.unwrap_or(1.0),- case_insensitive: true,+ case_insensitive: *case_insensitive,// ES rewrites a standalone `wildcard` query to a// constant_score. The `term{case_insensitive}` /// query_string lowerings that SHARE this node pass `false`@@ -41765,7 +41782,13 @@ enum ScoredFilterLeaf {/// per segment by matching the (distinct) dictionary terms with the/// SAME `wildcard_match` the brute path uses, narrowed to the leading/// literal's prefix range first.- KeywordWildcard { field: String, pattern: String },+ KeywordWildcard {+ field: String,+ pattern: String,+ /// #668: fold both sides when true (term-ci / query_string rewrites);+ /// match the raw term case-sensitively when false (standalone wildcard).+ case_insensitive: bool,+ },/// `term` on a numeric/boolean field, or `range` on a numeric field —/// both evaluate as an inclusive/exclusive f64 window over the numeric/// doc-values column (a term is the degenerate `[v, v]` window;@@ -42419,11 +42442,12 @@ fn scored_fast_plan(value,boost,constant_score: true,- case_insensitive: _,+ case_insensitive,} if fs.kw.contains(field) => Some(ScoredPlan::Filtered {filter: vec![ScoredFilterLeaf::KeywordWildcard {field: field.clone(),pattern: value.clone(),+ case_insensitive: *case_insensitive,}],must_not: Vec::new(),score: scoring_boost(boost)?,diff --git a/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs b/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rsindex d1c0d436e..d408c2e6a 100644--- a/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs+++ b/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs@@ -27,7 +27,6 @@ async fn hits(engine: &Engine, q: serde_json::Value) -> u64 {}#[tokio::test]-#[ignore = "#668: keyword wildcard folds case; un-ignored when the case_insensitive flag is threaded"]async fn keyword_wildcard_is_case_sensitive_but_term_ci_still_folds() {let dir = TempDir::new().unwrap();let mut config = Config::default();@@ -43,30 +42,44 @@ async fn keyword_wildcard_is_case_sensitive_but_term_ci_still_folds() {idx.index_document(Some("d0".to_string()), json!({ "name": "Hello" })).await.expect("index");++ let wc = |p: &str| json!({ "wildcard": { "name": p } });+ let term_ci = json!({ "term": { "name": { "value": "hello", "case_insensitive": true } } });++ // Capture the buffered (memtable) answers, then flush and re-query the+ // segment: every pair must agree (flush-invariance — the #667 lesson).+ let pre_lower = hits(&engine, wc("hell*")).await;+ let pre_cased = hits(&engine, wc("Hell*")).await;+ let pre_ci = hits(&engine, term_ci.clone()).await;idx.refresh().await.expect("refresh");+ let post_lower = hits(&engine, wc("hell*")).await;+ let post_cased = hits(&engine, wc("Hell*")).await;+ let post_ci = hits(&engine, term_ci.clone()).await;- // ES-correct: keyword wildcard is case-sensitive, so a lower-case pattern- // does not match a capitalized value. (Currently returns 1 — the bug.)assert_eq!(- hits(&engine, json!({ "wildcard": { "name": "hell*" } })).await,- 0,- "#668: keyword `wildcard` must be case-sensitive"+ pre_lower, post_lower,+ "#668: keyword wildcard must be flush-invariant"+ );+ assert_eq!(+ pre_cased, post_cased,+ "#668: keyword wildcard must be flush-invariant");- // Sanity: a correctly-cased pattern still matches.assert_eq!(- hits(&engine, json!({ "wildcard": { "name": "Hell*" } })).await,- 1,- "case-correct wildcard still matches"+ pre_ci, post_ci,+ "#668: term-ci wildcard must be flush-invariant"+ );++ // ES-correct: keyword wildcard is case-sensitive, so a lower-case pattern+ // does not match a capitalized value; a correctly-cased one does.+ assert_eq!(+ post_lower, 0,+ "#668: keyword `wildcard` must be case-sensitive");+ assert_eq!(post_cased, 1, "case-correct wildcard still matches");// CONSTRAINT: `term{case_insensitive:true}` (rewritten to a Wildcard) must// KEEP folding — the fix must not break this passing path.assert_eq!(- hits(- &engine,- json!({ "term": { "name": { "value": "hello", "case_insensitive": true } } })- )- .await,- 1,- "#668: term with case_insensitive true must still fold after the fix"+ post_ci, 1,+ "#668: term with case_insensitive true must still fold");}