Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(engine): #668 keyword wildcard honors case_insensitive on all matcher paths

xerj-orgc9eee4a6be20
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 1e5265ac3..caf2757c3 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -22547,28 +22547,34 @@ impl Index {
ords: (lo as u32..hi as u32).collect(),
});
}
- ScoredFilterLeaf::KeywordWildcard { field, pattern } => {
+ ScoredFilterLeaf::KeywordWildcard {
+ field,
+ pattern,
+ case_insensitive,
+ } => {
let Some(Column::Keyword(k)) = cols.get(field.as_str()) else {
return None;
};
- // SAME predicate as the brute path (`doc_matches_query`)
- // and the FST route (`term_matches_wildcard`): fold BOTH
- // sides, whole-value OR sub-token. The old range+match
- // here was case-SENSITIVE, so a cased pattern (`K*1`,
- // case_insensitive) found nothing in a lowercase
- // dictionary — DV-only keyword wildcards returned 0 hits
- // from segments while the memtable matched. A folded
- // pattern cannot bound a byte-ordered prefix range, so
- // walk the whole (distinct-term) dictionary — the same
- // cost class as the FST expansion.
+ // #668: honor the node's case flag — the SAME predicate
+ // the brute path (`doc_matches_query`) and the FST route
+ // now apply. `term{case_insensitive:true}` / query_string
+ // rewrites fold BOTH sides (whole-value OR sub-token); a
+ // standalone `{wildcard}` matches the RAW term
+ // case-sensitively (ES default). A folded pattern cannot
+ // bound a byte-ordered prefix range, so walk the whole
+ // (distinct-term) dictionary — same cost class as the FST.
let pat_lc = pattern.to_lowercase();
let ords: Vec<u32> = (0..k.terms.len())
.filter(|&o| {
- let lc = k.terms[o].to_lowercase();
- wildcard_match(&lc, &pat_lc)
- || lc
- .split(|c: char| !c.is_alphanumeric())
- .any(|tok| !tok.is_empty() && wildcard_match(tok, &pat_lc))
+ if *case_insensitive {
+ let lc = k.terms[o].to_lowercase();
+ wildcard_match(&lc, &pat_lc)
+ || lc.split(|c: char| !c.is_alphanumeric()).any(|tok| {
+ !tok.is_empty() && wildcard_match(tok, &pat_lc)
+ })
+ } else {
+ wildcard_match(&k.terms[o], pattern.as_str())
+ }
})
.map(|o| o as u32)
.collect();
@@ -35737,30 +35743,40 @@ fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Schema) -> bo
QueryNode::Wildcard {
field,
value: pattern,
+ case_insensitive,
..
} => {
- // #396: `wildcard` must mirror its segment path per field type
+ // #396/#668: `wildcard` must mirror its segment path per field type
// (ES never analyses the pattern):
// - analysed TEXT: the segment matches the RAW pattern against the
// lower-cased token stream (case_insensitive:false), so an
- // uppercase pattern matches nothing — do the same here (this is
- // the flush-divergence the KEYWORD-only earlier attempt missed).
- // - KEYWORD: the segment doc-values keyword-wildcard path folds
- // both sides (case-insensitive), so fold here to stay in
- // agreement. ES-correct case-sensitive keyword wildcard needs the
- // segment path changed too — tracked in #668.
- // - unknown field (schemaless shim caller): the pre-#423 fold-both
- // heuristic, byte-for-byte unchanged.
- let is_text = matches!(
- declared_field(schema, field).map(|f| &f.field_type),
- Some(FieldType::Text)
- );
+ // uppercase pattern matches nothing — do the same here.
+ // - KEYWORD, case-sensitive (standalone `{wildcard}`, the ES
+ // default): match the RAW value with `wildcard_match` (#668).
+ // - KEYWORD, case-insensitive (`term{case_insensitive:true}` /
+ // query_string lowerings set the flag): fold both sides.
+ // - field absent from the schema (the `EMPTY_SCHEMA` shim callers,
+ // or a genuinely-undeclared field): the pre-#423 fold-both
+ // heuristic. NB memtable search passes the real, dynamically-grown
+ // schema, where an undeclared string is already mapped to `text`
+ // and takes the branch above — so this arm is effectively the
+ // schemaless-shim path, byte-for-byte unchanged.
+ let ft = declared_field(schema, field).map(|f| &f.field_type);
+ let is_text = matches!(ft, Some(FieldType::Text));
+ // Case-sensitive only for a declared keyword whose node did NOT ask
+ // to fold. Unknown fields (ft == None) keep folding (shim parity).
+ let keyword_case_sensitive =
+ matches!(ft, Some(FieldType::Keyword)) && !*case_insensitive;
let pat_lc = pattern.to_lowercase();
let matches_str = |s: &str| -> bool {
if is_text {
s.split(|c: char| !c.is_alphanumeric()).any(|tok| {
!tok.is_empty() && wildcard_match(&tok.to_lowercase(), pattern.as_str())
})
+ } else if keyword_case_sensitive {
+ // A keyword is a single un-analysed term: match the whole
+ // raw value, case-sensitively (mirrors the segment FST).
+ wildcard_match(s, pattern.as_str())
} else {
let lc = s.to_lowercase();
wildcard_match(&lc, &pat_lc)
@@ -41052,7 +41068,7 @@ fn query_node_to_fts_with_keyword_fields(
value,
boost,
constant_score,
- case_insensitive: _,
+ case_insensitive,
} => {
// `wildcard` on a KEYWORD field: expand the keyword FST term
// dictionary to every term matching the pattern (`*`=0+ chars,
@@ -41063,16 +41079,17 @@ fn query_node_to_fts_with_keyword_fields(
// (≪ docs) instead of a per-doc O(N) scan (~150 ms/100k → ~1 ms).
//
// Case-insensitivity is REQUIRED, not a shortcut: XERJ's parser
- // drops `case_insensitive` and rewrites `term{case_insensitive:true}`
- // to a Wildcard, both relying on the matcher folding case — so a
- // case-sensitive FST route would break those (passing) paths.
+ // rewrites `term{case_insensitive:true}` to a Wildcard, which sets
+ // the node's `case_insensitive` flag so this route keeps folding for
+ // that (passing) path. A standalone `{wildcard}` sets the flag
+ // `false` → the FST route is case-SENSITIVE (ES-correct, #668).
// TEXT fields keep the doc-scan (None): analyzed-token semantics.
if exact_fields.contains(field.as_str()) {
return Some(FtsQuery::Wildcard(xerj_fts::search::WildcardQuery {
field: field.clone(),
pattern: value.clone(),
boost: boost.unwrap_or(1.0),
- case_insensitive: true,
+ case_insensitive: *case_insensitive,
// ES rewrites a standalone `wildcard` query to a
// constant_score. The `term{case_insensitive}` /
// query_string lowerings that SHARE this node pass `false`
@@ -41765,7 +41782,13 @@ enum ScoredFilterLeaf {
/// per segment by matching the (distinct) dictionary terms with the
/// SAME `wildcard_match` the brute path uses, narrowed to the leading
/// literal's prefix range first.
- KeywordWildcard { field: String, pattern: String },
+ KeywordWildcard {
+ field: String,
+ pattern: String,
+ /// #668: fold both sides when true (term-ci / query_string rewrites);
+ /// match the raw term case-sensitively when false (standalone wildcard).
+ case_insensitive: bool,
+ },
/// `term` on a numeric/boolean field, or `range` on a numeric field —
/// both evaluate as an inclusive/exclusive f64 window over the numeric
/// doc-values column (a term is the degenerate `[v, v]` window;
@@ -42419,11 +42442,12 @@ fn scored_fast_plan(
value,
boost,
constant_score: true,
- case_insensitive: _,
+ case_insensitive,
} if fs.kw.contains(field) => Some(ScoredPlan::Filtered {
filter: vec![ScoredFilterLeaf::KeywordWildcard {
field: field.clone(),
pattern: value.clone(),
+ case_insensitive: *case_insensitive,
}],
must_not: Vec::new(),
score: scoring_boost(boost)?,
diff --git a/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs b/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs
index d1c0d436e..d408c2e6a 100644
--- a/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs
+++ b/engine/crates/xerj-engine/tests/wildcard_keyword_case_sensitive.rs
@@ -27,7 +27,6 @@ async fn hits(engine: &Engine, q: serde_json::Value) -> u64 {
}
#[tokio::test]
-#[ignore = "#668: keyword wildcard folds case; un-ignored when the case_insensitive flag is threaded"]
async fn keyword_wildcard_is_case_sensitive_but_term_ci_still_folds() {
let dir = TempDir::new().unwrap();
let mut config = Config::default();
@@ -43,30 +42,44 @@ async fn keyword_wildcard_is_case_sensitive_but_term_ci_still_folds() {
idx.index_document(Some("d0".to_string()), json!({ "name": "Hello" }))
.await
.expect("index");
+
+ let wc = |p: &str| json!({ "wildcard": { "name": p } });
+ let term_ci = json!({ "term": { "name": { "value": "hello", "case_insensitive": true } } });
+
+ // Capture the buffered (memtable) answers, then flush and re-query the
+ // segment: every pair must agree (flush-invariance — the #667 lesson).
+ let pre_lower = hits(&engine, wc("hell*")).await;
+ let pre_cased = hits(&engine, wc("Hell*")).await;
+ let pre_ci = hits(&engine, term_ci.clone()).await;
idx.refresh().await.expect("refresh");
+ let post_lower = hits(&engine, wc("hell*")).await;
+ let post_cased = hits(&engine, wc("Hell*")).await;
+ let post_ci = hits(&engine, term_ci.clone()).await;
- // ES-correct: keyword wildcard is case-sensitive, so a lower-case pattern
- // does not match a capitalized value. (Currently returns 1 — the bug.)
assert_eq!(
- hits(&engine, json!({ "wildcard": { "name": "hell*" } })).await,
- 0,
- "#668: keyword `wildcard` must be case-sensitive"
+ pre_lower, post_lower,
+ "#668: keyword wildcard must be flush-invariant"
+ );
+ assert_eq!(
+ pre_cased, post_cased,
+ "#668: keyword wildcard must be flush-invariant"
);
- // Sanity: a correctly-cased pattern still matches.
assert_eq!(
- hits(&engine, json!({ "wildcard": { "name": "Hell*" } })).await,
- 1,
- "case-correct wildcard still matches"
+ pre_ci, post_ci,
+ "#668: term-ci wildcard must be flush-invariant"
+ );
+
+ // ES-correct: keyword wildcard is case-sensitive, so a lower-case pattern
+ // does not match a capitalized value; a correctly-cased one does.
+ assert_eq!(
+ post_lower, 0,
+ "#668: keyword `wildcard` must be case-sensitive"
);
+ assert_eq!(post_cased, 1, "case-correct wildcard still matches");
// CONSTRAINT: `term{case_insensitive:true}` (rewritten to a Wildcard) must
// KEEP folding — the fix must not break this passing path.
assert_eq!(
- hits(
- &engine,
- json!({ "term": { "name": { "value": "hello", "case_insensitive": true } } })
- )
- .await,
- 1,
- "#668: term with case_insensitive true must still fold after the fix"
+ post_ci, 1,
+ "#668: term with case_insensitive true must still fold"
);
}