Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(query): honor case_insensitive on prefix and regexp queries (#1812) (#1816)

rusthatesmarty1f5c0e76d791
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rs
index 8ab8bcede..b8125f489 100644
--- a/engine/crates/xerj-engine/src/aggs.rs
+++ b/engine/crates/xerj-engine/src/aggs.rs
@@ -7217,13 +7217,27 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
if let Some(field_map) = query_body.as_object() {
for (field, expected) in field_map {
let actual_values = extract_field_values(doc, field);
- let prefix = match expected {
- Value::Object(o) => {
- o.get("value").map(value_to_string).unwrap_or_default()
- }
- other => value_to_string(other),
+ let (prefix, case_insensitive) = match expected {
+ Value::Object(o) => (
+ o.get("value").map(value_to_string).unwrap_or_default(),
+ o.get("case_insensitive")
+ .and_then(Value::as_bool)
+ .unwrap_or(false),
+ ),
+ other => (value_to_string(other), false),
+ };
+ // #1812: fold both sides when `case_insensitive` is
+ // set, mirroring the `wildcard` arm above and the
+ // keyword branch of `QueryNode::Prefix` (index.rs).
+ let matched = if case_insensitive {
+ let prefix_lc = prefix.to_lowercase();
+ actual_values
+ .iter()
+ .any(|v| v.to_lowercase().starts_with(&prefix_lc))
+ } else {
+ actual_values.iter().any(|v| v.starts_with(&prefix))
};
- if !actual_values.iter().any(|v| v.starts_with(&prefix)) {
+ if !matched {
return false;
}
}
@@ -7273,11 +7287,14 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
if let Some(field_map) = query_body.as_object() {
for (field, expected) in field_map {
let actual_values = extract_field_values(doc, field);
- let pattern = match expected {
- Value::Object(o) => {
- o.get("value").map(value_to_string).unwrap_or_default()
- }
- other => value_to_string(other),
+ let (pattern, case_insensitive) = match expected {
+ Value::Object(o) => (
+ o.get("value").map(value_to_string).unwrap_or_default(),
+ o.get("case_insensitive")
+ .and_then(Value::as_bool)
+ .unwrap_or(false),
+ ),
+ other => (value_to_string(other), false),
};
// #959: `regexp` had no arm at all and fell through
// the catch-all ("unknown filter type — pass
@@ -7289,6 +7306,15 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {
// multi-valued field — mirroring `QueryNode::Regexp`
// (index.rs). An invalid pattern matches nothing,
// same as the doc-scan arm.
+ // #1812: `case_insensitive` prepends the inline
+ // `(?i)` flag inside the same anchored compile —
+ // still a full-string match, mirroring the query
+ // path's keyword branch.
+ let pattern = if case_insensitive {
+ format!("(?i){}", pattern)
+ } else {
+ pattern
+ };
let matched = crate::index::compiled_anchored_regex(&pattern)
.map(|re| actual_values.iter().any(|v| re.is_match(v)))
.unwrap_or(false);
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 0dfd62d41..724a93287 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -23961,7 +23961,12 @@ impl Index {
sort_cand
.as_ref()
.map(|sc| Arc::new(PrefilterSet::from_set(sc.set.clone())))
- } else if let QueryNode::Regexp { field, pattern } = query {
+ } else if let QueryNode::Regexp {
+ field,
+ pattern,
+ case_insensitive,
+ } = query
+ {
// Dictionary-expansion pre-filter (mirrors the
// `try_shortcut_count` Regexp arm). When the
// exact total comes from the shortcut
@@ -23991,15 +23996,37 @@ impl Index {
} else {
None
};
- self.build_regexp_prefilter_cached(
- &segments_dir,
- &seg_id,
- field,
- pattern,
- cap,
- regexp_field_is_keyword,
- )
- .map(|s| Arc::new(PrefilterSet::from_set(s)))
+ // #1812: the dictionary expansion compiles this same
+ // pattern, so it must carry the inline `(?i)` flag too
+ // — a case-sensitive expansion would pre-filter away
+ // exactly the hits the (case-insensitive) doc-scan arm
+ // would have matched. This arm is a CANDIDATE filter:
+ // the scan re-checks every position against
+ // `doc_matches_query`, so a folded (wider) candidate
+ // set is safe for any field type, and required for
+ // keyword (where the doc-scan folds too).
+ if *case_insensitive {
+ let folded = format!("(?i){}", pattern);
+ self.build_regexp_prefilter_cached(
+ &segments_dir,
+ &seg_id,
+ field,
+ &folded,
+ cap,
+ regexp_field_is_keyword,
+ )
+ .map(|s| Arc::new(PrefilterSet::from_set(s)))
+ } else {
+ self.build_regexp_prefilter_cached(
+ &segments_dir,
+ &seg_id,
+ field,
+ pattern,
+ cap,
+ regexp_field_is_keyword,
+ )
+ .map(|s| Arc::new(PrefilterSet::from_set(s)))
+ }
} else {
None
};
@@ -28151,7 +28178,12 @@ impl Index {
// segment has neither source (dotted / nested fields, pre-FTS
// segments) or when deletes are present (both dictionaries tally
// physical docs, not live ones).
- if let QueryNode::Regexp { field, pattern } = query {
+ if let QueryNode::Regexp {
+ field,
+ pattern,
+ case_insensitive,
+ } = query
+ {
// Delete-blind guard — same conservative signal as the F1
// `deletes_present` gate in `search_inner`.
let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();
@@ -28181,6 +28213,20 @@ impl Index {
// per query after that. `None` (no usable dictionary /
// non-keyword field / pathological pattern) abandons to
// the stored-doc scan.
+ //
+ // #1812: this shortcut's segment total is used VERBATIM (no
+ // doc-scan re-check), so a `case_insensitive` pattern must
+ // also fold inside the dictionary expansion — otherwise the
+ // count would silently drop the case-variant matches the
+ // doc-scan arm would return. Gated on `field_is_keyword`,
+ // the same gate the doc-scan arm applies.
+ let folded;
+ let pattern: &str = if *case_insensitive && field_is_keyword {
+ folded = format!("(?i){}", pattern);
+ &folded
+ } else {
+ pattern
+ };
let exp = self.regexp_segment_expansion(
&segments_dir,
&meta.id,
@@ -42813,6 +42859,7 @@ mod lexically_typeless_lowering_tests {
value: "0".into(),
boost: None,
constant_score: true,
+ case_insensitive: false,
},
QueryNode::Wildcard {
field: "emb".into(),
@@ -44158,11 +44205,13 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
value,
boost,
constant_score,
+ case_insensitive,
} => QueryNode::Prefix {
field: strip(field),
value: value.clone(),
boost: *boost,
constant_score: *constant_score,
+ case_insensitive: *case_insensitive,
},
QueryNode::Wildcard {
field,
@@ -44236,9 +44285,14 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {
case_insensitive: *case_insensitive,
prefix_length: *prefix_length,
},
- QueryNode::Regexp { field, pattern } => QueryNode::Regexp {
+ QueryNode::Regexp {
+ field,
+ pattern,
+ case_insensitive,
+ } => QueryNode::Regexp {
field: strip(field),
pattern: pattern.clone(),
+ case_insensitive: *case_insensitive,
},
QueryNode::MultiMatch {
fields,
@@ -46153,11 +46207,16 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sc
_ => get_field_value(source, field).is_some_and(|v| value_present(&v)),
},
- QueryNode::Prefix { field, value, .. } => {
+ QueryNode::Prefix {
+ field,
+ value,
+ case_insensitive,
+ ..
+ } => {
// #396: prefix semantics depend on the field's mapping, and ES never
// analyses the prefix VALUE itself.
- // - keyword: match the RAW stored value, case-SENSITIVE ("Hello"
- // does not start with "hel").
+ // - keyword: match the RAW stored value, case-SENSITIVE by default
+ // ("Hello" does not start with "hel") unless `case_insensitive` true.
// - analysed text: match the analysed (lower-cased) token stream
// against the RAW query ("hel" prefixes "hello"; "Hel" does not).
// - unknown field (schemaless shim caller — no `Schema` in scope):
@@ -46169,7 +46228,11 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sc
let value_lc = value.to_lowercase();
let matches_str = |s: &str| -> bool {
if is_keyword {
- s.starts_with(value.as_str())
+ if *case_insensitive {
+ s.to_lowercase().starts_with(&value_lc)
+ } else {
+ s.starts_with(value.as_str())
+ }
} else if is_text {
s.split(|c: char| !c.is_alphanumeric()).any(|tok| {
!tok.is_empty() && tok.to_lowercase().starts_with(value.as_str())
@@ -47031,13 +47094,28 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sc
.unwrap_or(false)
}
- QueryNode::Regexp { field, pattern } => {
+ QueryNode::Regexp {
+ field,
+ pattern,
+ case_insensitive,
+ } => {
get_field_value(source, field)
.and_then(|v| match v {
Value::String(s) => {
// Compile-once (thread-local cache) — this arm runs
// per candidate doc and used to recompile every time.
- compiled_anchored_regex(pattern).map(|re| re.is_match(&s))
+ // #1812: `case_insensitive` on a KEYWORD field compiles
+ // the same anchored pattern with the inline `(?i)` flag
+ // (still full-string match — the anchored wrapper is
+ // unchanged, so complex patterns keep ES semantics).
+ let ft = declared_field(schema, field).map(|f| &f.field_type);
+ let is_keyword = matches!(ft, Some(FieldType::Keyword));
+ if is_keyword && *case_insensitive {
+ compiled_anchored_regex(&format!("(?i){}", pattern))
+ .map(|re| re.is_match(&s))
+ } else {
+ compiled_anchored_regex(pattern).map(|re| re.is_match(&s))
+ }
}
_ => None,
})
@@ -52290,6 +52368,7 @@ fn query_node_to_fts_projected(
value,
boost,
constant_score,
+ case_insensitive,
} => {
// `prefix` on a KEYWORD field: whole-value, case-sensitive prefix
// match over the keyword FST term dictionary — byte-identical to ES
@@ -52306,7 +52385,17 @@ fn query_node_to_fts_projected(
// exact. TEXT fields keep the doc-scan (None): their prefix runs
// against analyzed tokens, which the FST-whole-value route can't
// reproduce.
+ //
+ // #1812: `case_insensitive: true` makes the KEYWORD FST route
+ // bail to the brute scan — that route expands CASE-SENSITIVELY
+ // and can't fold, while the doc-scan arm folds both sides for
+ // keyword fields. TEXT keeps its route: analyzed terms are
+ // already lowercased and the pattern is matched raw (flag
+ // irrelevant on text, per ES).
if exact_fields.contains(field.as_str()) {
+ if *case_insensitive {
+ return None;
+ }
return Some(FtsQuery::Prefix(xerj_fts::search::PrefixQuery {
field: field.clone(),
prefix: value.clone(),
@@ -53828,14 +53917,25 @@ fn scored_fast_plan(
value,
boost,
constant_score: true,
- } if fs.kw.contains(field) => Some(ScoredPlan::Filtered {
- filter: vec![ScoredFilterLeaf::KeywordPrefix {
- field: field.clone(),
- prefix: value.clone(),
- }],
- must_not: Vec::new(),
- score: scoring_boost(boost)?,
- }),
+ case_insensitive,
+ } if fs.kw.contains(field) => {
+ // #1812: the keyword-ordinal prefix filter is case-SENSITIVE —
+ // a `case_insensitive` prefix must fall through to the brute
+ // scan (which folds both sides), like the `Wildcard` arm below
+ // threads the flag into its filter instead.
+ if *case_insensitive {
+ None
+ } else {
+ Some(ScoredPlan::Filtered {
+ filter: vec![ScoredFilterLeaf::KeywordPrefix {
+ field: field.clone(),
+ prefix: value.clone(),
+ }],
+ must_not: Vec::new(),
+ score: scoring_boost(boost)?,
+ })
+ }
+ }
QueryNode::Wildcard {
field,
value,
@@ -56101,6 +56201,7 @@ mod fts_projection_tests {
value: "run".into(),
boost: None,
constant_score: true,
+ case_insensitive: false,
};
match query_node_to_fts(&q, &tf, &kw(&[])).expect("text prefix projects") {
FtsQuery::Prefix(p) => {
@@ -60725,6 +60826,7 @@ mod exact_scan_hydration_tests {
value: "t".into(),
boost: None,
constant_score: false,
+ case_insensitive: false,
},
QueryNode::Bool {
must: vec![exists("tag")],
diff --git a/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rs b/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rs
new file mode 100644
index 000000000..566aca15e
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rs
@@ -0,0 +1,181 @@
+//! #1812: `prefix` on a `keyword` field must honor `case_insensitive`.
+//!
+//! ES matches `prefix` against the RAW keyword value (case-SENSITIVE by
+//! default); `case_insensitive: true` folds both sides instead. XERJ's
+//! `Prefix` AST node dropped the flag entirely, so
+//! `prefix:{k:{value:"HELLO",case_insensitive:true}}` on `"Hello World"`
+//! returned 0 (wrong — ES returns the doc).
+//!
+//! Both the buffered (memtable) and flushed (segment) paths are asserted
+//! pre- and post-`refresh`, mirroring `prefix_keyword_case_flush_parity.rs`:
+//! a fix applied to only one path split-brains across a background flush,
+//! which this repo treats as a regression class of its own.
+
+use std::sync::Arc;
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+/// Index two keyword docs (capitalized + lowercase) and return the index.
+async fn setup_docs(engine: &Engine) -> Arc<Index> {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("k", FieldType::Keyword));
+ engine.create_index("docs", schema).expect("create");
+
+ let idx = engine.get_index("docs").expect("get");
+ idx.index_document(Some("d1".to_string()), json!({ "k": "Hello World" }))
+ .await
+ .expect("index");
+ idx.index_document(Some("d2".to_string()), json!({ "k": "hello there" }))
+ .await
+ .expect("index");
+ idx
+}
+
+async fn hits(idx: &Index, query: serde_json::Value) -> u64 {
+ let req = parse_request(&json!({ "query": query, "size": 0 })).expect("parse_request");
+ idx.search(&req).await.expect("search").total.value
+}
+
+/// FAIL-BEFORE (#1812): the flag was dropped at parse time, so this matched
+/// 0 docs. ES returns 2 (both keyword values start with "hello", folded).
+#[tokio::test]
+async fn prefix_case_insensitive_true_folds_both_paths() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ let q = json!({ "prefix": { "k": { "value": "HELLO", "case_insensitive": true } } });
+
+ // Buffered (memtable) path.
+ let pre = hits(&idx, q.clone()).await;
+ // Flushed (segment) path.
+ idx.refresh().await.expect("refresh");
+ let post = hits(&idx, q).await;
+
+ assert_eq!(
+ pre, 2,
+ "memtable: case_insensitive prefix must match both docs"
+ );
+ assert_eq!(
+ pre, post,
+ "flush-invariant: memtable={pre} vs segment={post} disagree"
+ );
+ assert_eq!(
+ post, 2,
+ "segment: case_insensitive prefix must match both docs"
+ );
+}
+
+/// Explicit `case_insensitive: false` stays ES-correct case-SENSITIVE.
+#[tokio::test]
+async fn prefix_case_insensitive_false_is_case_sensitive() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ // "HELLO" prefixes neither "Hello World" nor "hello there" byte-wise.
+ let q_upper = json!({ "prefix": { "k": { "value": "HELLO", "case_insensitive": false } } });
+ // "hello" prefixes only the lowercase doc byte-wise.
+ let q_lower = json!({ "prefix": { "k": { "value": "hello", "case_insensitive": false } } });
+
+ let pre_upper = hits(&idx, q_upper.clone()).await;
+ let pre_lower = hits(&idx, q_lower.clone()).await;
+ idx.refresh().await.expect("refresh");
+ let post_upper = hits(&idx, q_upper).await;
+ let post_lower = hits(&idx, q_lower).await;
+
+ assert_eq!(
+ pre_upper, 0,
+ "memtable: HELLO must not match capitalized values"
+ );
+ assert_eq!(
+ pre_lower, 1,
+ "memtable: hello must match only the lowercase doc"
+ );
+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");
+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");
+ assert_eq!(
+ post_upper, 0,
+ "segment: HELLO must not match capitalized values"
+ );
+ assert_eq!(
+ post_lower, 1,
+ "segment: hello must match only the lowercase doc"
+ );
+}
+
+/// Omitted parameter defaults to `false` (shorthand and expanded forms).
+#[tokio::test]
+async fn prefix_omitted_case_insensitive_defaults_false() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ let q_upper = json!({ "prefix": { "k": "HELLO" } });
+ let q_lower = json!({ "prefix": { "k": "hello" } });
+
+ let pre_upper = hits(&idx, q_upper.clone()).await;
+ let pre_lower = hits(&idx, q_lower.clone()).await;
+ idx.refresh().await.expect("refresh");
+ let post_upper = hits(&idx, q_upper).await;
+ let post_lower = hits(&idx, q_lower).await;
+
+ assert_eq!(pre_upper, 0, "shorthand HELLO defaults to case-sensitive");
+ assert_eq!(
+ pre_lower, 1,
+ "shorthand hello matches only the lowercase doc"
+ );
+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");
+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");
+}
+
+/// TEXT-field semantics are UNCHANGED by the flag: prefix runs against the
+/// analyzed (lowercased) token stream, so a lowercase pattern matches and an
+/// uppercase one does not — with or without `case_insensitive: true`.
+#[tokio::test]
+async fn prefix_case_insensitive_does_not_change_text_semantics() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+
+ let mut schema = Schema::empty();
+ schema.fields.push(FieldConfig::new("t", FieldType::Text));
+ engine.create_index("docs", schema).expect("create");
+ let idx = engine.get_index("docs").expect("get");
+ idx.index_document(Some("d1".to_string()), json!({ "t": "Hello World" }))
+ .await
+ .expect("index");
+ idx.refresh().await.expect("refresh");
+
+ let q_lower = json!({ "prefix": { "t": { "value": "hel", "case_insensitive": false } } });
+ let q_lower_ci = json!({ "prefix": { "t": { "value": "hel", "case_insensitive": true } } });
+ let q_upper = json!({ "prefix": { "t": { "value": "HEL", "case_insensitive": true } } });
+
+ assert_eq!(
+ hits(&idx, q_lower).await,
+ 1,
+ "text: lowercase prefix matches the analyzed token"
+ );
+ assert_eq!(
+ hits(&idx, q_lower_ci).await,
+ 1,
+ "text: case_insensitive must not change analyzed-token matching"
+ );
+ assert_eq!(
+ hits(&idx, q_upper).await,
+ 0,
+ "text: uppercase prefix does not match the lowercased token (ES analyzes the doc, not the pattern)"
+ );
+}
diff --git a/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rs b/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rs
new file mode 100644
index 000000000..e16c8ca2e
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rs
@@ -0,0 +1,167 @@
+//! #1812: `regexp` on a `keyword` field must honor `case_insensitive`.
+//!
+//! ES matches `regexp` against the RAW keyword value (case-SENSITIVE by
+//! default); `case_insensitive: true` folds both sides instead. XERJ's
+//! `Regexp` AST node dropped the flag entirely, so
+//! `regexp:{k:{value:"HEL+O.*",case_insensitive:true}}` on `"Hello World"`
+//! returned 0 (wrong — ES returns the doc).
+//!
+//! Both the buffered (memtable) and flushed (segment) paths are asserted
+//! pre- and post-`refresh`: the segment path answers via a dictionary
+//! expansion pre-filter (`regexp_segment_expansion` /
+//! `build_regexp_prefilter_cached`) that compiles the pattern itself, so a
+//! fix only in the doc-scan arm would split-brain across a flush.
+
+use std::sync::Arc;
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::{FieldConfig, FieldType, Schema};
+use xerj_engine::{Engine, Index};
+use xerj_query::parse_request;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+/// Index two keyword docs (capitalized + lowercase) and return the index.
+async fn setup_docs(engine: &Engine) -> Arc<Index> {
+ let mut schema = Schema::empty();
+ schema
+ .fields
+ .push(FieldConfig::new("k", FieldType::Keyword));
+ engine.create_index("docs", schema).expect("create");
+
+ let idx = engine.get_index("docs").expect("get");
+ idx.index_document(Some("d1".to_string()), json!({ "k": "Hello World" }))
+ .await
+ .expect("index");
+ idx.index_document(Some("d2".to_string()), json!({ "k": "hello there" }))
+ .await
+ .expect("index");
+ idx
+}
+
+async fn hits(idx: &Index, query: serde_json::Value) -> u64 {
+ let req = parse_request(&json!({ "query": query, "size": 0 })).expect("parse_request");
+ idx.search(&req).await.expect("search").total.value
+}
+
+/// FAIL-BEFORE (#1812): the flag was dropped at parse time, so this matched
+/// 0 docs. ES returns 2 (both keyword values full-match "hel+o.*", folded).
+#[tokio::test]
+async fn regexp_case_insensitive_true_folds_both_paths() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ let q = json!({ "regexp": { "k": { "value": "HEL+O.*", "case_insensitive": true } } });
+
+ // Buffered (memtable) path.
+ let pre = hits(&idx, q.clone()).await;
+ // Flushed (segment) path — exercised through the dictionary expansion.
+ idx.refresh().await.expect("refresh");
+ let post = hits(&idx, q).await;
+
+ assert_eq!(
+ pre, 2,
+ "memtable: case_insensitive regexp must match both docs"
+ );
+ assert_eq!(
+ pre, post,
+ "flush-invariant: memtable={pre} vs segment={post} disagree"
+ );
+ assert_eq!(
+ post, 2,
+ "segment: case_insensitive regexp must match both docs"
+ );
+}
+
+/// Explicit `case_insensitive: false` stays ES-correct case-SENSITIVE.
+#[tokio::test]
+async fn regexp_case_insensitive_false_is_case_sensitive() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ // "HEL+O.*" full-matches neither value byte-wise.
+ let q_upper = json!({ "regexp": { "k": { "value": "HEL+O.*", "case_insensitive": false } } });
+ // "hel+o.*" full-matches only the lowercase doc byte-wise.
+ let q_lower = json!({ "regexp": { "k": { "value": "hel+o.*", "case_insensitive": false } } });
+
+ let pre_upper = hits(&idx, q_upper.clone()).await;
+ let pre_lower = hits(&idx, q_lower.clone()).await;
+ idx.refresh().await.expect("refresh");
+ let post_upper = hits(&idx, q_upper).await;
+ let post_lower = hits(&idx, q_lower).await;
+
+ assert_eq!(
+ pre_upper, 0,
+ "memtable: HEL+O.* must not match capitalized values"
+ );
+ assert_eq!(
+ pre_lower, 1,
+ "memtable: hel+o.* must match only the lowercase doc"
+ );
+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");
+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");
+ assert_eq!(
+ post_upper, 0,
+ "segment: HEL+O.* must not match capitalized values"
+ );
+ assert_eq!(
+ post_lower, 1,
+ "segment: hel+o.* must match only the lowercase doc"
+ );
+}
+
+/// Omitted parameter defaults to `false` (shorthand and expanded forms).
+#[tokio::test]
+async fn regexp_omitted_case_insensitive_defaults_false() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ let q_upper = json!({ "regexp": { "k": "HEL+O.*" } });
+ let q_lower = json!({ "regexp": { "k": "hel+o.*" } });
+
+ let pre_upper = hits(&idx, q_upper.clone()).await;
+ let pre_lower = hits(&idx, q_lower.clone()).await;
+ idx.refresh().await.expect("refresh");
+ let post_upper = hits(&idx, q_upper).await;
+ let post_lower = hits(&idx, q_lower).await;
+
+ assert_eq!(pre_upper, 0, "shorthand HEL+O.* defaults to case-sensitive");
+ assert_eq!(
+ pre_lower, 1,
+ "shorthand hel+o.* matches only the lowercase doc"
+ );
+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");
+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");
+}
+
+/// The flag must not change matching for values it shouldn't reach: a
+/// case-insensitive pattern that only matches via folding must NOT match a
+/// non-string value path, and an invalid pattern still matches nothing.
+#[tokio::test]
+async fn regexp_case_insensitive_invalid_pattern_matches_nothing() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = setup_docs(&engine).await;
+
+ let q = json!({ "regexp": { "k": { "value": "([", "case_insensitive": true } } });
+ assert_eq!(
+ hits(&idx, q.clone()).await,
+ 0,
+ "invalid pattern matches nothing (memtable)"
+ );
+ idx.refresh().await.expect("refresh");
+ assert_eq!(
+ hits(&idx, q).await,
+ 0,
+ "invalid pattern matches nothing (segment)"
+ );
+}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index c38f1b152..ab0d50a6e 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -490,6 +490,12 @@ pub enum QueryNode {
/// match_bool_prefix) that keep BM25 scoring.
#[serde(default, skip_serializing_if = "is_false")]
constant_score: bool,
+ /// #1812: fold case on both sides when matching a KEYWORD field.
+ /// `false` (default) = ES-correct case-sensitive keyword prefix.
+ /// `true` is set for `prefix{case_insensitive:true}`.
+ /// Ignored for `text` fields.
+ #[serde(default, skip_serializing_if = "is_false")]
+ case_insensitive: bool,
},
/// Wildcard pattern match (`?` = any char, `*` = zero-or-more chars).
@@ -714,7 +720,15 @@ pub enum QueryNode {
},
/// Regular expression match against a field value.
- Regexp { field: String, pattern: String },
+ Regexp {
+ field: String,
+ pattern: String,
+ /// #1812: fold case on both sides when matching a KEYWORD field.
+ /// `false` (default) = ES-correct case-sensitive keyword regexp.
+ /// Ignored for `text` fields.
+ #[serde(default, skip_serializing_if = "is_false")]
+ case_insensitive: bool,
+ },
/// Position-aware text match (ES `intervals` query). The `rule` is the
/// raw rule JSON (one of match/all_of/any_of/prefix/wildcard/fuzzy,
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index 870cdad65..4b29c39a6 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -924,6 +924,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
value: tokens[0].clone(),
boost: fb,
constant_score: false,
+ case_insensitive: false,
};
should.push(prefix);
continue;
@@ -957,6 +958,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {
value: tokens[last].clone(),
boost: None,
constant_score: false,
+ case_insensitive: false,
});
let (im, is, imm) = if operator_and {
(inner_clauses, vec![], None)
@@ -1301,6 +1303,7 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {
value: value.to_string(),
boost: None,
constant_score: true,
+ case_insensitive: false,
});
}
@@ -1326,6 +1329,10 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {
value,
boost,
constant_score: true,
+ case_insensitive: inner
+ .get("case_insensitive")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false),
},
name,
))
@@ -2413,6 +2420,7 @@ fn parse_regexp(params: &Value) -> Result<QueryNode> {
return Ok(QueryNode::Regexp {
field,
pattern: pattern.to_string(),
+ case_insensitive: false,
});
}
@@ -2421,7 +2429,15 @@ fn parse_regexp(params: &Value) -> Result<QueryNode> {
.ok_or_else(|| qerr("`regexp` field value must be a string or object"))?;
let pattern = string_field(inner, "value")?;
- Ok(QueryNode::Regexp { field, pattern })
+ let case_insensitive = inner
+ .get("case_insensitive")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
+ Ok(QueryNode::Regexp {
+ field,
+ pattern,
+ case_insensitive,
+ })
}
fn parse_match_phrase_prefix(params: &Value) -> Result<QueryNode> {
@@ -4728,6 +4744,7 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
value: fold(&raw_tokens[0]),
boost: None,
constant_score: true,
+ case_insensitive: false,
});
}
@@ -4764,6 +4781,9 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {
value: fold(&raw_tokens[last_idx]),
boost: None,
constant_score: false,
+ // `match_bool_prefix` lowers to an internal prefix over ANALYZED
+ // tokens — no user-facing `case_insensitive` option to thread.
+ case_insensitive: false,
});
// operator:and → every clause must match; otherwise should + mm.
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index cd366523e..2546f591c 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -496,7 +496,7 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
},
// Regexp: full scan cost since we must evaluate each doc.
- QueryNode::Regexp { field, pattern } => ExecutionPlan::FtsScan {
+ QueryNode::Regexp { field, pattern, .. } => ExecutionPlan::FtsScan {
field,
terms: vec![serde_json::Value::String(pattern)],
cost: 20.0,
diff --git a/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.yml b/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.yml
new file mode 100644
index 000000000..f571dbc21
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.yml
@@ -0,0 +1,122 @@
+setup:
+
+ - do:
+ indices.create:
+ index: ci
+ body:
+ mappings:
+ properties:
+ k:
+ type: keyword
+
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{ "index": { "_index": "ci", "_id": "1" } }'
+ - '{ "k": "Hello World" }'
+ - '{ "index": { "_index": "ci", "_id": "2" } }'
+ - '{ "k": "hello there" }'
+
+---
+"prefix case_insensitive true finds both":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ prefix:
+ k:
+ value: "HELLO"
+ case_insensitive: true
+ - match: {count: 2}
+
+---
+"regexp case_insensitive true finds both":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ regexp:
+ k:
+ value: "HEL+O.*"
+ case_insensitive: true
+ - match: {count: 2}
+
+---
+"prefix case_insensitive false finds none":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ prefix:
+ k:
+ value: "HELLO"
+ case_insensitive: false
+ - match: {count: 0}
+
+---
+"regexp case_insensitive false finds none":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ regexp:
+ k:
+ value: "HEL+O.*"
+ case_insensitive: false
+ - match: {count: 0}
+
+---
+"prefix default false finds none for HELLO, one for hello":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ prefix:
+ k: "HELLO"
+ - match: {count: 0}
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ prefix:
+ k: "hello"
+ - match: {count: 1}
+
+---
+"regexp default false finds none for HELLO pattern, one for lowercase":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ regexp:
+ k: "HEL+O.*"
+ - match: {count: 0}
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ regexp:
+ k: "hel+o.*"
+ - match: {count: 1}
+
+---
+"wildcard control with case_insensitive true finds both":
+ - do:
+ count:
+ index: ci
+ body:
+ query:
+ wildcard:
+ k:
+ value: "HELLO*"
+ case_insensitive: true
+ - match: {count: 2}