AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(query): honor case_insensitive on prefix and regexp queries (#1812) (#1816)
rusthatesmarty1f5c0e76d791
diff --git a/engine/crates/xerj-engine/src/aggs.rs b/engine/crates/xerj-engine/src/aggs.rsindex 8ab8bcede..b8125f489 100644--- a/engine/crates/xerj-engine/src/aggs.rs+++ b/engine/crates/xerj-engine/src/aggs.rs@@ -7217,13 +7217,27 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {if let Some(field_map) = query_body.as_object() {for (field, expected) in field_map {let actual_values = extract_field_values(doc, field);- let prefix = match expected {- Value::Object(o) => {- o.get("value").map(value_to_string).unwrap_or_default()- }- other => value_to_string(other),+ let (prefix, case_insensitive) = match expected {+ Value::Object(o) => (+ o.get("value").map(value_to_string).unwrap_or_default(),+ o.get("case_insensitive")+ .and_then(Value::as_bool)+ .unwrap_or(false),+ ),+ other => (value_to_string(other), false),+ };+ // #1812: fold both sides when `case_insensitive` is+ // set, mirroring the `wildcard` arm above and the+ // keyword branch of `QueryNode::Prefix` (index.rs).+ let matched = if case_insensitive {+ let prefix_lc = prefix.to_lowercase();+ actual_values+ .iter()+ .any(|v| v.to_lowercase().starts_with(&prefix_lc))+ } else {+ actual_values.iter().any(|v| v.starts_with(&prefix))};- if !actual_values.iter().any(|v| v.starts_with(&prefix)) {+ if !matched {return false;}}@@ -7273,11 +7287,14 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {if let Some(field_map) = query_body.as_object() {for (field, expected) in field_map {let actual_values = extract_field_values(doc, field);- let pattern = match expected {- Value::Object(o) => {- o.get("value").map(value_to_string).unwrap_or_default()- }- other => value_to_string(other),+ let (pattern, case_insensitive) = match expected {+ Value::Object(o) => (+ o.get("value").map(value_to_string).unwrap_or_default(),+ o.get("case_insensitive")+ .and_then(Value::as_bool)+ .unwrap_or(false),+ ),+ other => (value_to_string(other), false),};// #959: `regexp` had no arm at all and fell through// the catch-all ("unknown filter type — pass@@ -7289,6 +7306,15 @@ pub(crate) fn doc_matches_filter(doc: &Value, filter: &Value) -> bool {// multi-valued field — mirroring `QueryNode::Regexp`// (index.rs). An invalid pattern matches nothing,// same as the doc-scan arm.+ // #1812: `case_insensitive` prepends the inline+ // `(?i)` flag inside the same anchored compile —+ // still a full-string match, mirroring the query+ // path's keyword branch.+ let pattern = if case_insensitive {+ format!("(?i){}", pattern)+ } else {+ pattern+ };let matched = crate::index::compiled_anchored_regex(&pattern).map(|re| actual_values.iter().any(|v| re.is_match(v))).unwrap_or(false);diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 0dfd62d41..724a93287 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -23961,7 +23961,12 @@ impl Index {sort_cand.as_ref().map(|sc| Arc::new(PrefilterSet::from_set(sc.set.clone())))- } else if let QueryNode::Regexp { field, pattern } = query {+ } else if let QueryNode::Regexp {+ field,+ pattern,+ case_insensitive,+ } = query+ {// Dictionary-expansion pre-filter (mirrors the// `try_shortcut_count` Regexp arm). When the// exact total comes from the shortcut@@ -23991,15 +23996,37 @@ impl Index {} else {None};- self.build_regexp_prefilter_cached(- &segments_dir,- &seg_id,- field,- pattern,- cap,- regexp_field_is_keyword,- )- .map(|s| Arc::new(PrefilterSet::from_set(s)))+ // #1812: the dictionary expansion compiles this same+ // pattern, so it must carry the inline `(?i)` flag too+ // — a case-sensitive expansion would pre-filter away+ // exactly the hits the (case-insensitive) doc-scan arm+ // would have matched. This arm is a CANDIDATE filter:+ // the scan re-checks every position against+ // `doc_matches_query`, so a folded (wider) candidate+ // set is safe for any field type, and required for+ // keyword (where the doc-scan folds too).+ if *case_insensitive {+ let folded = format!("(?i){}", pattern);+ self.build_regexp_prefilter_cached(+ &segments_dir,+ &seg_id,+ field,+ &folded,+ cap,+ regexp_field_is_keyword,+ )+ .map(|s| Arc::new(PrefilterSet::from_set(s)))+ } else {+ self.build_regexp_prefilter_cached(+ &segments_dir,+ &seg_id,+ field,+ pattern,+ cap,+ regexp_field_is_keyword,+ )+ .map(|s| Arc::new(PrefilterSet::from_set(s)))+ }} else {None};@@ -28151,7 +28178,12 @@ impl Index {// segment has neither source (dotted / nested fields, pre-FTS// segments) or when deletes are present (both dictionaries tally// physical docs, not live ones).- if let QueryNode::Regexp { field, pattern } = query {+ if let QueryNode::Regexp {+ field,+ pattern,+ case_insensitive,+ } = query+ {// Delete-blind guard — same conservative signal as the F1// `deletes_present` gate in `search_inner`.let seg_physical: u64 = snap.segments.iter().map(|m| m.doc_count).sum();@@ -28181,6 +28213,20 @@ impl Index {// per query after that. `None` (no usable dictionary /// non-keyword field / pathological pattern) abandons to// the stored-doc scan.+ //+ // #1812: this shortcut's segment total is used VERBATIM (no+ // doc-scan re-check), so a `case_insensitive` pattern must+ // also fold inside the dictionary expansion — otherwise the+ // count would silently drop the case-variant matches the+ // doc-scan arm would return. Gated on `field_is_keyword`,+ // the same gate the doc-scan arm applies.+ let folded;+ let pattern: &str = if *case_insensitive && field_is_keyword {+ folded = format!("(?i){}", pattern);+ &folded+ } else {+ pattern+ };let exp = self.regexp_segment_expansion(&segments_dir,&meta.id,@@ -42813,6 +42859,7 @@ mod lexically_typeless_lowering_tests {value: "0".into(),boost: None,constant_score: true,+ case_insensitive: false,},QueryNode::Wildcard {field: "emb".into(),@@ -44158,11 +44205,13 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {value,boost,constant_score,+ case_insensitive,} => QueryNode::Prefix {field: strip(field),value: value.clone(),boost: *boost,constant_score: *constant_score,+ case_insensitive: *case_insensitive,},QueryNode::Wildcard {field,@@ -44236,9 +44285,14 @@ fn strip_nested_path_in_query(q: &QueryNode, path: &str) -> QueryNode {case_insensitive: *case_insensitive,prefix_length: *prefix_length,},- QueryNode::Regexp { field, pattern } => QueryNode::Regexp {+ QueryNode::Regexp {+ field,+ pattern,+ case_insensitive,+ } => QueryNode::Regexp {field: strip(field),pattern: pattern.clone(),+ case_insensitive: *case_insensitive,},QueryNode::MultiMatch {fields,@@ -46153,11 +46207,16 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sc_ => get_field_value(source, field).is_some_and(|v| value_present(&v)),},- QueryNode::Prefix { field, value, .. } => {+ QueryNode::Prefix {+ field,+ value,+ case_insensitive,+ ..+ } => {// #396: prefix semantics depend on the field's mapping, and ES never// analyses the prefix VALUE itself.- // - keyword: match the RAW stored value, case-SENSITIVE ("Hello"- // does not start with "hel").+ // - keyword: match the RAW stored value, case-SENSITIVE by default+ // ("Hello" does not start with "hel") unless `case_insensitive` true.// - analysed text: match the analysed (lower-cased) token stream// against the RAW query ("hel" prefixes "hello"; "Hel" does not).// - unknown field (schemaless shim caller — no `Schema` in scope):@@ -46169,7 +46228,11 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sclet value_lc = value.to_lowercase();let matches_str = |s: &str| -> bool {if is_keyword {- s.starts_with(value.as_str())+ if *case_insensitive {+ s.to_lowercase().starts_with(&value_lc)+ } else {+ s.starts_with(value.as_str())+ }} else if is_text {s.split(|c: char| !c.is_alphanumeric()).any(|tok| {!tok.is_empty() && tok.to_lowercase().starts_with(value.as_str())@@ -47031,13 +47094,28 @@ pub(crate) fn doc_matches_query_typed(q: &QueryNode, source: &Value, schema: &Sc.unwrap_or(false)}- QueryNode::Regexp { field, pattern } => {+ QueryNode::Regexp {+ field,+ pattern,+ case_insensitive,+ } => {get_field_value(source, field).and_then(|v| match v {Value::String(s) => {// Compile-once (thread-local cache) — this arm runs// per candidate doc and used to recompile every time.- compiled_anchored_regex(pattern).map(|re| re.is_match(&s))+ // #1812: `case_insensitive` on a KEYWORD field compiles+ // the same anchored pattern with the inline `(?i)` flag+ // (still full-string match — the anchored wrapper is+ // unchanged, so complex patterns keep ES semantics).+ let ft = declared_field(schema, field).map(|f| &f.field_type);+ let is_keyword = matches!(ft, Some(FieldType::Keyword));+ if is_keyword && *case_insensitive {+ compiled_anchored_regex(&format!("(?i){}", pattern))+ .map(|re| re.is_match(&s))+ } else {+ compiled_anchored_regex(pattern).map(|re| re.is_match(&s))+ }}_ => None,})@@ -52290,6 +52368,7 @@ fn query_node_to_fts_projected(value,boost,constant_score,+ case_insensitive,} => {// `prefix` on a KEYWORD field: whole-value, case-sensitive prefix// match over the keyword FST term dictionary — byte-identical to ES@@ -52306,7 +52385,17 @@ fn query_node_to_fts_projected(// exact. TEXT fields keep the doc-scan (None): their prefix runs// against analyzed tokens, which the FST-whole-value route can't// reproduce.+ //+ // #1812: `case_insensitive: true` makes the KEYWORD FST route+ // bail to the brute scan — that route expands CASE-SENSITIVELY+ // and can't fold, while the doc-scan arm folds both sides for+ // keyword fields. TEXT keeps its route: analyzed terms are+ // already lowercased and the pattern is matched raw (flag+ // irrelevant on text, per ES).if exact_fields.contains(field.as_str()) {+ if *case_insensitive {+ return None;+ }return Some(FtsQuery::Prefix(xerj_fts::search::PrefixQuery {field: field.clone(),prefix: value.clone(),@@ -53828,14 +53917,25 @@ fn scored_fast_plan(value,boost,constant_score: true,- } if fs.kw.contains(field) => Some(ScoredPlan::Filtered {- filter: vec![ScoredFilterLeaf::KeywordPrefix {- field: field.clone(),- prefix: value.clone(),- }],- must_not: Vec::new(),- score: scoring_boost(boost)?,- }),+ case_insensitive,+ } if fs.kw.contains(field) => {+ // #1812: the keyword-ordinal prefix filter is case-SENSITIVE —+ // a `case_insensitive` prefix must fall through to the brute+ // scan (which folds both sides), like the `Wildcard` arm below+ // threads the flag into its filter instead.+ if *case_insensitive {+ None+ } else {+ Some(ScoredPlan::Filtered {+ filter: vec![ScoredFilterLeaf::KeywordPrefix {+ field: field.clone(),+ prefix: value.clone(),+ }],+ must_not: Vec::new(),+ score: scoring_boost(boost)?,+ })+ }+ }QueryNode::Wildcard {field,value,@@ -56101,6 +56201,7 @@ mod fts_projection_tests {value: "run".into(),boost: None,constant_score: true,+ case_insensitive: false,};match query_node_to_fts(&q, &tf, &kw(&[])).expect("text prefix projects") {FtsQuery::Prefix(p) => {@@ -60725,6 +60826,7 @@ mod exact_scan_hydration_tests {value: "t".into(),boost: None,constant_score: false,+ case_insensitive: false,},QueryNode::Bool {must: vec![exists("tag")],diff --git a/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rs b/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rsnew file mode 100644index 000000000..566aca15e--- /dev/null+++ b/engine/crates/xerj-engine/tests/prefix_keyword_case_insensitive.rs@@ -0,0 +1,181 @@+//! #1812: `prefix` on a `keyword` field must honor `case_insensitive`.+//!+//! ES matches `prefix` against the RAW keyword value (case-SENSITIVE by+//! default); `case_insensitive: true` folds both sides instead. XERJ's+//! `Prefix` AST node dropped the flag entirely, so+//! `prefix:{k:{value:"HELLO",case_insensitive:true}}` on `"Hello World"`+//! returned 0 (wrong — ES returns the doc).+//!+//! Both the buffered (memtable) and flushed (segment) paths are asserted+//! pre- and post-`refresh`, mirroring `prefix_keyword_case_flush_parity.rs`:+//! a fix applied to only one path split-brains across a background flush,+//! which this repo treats as a regression class of its own.++use std::sync::Arc;++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++/// Index two keyword docs (capitalized + lowercase) and return the index.+async fn setup_docs(engine: &Engine) -> Arc<Index> {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("k", FieldType::Keyword));+ engine.create_index("docs", schema).expect("create");++ let idx = engine.get_index("docs").expect("get");+ idx.index_document(Some("d1".to_string()), json!({ "k": "Hello World" }))+ .await+ .expect("index");+ idx.index_document(Some("d2".to_string()), json!({ "k": "hello there" }))+ .await+ .expect("index");+ idx+}++async fn hits(idx: &Index, query: serde_json::Value) -> u64 {+ let req = parse_request(&json!({ "query": query, "size": 0 })).expect("parse_request");+ idx.search(&req).await.expect("search").total.value+}++/// FAIL-BEFORE (#1812): the flag was dropped at parse time, so this matched+/// 0 docs. ES returns 2 (both keyword values start with "hello", folded).+#[tokio::test]+async fn prefix_case_insensitive_true_folds_both_paths() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ let q = json!({ "prefix": { "k": { "value": "HELLO", "case_insensitive": true } } });++ // Buffered (memtable) path.+ let pre = hits(&idx, q.clone()).await;+ // Flushed (segment) path.+ idx.refresh().await.expect("refresh");+ let post = hits(&idx, q).await;++ assert_eq!(+ pre, 2,+ "memtable: case_insensitive prefix must match both docs"+ );+ assert_eq!(+ pre, post,+ "flush-invariant: memtable={pre} vs segment={post} disagree"+ );+ assert_eq!(+ post, 2,+ "segment: case_insensitive prefix must match both docs"+ );+}++/// Explicit `case_insensitive: false` stays ES-correct case-SENSITIVE.+#[tokio::test]+async fn prefix_case_insensitive_false_is_case_sensitive() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ // "HELLO" prefixes neither "Hello World" nor "hello there" byte-wise.+ let q_upper = json!({ "prefix": { "k": { "value": "HELLO", "case_insensitive": false } } });+ // "hello" prefixes only the lowercase doc byte-wise.+ let q_lower = json!({ "prefix": { "k": { "value": "hello", "case_insensitive": false } } });++ let pre_upper = hits(&idx, q_upper.clone()).await;+ let pre_lower = hits(&idx, q_lower.clone()).await;+ idx.refresh().await.expect("refresh");+ let post_upper = hits(&idx, q_upper).await;+ let post_lower = hits(&idx, q_lower).await;++ assert_eq!(+ pre_upper, 0,+ "memtable: HELLO must not match capitalized values"+ );+ assert_eq!(+ pre_lower, 1,+ "memtable: hello must match only the lowercase doc"+ );+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");+ assert_eq!(+ post_upper, 0,+ "segment: HELLO must not match capitalized values"+ );+ assert_eq!(+ post_lower, 1,+ "segment: hello must match only the lowercase doc"+ );+}++/// Omitted parameter defaults to `false` (shorthand and expanded forms).+#[tokio::test]+async fn prefix_omitted_case_insensitive_defaults_false() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ let q_upper = json!({ "prefix": { "k": "HELLO" } });+ let q_lower = json!({ "prefix": { "k": "hello" } });++ let pre_upper = hits(&idx, q_upper.clone()).await;+ let pre_lower = hits(&idx, q_lower.clone()).await;+ idx.refresh().await.expect("refresh");+ let post_upper = hits(&idx, q_upper).await;+ let post_lower = hits(&idx, q_lower).await;++ assert_eq!(pre_upper, 0, "shorthand HELLO defaults to case-sensitive");+ assert_eq!(+ pre_lower, 1,+ "shorthand hello matches only the lowercase doc"+ );+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");+}++/// TEXT-field semantics are UNCHANGED by the flag: prefix runs against the+/// analyzed (lowercased) token stream, so a lowercase pattern matches and an+/// uppercase one does not — with or without `case_insensitive: true`.+#[tokio::test]+async fn prefix_case_insensitive_does_not_change_text_semantics() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);++ let mut schema = Schema::empty();+ schema.fields.push(FieldConfig::new("t", FieldType::Text));+ engine.create_index("docs", schema).expect("create");+ let idx = engine.get_index("docs").expect("get");+ idx.index_document(Some("d1".to_string()), json!({ "t": "Hello World" }))+ .await+ .expect("index");+ idx.refresh().await.expect("refresh");++ let q_lower = json!({ "prefix": { "t": { "value": "hel", "case_insensitive": false } } });+ let q_lower_ci = json!({ "prefix": { "t": { "value": "hel", "case_insensitive": true } } });+ let q_upper = json!({ "prefix": { "t": { "value": "HEL", "case_insensitive": true } } });++ assert_eq!(+ hits(&idx, q_lower).await,+ 1,+ "text: lowercase prefix matches the analyzed token"+ );+ assert_eq!(+ hits(&idx, q_lower_ci).await,+ 1,+ "text: case_insensitive must not change analyzed-token matching"+ );+ assert_eq!(+ hits(&idx, q_upper).await,+ 0,+ "text: uppercase prefix does not match the lowercased token (ES analyzes the doc, not the pattern)"+ );+}diff --git a/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rs b/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rsnew file mode 100644index 000000000..e16c8ca2e--- /dev/null+++ b/engine/crates/xerj-engine/tests/regexp_keyword_case_insensitive.rs@@ -0,0 +1,167 @@+//! #1812: `regexp` on a `keyword` field must honor `case_insensitive`.+//!+//! ES matches `regexp` against the RAW keyword value (case-SENSITIVE by+//! default); `case_insensitive: true` folds both sides instead. XERJ's+//! `Regexp` AST node dropped the flag entirely, so+//! `regexp:{k:{value:"HEL+O.*",case_insensitive:true}}` on `"Hello World"`+//! returned 0 (wrong — ES returns the doc).+//!+//! Both the buffered (memtable) and flushed (segment) paths are asserted+//! pre- and post-`refresh`: the segment path answers via a dictionary+//! expansion pre-filter (`regexp_segment_expansion` /+//! `build_regexp_prefilter_cached`) that compiles the pattern itself, so a+//! fix only in the doc-scan arm would split-brain across a flush.++use std::sync::Arc;++use serde_json::json;+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::{FieldConfig, FieldType, Schema};+use xerj_engine::{Engine, Index};+use xerj_query::parse_request;++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++/// Index two keyword docs (capitalized + lowercase) and return the index.+async fn setup_docs(engine: &Engine) -> Arc<Index> {+ let mut schema = Schema::empty();+ schema+ .fields+ .push(FieldConfig::new("k", FieldType::Keyword));+ engine.create_index("docs", schema).expect("create");++ let idx = engine.get_index("docs").expect("get");+ idx.index_document(Some("d1".to_string()), json!({ "k": "Hello World" }))+ .await+ .expect("index");+ idx.index_document(Some("d2".to_string()), json!({ "k": "hello there" }))+ .await+ .expect("index");+ idx+}++async fn hits(idx: &Index, query: serde_json::Value) -> u64 {+ let req = parse_request(&json!({ "query": query, "size": 0 })).expect("parse_request");+ idx.search(&req).await.expect("search").total.value+}++/// FAIL-BEFORE (#1812): the flag was dropped at parse time, so this matched+/// 0 docs. ES returns 2 (both keyword values full-match "hel+o.*", folded).+#[tokio::test]+async fn regexp_case_insensitive_true_folds_both_paths() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ let q = json!({ "regexp": { "k": { "value": "HEL+O.*", "case_insensitive": true } } });++ // Buffered (memtable) path.+ let pre = hits(&idx, q.clone()).await;+ // Flushed (segment) path — exercised through the dictionary expansion.+ idx.refresh().await.expect("refresh");+ let post = hits(&idx, q).await;++ assert_eq!(+ pre, 2,+ "memtable: case_insensitive regexp must match both docs"+ );+ assert_eq!(+ pre, post,+ "flush-invariant: memtable={pre} vs segment={post} disagree"+ );+ assert_eq!(+ post, 2,+ "segment: case_insensitive regexp must match both docs"+ );+}++/// Explicit `case_insensitive: false` stays ES-correct case-SENSITIVE.+#[tokio::test]+async fn regexp_case_insensitive_false_is_case_sensitive() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ // "HEL+O.*" full-matches neither value byte-wise.+ let q_upper = json!({ "regexp": { "k": { "value": "HEL+O.*", "case_insensitive": false } } });+ // "hel+o.*" full-matches only the lowercase doc byte-wise.+ let q_lower = json!({ "regexp": { "k": { "value": "hel+o.*", "case_insensitive": false } } });++ let pre_upper = hits(&idx, q_upper.clone()).await;+ let pre_lower = hits(&idx, q_lower.clone()).await;+ idx.refresh().await.expect("refresh");+ let post_upper = hits(&idx, q_upper).await;+ let post_lower = hits(&idx, q_lower).await;++ assert_eq!(+ pre_upper, 0,+ "memtable: HEL+O.* must not match capitalized values"+ );+ assert_eq!(+ pre_lower, 1,+ "memtable: hel+o.* must match only the lowercase doc"+ );+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");+ assert_eq!(+ post_upper, 0,+ "segment: HEL+O.* must not match capitalized values"+ );+ assert_eq!(+ post_lower, 1,+ "segment: hel+o.* must match only the lowercase doc"+ );+}++/// Omitted parameter defaults to `false` (shorthand and expanded forms).+#[tokio::test]+async fn regexp_omitted_case_insensitive_defaults_false() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ let q_upper = json!({ "regexp": { "k": "HEL+O.*" } });+ let q_lower = json!({ "regexp": { "k": "hel+o.*" } });++ let pre_upper = hits(&idx, q_upper.clone()).await;+ let pre_lower = hits(&idx, q_lower.clone()).await;+ idx.refresh().await.expect("refresh");+ let post_upper = hits(&idx, q_upper).await;+ let post_lower = hits(&idx, q_lower).await;++ assert_eq!(pre_upper, 0, "shorthand HEL+O.* defaults to case-sensitive");+ assert_eq!(+ pre_lower, 1,+ "shorthand hel+o.* matches only the lowercase doc"+ );+ assert_eq!(pre_upper, post_upper, "flush-invariant (upper)");+ assert_eq!(pre_lower, post_lower, "flush-invariant (lower)");+}++/// The flag must not change matching for values it shouldn't reach: a+/// case-insensitive pattern that only matches via folding must NOT match a+/// non-string value path, and an invalid pattern still matches nothing.+#[tokio::test]+async fn regexp_case_insensitive_invalid_pattern_matches_nothing() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = setup_docs(&engine).await;++ let q = json!({ "regexp": { "k": { "value": "([", "case_insensitive": true } } });+ assert_eq!(+ hits(&idx, q.clone()).await,+ 0,+ "invalid pattern matches nothing (memtable)"+ );+ idx.refresh().await.expect("refresh");+ assert_eq!(+ hits(&idx, q).await,+ 0,+ "invalid pattern matches nothing (segment)"+ );+}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex c38f1b152..ab0d50a6e 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -490,6 +490,12 @@ pub enum QueryNode {/// match_bool_prefix) that keep BM25 scoring.#[serde(default, skip_serializing_if = "is_false")]constant_score: bool,+ /// #1812: fold case on both sides when matching a KEYWORD field.+ /// `false` (default) = ES-correct case-sensitive keyword prefix.+ /// `true` is set for `prefix{case_insensitive:true}`.+ /// Ignored for `text` fields.+ #[serde(default, skip_serializing_if = "is_false")]+ case_insensitive: bool,},/// Wildcard pattern match (`?` = any char, `*` = zero-or-more chars).@@ -714,7 +720,15 @@ pub enum QueryNode {},/// Regular expression match against a field value.- Regexp { field: String, pattern: String },+ Regexp {+ field: String,+ pattern: String,+ /// #1812: fold case on both sides when matching a KEYWORD field.+ /// `false` (default) = ES-correct case-sensitive keyword regexp.+ /// Ignored for `text` fields.+ #[serde(default, skip_serializing_if = "is_false")]+ case_insensitive: bool,+ },/// Position-aware text match (ES `intervals` query). The `rule` is the/// raw rule JSON (one of match/all_of/any_of/prefix/wildcard/fuzzy,diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex 870cdad65..4b29c39a6 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -924,6 +924,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {value: tokens[0].clone(),boost: fb,constant_score: false,+ case_insensitive: false,};should.push(prefix);continue;@@ -957,6 +958,7 @@ fn parse_multi_match(params: &Value) -> Result<QueryNode> {value: tokens[last].clone(),boost: None,constant_score: false,+ case_insensitive: false,});let (im, is, imm) = if operator_and {(inner_clauses, vec![], None)@@ -1301,6 +1303,7 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {value: value.to_string(),boost: None,constant_score: true,+ case_insensitive: false,});}@@ -1326,6 +1329,10 @@ fn parse_prefix(params: &Value) -> Result<QueryNode> {value,boost,constant_score: true,+ case_insensitive: inner+ .get("case_insensitive")+ .and_then(|v| v.as_bool())+ .unwrap_or(false),},name,))@@ -2413,6 +2420,7 @@ fn parse_regexp(params: &Value) -> Result<QueryNode> {return Ok(QueryNode::Regexp {field,pattern: pattern.to_string(),+ case_insensitive: false,});}@@ -2421,7 +2429,15 @@ fn parse_regexp(params: &Value) -> Result<QueryNode> {.ok_or_else(|| qerr("`regexp` field value must be a string or object"))?;let pattern = string_field(inner, "value")?;- Ok(QueryNode::Regexp { field, pattern })+ let case_insensitive = inner+ .get("case_insensitive")+ .and_then(|v| v.as_bool())+ .unwrap_or(false);+ Ok(QueryNode::Regexp {+ field,+ pattern,+ case_insensitive,+ })}fn parse_match_phrase_prefix(params: &Value) -> Result<QueryNode> {@@ -4728,6 +4744,7 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {value: fold(&raw_tokens[0]),boost: None,constant_score: true,+ case_insensitive: false,});}@@ -4764,6 +4781,9 @@ fn parse_match_bool_prefix(params: &Value) -> Result<QueryNode> {value: fold(&raw_tokens[last_idx]),boost: None,constant_score: false,+ // `match_bool_prefix` lowers to an internal prefix over ANALYZED+ // tokens — no user-facing `case_insensitive` option to thread.+ case_insensitive: false,});// operator:and → every clause must match; otherwise should + mm.diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rsindex cd366523e..2546f591c 100644--- a/engine/crates/xerj-query/src/planner.rs+++ b/engine/crates/xerj-query/src/planner.rs@@ -496,7 +496,7 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {},// Regexp: full scan cost since we must evaluate each doc.- QueryNode::Regexp { field, pattern } => ExecutionPlan::FtsScan {+ QueryNode::Regexp { field, pattern, .. } => ExecutionPlan::FtsScan {field,terms: vec![serde_json::Value::String(pattern)],cost: 20.0,diff --git a/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.yml b/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.ymlnew file mode 100644index 000000000..f571dbc21--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/770_prefix_regexp_case_insensitive.yml@@ -0,0 +1,122 @@+setup:++ - do:+ indices.create:+ index: ci+ body:+ mappings:+ properties:+ k:+ type: keyword++ - do:+ bulk:+ refresh: true+ body:+ - '{ "index": { "_index": "ci", "_id": "1" } }'+ - '{ "k": "Hello World" }'+ - '{ "index": { "_index": "ci", "_id": "2" } }'+ - '{ "k": "hello there" }'++---+"prefix case_insensitive true finds both":+ - do:+ count:+ index: ci+ body:+ query:+ prefix:+ k:+ value: "HELLO"+ case_insensitive: true+ - match: {count: 2}++---+"regexp case_insensitive true finds both":+ - do:+ count:+ index: ci+ body:+ query:+ regexp:+ k:+ value: "HEL+O.*"+ case_insensitive: true+ - match: {count: 2}++---+"prefix case_insensitive false finds none":+ - do:+ count:+ index: ci+ body:+ query:+ prefix:+ k:+ value: "HELLO"+ case_insensitive: false+ - match: {count: 0}++---+"regexp case_insensitive false finds none":+ - do:+ count:+ index: ci+ body:+ query:+ regexp:+ k:+ value: "HEL+O.*"+ case_insensitive: false+ - match: {count: 0}++---+"prefix default false finds none for HELLO, one for hello":+ - do:+ count:+ index: ci+ body:+ query:+ prefix:+ k: "HELLO"+ - match: {count: 0}+ - do:+ count:+ index: ci+ body:+ query:+ prefix:+ k: "hello"+ - match: {count: 1}++---+"regexp default false finds none for HELLO pattern, one for lowercase":+ - do:+ count:+ index: ci+ body:+ query:+ regexp:+ k: "HEL+O.*"+ - match: {count: 0}+ - do:+ count:+ index: ci+ body:+ query:+ regexp:+ k: "hel+o.*"+ - match: {count: 1}++---+"wildcard control with case_insensitive true finds both":+ - do:+ count:+ index: ci+ body:+ query:+ wildcard:+ k:+ value: "HELLO*"+ case_insensitive: true+ - match: {count: 2}