Apache DataFusion SQL Query Engine
fix: regex simplification of anchored patterns produces wrong results (#22727)
linfengc0dc5714f3b3
diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rsindex b341c328e9..df4c344b2e 100644--- a/datafusion/optimizer/src/simplify_expressions/regex.rs+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs@@ -283,20 +283,23 @@ fn partial_anchored_literal_to_like(v: &[Hir]) -> Option<String> {/// Extracts a string literal expression assuming that [`is_anchored_literal`]/// returned true.-fn anchored_literal_to_expr(v: &[Hir]) -> Option<Expr> {+fn anchored_literal_to_expr(v: &[Hir], string_scalar: &StringScalar) -> Option<Expr> {match v.len() {- 2 => Some(lit("")),+ 2 => Some(string_scalar.to_expr("")),3 => {let HirKind::Literal(l) = v[1].kind() else {return None;};- like_str_from_literal(l).map(lit)+ like_str_from_literal(l).map(|s| string_scalar.to_expr(s))}_ => None,}}-fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {+fn anchored_alternation_to_exprs(+ v: &[Hir],+ string_scalar: &StringScalar,+) -> Option<Vec<Expr>> {if 3 != v.len() {return None;}@@ -308,7 +311,8 @@ fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {for hir in alters {let mut is_safe = false;if let HirKind::Literal(l) = hir.kind()- && let Some(safe_literal) = str_from_literal(l).map(lit)+ && let Some(safe_literal) =+ str_from_literal(l).map(|s| string_scalar.to_expr(s)){literals.push(safe_literal);is_safe = true;@@ -321,7 +325,9 @@ fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {return Some(literals);} else if let HirKind::Literal(l) = sub.kind() {- if let Some(safe_literal) = str_from_literal(l).map(lit) {+ if let Some(safe_literal) =+ str_from_literal(l).map(|s| string_scalar.to_expr(s))+ {return Some(vec![safe_literal]);}return None;@@ -351,12 +357,18 @@ fn lower_simple());}HirKind::Concat(inner) if is_anchored_literal(inner) => {- return anchored_literal_to_expr(inner).map(|right| {- mode.expr_matches_literal(Box::new(left.clone()), Box::new(right))+ return anchored_literal_to_expr(inner, string_scalar).map(|right| {+ if mode.i {+ // Case-insensitive: use ILIKE for exact match (no wildcards)+ mode.expr(Box::new(left.clone()), Box::new(right))+ } else {+ // Case-sensitive: use Eq / NotEq+ mode.expr_matches_literal(Box::new(left.clone()), Box::new(right))+ }});}- HirKind::Concat(inner) if is_anchored_capture(inner) => {- return anchored_alternation_to_exprs(inner)+ HirKind::Concat(inner) if !mode.i && is_anchored_capture(inner) => {+ return anchored_alternation_to_exprs(inner, string_scalar).map(|right| left.clone().in_list(right, mode.not));}HirKind::Concat(inner) => {diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.sltindex 5e68aba1f4..b4482a3af1 100644--- a/datafusion/sqllogictest/test_files/predicates.slt+++ b/datafusion/sqllogictest/test_files/predicates.slt@@ -204,12 +204,171 @@ SELECT * FROM test WHERE column1 ~ 'z'----Bazzz+query T+SELECT * FROM test WHERE column1 ~ '^Bazzz$'+----+Bazzz++query T+SELECT * FROM test WHERE column1 ~ '^(foo|Bazzz)$'+----+foo+Bazzz++statement ok+CREATE TABLE test_regex_utf8view(s VARCHAR) AS VALUES ('foo'), ('Bazzz');++statement ok+set datafusion.explain.logical_plan_only = true++# `~` anchored literal -> `= Utf8View(..)`+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~ '^Bazzz$'+----+logical_plan+01)Filter: test_regex_utf8view.s = Utf8View("Bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `~*` anchored literal -> `ILIKE Utf8View(..)`+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~* '^bazzz$'+----+logical_plan+01)Filter: test_regex_utf8view.s ILIKE Utf8View("bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `~` anchored alternation -> OR of `= Utf8View(..)` comparisons.+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~ '^(foo|Bazzz)$'+----+logical_plan+01)Filter: test_regex_utf8view.s = Utf8View("foo") OR test_regex_utf8view.s = Utf8View("Bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `~*` anchored alternation -> NOT simplified: it falls back to a regex match,+# because `IN`/`=` cannot express case-insensitive matching.+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~* '^(foo|bazzz)$'+----+logical_plan+01)Filter: test_regex_utf8view.s ~* Utf8View("^(foo|bazzz)$")+02)--TableScan: test_regex_utf8view projection=[s]++# `!~` -> `!= Utf8View(..)`+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~ '^Bazzz$'+----+logical_plan+01)Filter: test_regex_utf8view.s != Utf8View("Bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `!~*` -> `NOT ILIKE Utf8View(..)`+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~* '^bazzz$'+----+logical_plan+01)Filter: test_regex_utf8view.s NOT ILIKE Utf8View("bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `!~` anchored alternation -> AND of `!= Utf8View(..)` comparisons.+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~ '^(foo|Bazzz)$'+----+logical_plan+01)Filter: test_regex_utf8view.s != Utf8View("foo") AND test_regex_utf8view.s != Utf8View("Bazzz")+02)--TableScan: test_regex_utf8view projection=[s]++# `!~*` anchored alternation -> NOT simplified: it falls back to a regex match,+# same reason as the `~*` alternation above.+query TT+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~* '^(foo|bazzz)$'+----+logical_plan+01)Filter: test_regex_utf8view.s !~* Utf8View("^(foo|bazzz)$")+02)--TableScan: test_regex_utf8view projection=[s]++statement ok+set datafusion.explain.logical_plan_only = false++# Result assertions+query T+SELECT * FROM test_regex_utf8view WHERE s ~ '^Bazzz$'+----+Bazzz++query T+SELECT * FROM test_regex_utf8view WHERE s ~ '^(foo|Bazzz)$'+----+foo+Bazzz++# Case-insensitive anchored match over Utf8View: must be simplified to ILIKE+# (not a case-sensitive Eq) and must keep operand types as Utf8View.+query T+SELECT * FROM test_regex_utf8view WHERE s ~* '^bazzz$'+----+Bazzz++# Case-insensitive anchored alternation over Utf8View+query T rowsort+SELECT * FROM test_regex_utf8view WHERE s ~* '^(foo|bazzz)$'+----+Bazzz+foo++query T rowsort+SELECT * FROM test_regex_utf8view WHERE s !~ '^Bazzz$'+----+foo++query T rowsort+SELECT * FROM test_regex_utf8view WHERE s !~* '^bazzz$'+----+foo++# Both rows match the alternation, so the negated forms return nothing.+query T rowsort+SELECT * FROM test_regex_utf8view WHERE s !~ '^(foo|Bazzz)$'+----++query T rowsort+SELECT * FROM test_regex_utf8view WHERE s !~* '^(foo|bazzz)$'+----++statement ok+DROP TABLE test_regex_utf8view;+query TSELECT * FROM test WHERE column1 ~* 'z'----BazzzZZZZZ+query T+SELECT * FROM test WHERE column1 ~* '^barrr$'+----+Barrr++query T+SELECT * FROM test WHERE column1 ~* '^(barrr|bazzz)$'+----+Barrr+Bazzz++query T rowsort+SELECT * FROM test WHERE column1 !~ '^Bazzz$'+----+Barrr+ZZZZZ+foo++query T rowsort+SELECT * FROM test WHERE column1 !~* '^barrr$'+----+Bazzz+ZZZZZ+foo+query TSELECT * FROM test WHERE column1 !~ 'z'----