Apache DataFusion SQL Query Engine
fix: preserve NULL semantics when simplifying col ~ '.*' (#24380)
Liang-Chi Hsieh16b08db00ddf
diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rsindex f04d9476c4..7dccb5b1b9 100644--- a/datafusion/optimizer/src/simplify_expressions/regex.rs+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs@@ -38,7 +38,7 @@ const ANY_CHAR_REGEX_PATTERN: &str = ".*";/// - full anchored regex patterns (e.g. `^foo$`) to `= 'foo'`/// - partial anchored regex patterns (e.g. `^foo`) to `LIKE 'foo%'`/// - combinations (alternatives) of the above, will be concatenated with `OR` or `AND`-/// - `EQ .*` to NotNull+/// - `EQ .*` to `col IS NOT NULL OR NULL` (true for any string, NULL if col is NULL)/// - `NE .*` to col IS NULL AND Boolean(NULL) (false for any string, or NULL if col is NULL)////// Dev note: unit tests of this function are in `expr_simplifier.rs`, case `test_simplify_regex`.@@ -75,8 +75,11 @@ pub fn simplify_regex_expr(right: Box::new(null_bool),})} else {- // not null- left.is_not_null()+ // `col ~ '.*'` matches every non-NULL string and yields NULL for a+ // NULL input (three-valued logic). Mirror the `not` branch so the+ // NULL row is preserved (`NULL`) instead of collapsing to `false`.+ let null_bool = lit(ScalarValue::Boolean(None));+ left.is_not_null().or(null_bool)};return Ok(Transformed::yes(new_expr));}diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rsindex 0e72a17abc..1c5a4a1869 100644--- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs+++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs@@ -1026,7 +1026,7 @@ mod tests {]);let table_scan = table_scan(Some("test"), &schema, None)?.build()?;- // Test `~ ".*"` transforms to true for any non-NULL string+ // Test `~ ".*"` is TRUE for any non-NULL string and NULL for a NULL inputlet plan = LogicalPlanBuilder::from(table_scan.clone()).filter(binary_expr(col("a"), Operator::RegexMatch, lit(".*")))?.build()?;@@ -1034,7 +1034,7 @@ mod tests {assert_optimized_plan_equal!(plan,@ r"- Filter: test.a IS NOT NULL+ Filter: test.a IS NOT NULL OR Boolean(NULL)TableScan: test")?;diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.sltindex 57dc440407..158096328e 100644--- a/datafusion/sqllogictest/test_files/simplify_expr.slt+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt@@ -34,10 +34,10 @@ query TTexplain select b from t where b ~ '.*'----logical_plan-01)Filter: t.b IS NOT NULL+01)Filter: t.b IS NOT NULL OR Boolean(NULL)02)--TableScan: t projection=[b]physical_plan-01)FilterExec: b@0 IS NOT NULL+01)FilterExec: b@0 IS NOT NULL OR NULL02)--DataSourceExec: partitions=1, partition_sizes=[1]query TT@@ -50,6 +50,24 @@ physical_plan01)FilterExec: b@0 IS NULL AND NULL02)--DataSourceExec: partitions=1, partition_sizes=[1]+# `col ~ '.*'` is TRUE for any non-NULL string and NULL for a NULL input.+# The `.*` -> IS NOT NULL simplification must preserve that NULL (returning+# `false` for the NULL row would be wrong in a projection context).+query TB+WITH vals(id, col) AS (+ VALUES+ (1, 'foo'::text),+ (2, ''::text),+ (3, NULL::text)+)+SELECT col, col ~ '.*'+FROM vals+ORDER BY id+----+foo true+(empty) true+NULL NULL+query TBWITH vals(id, col) AS (VALUES