Skip to content

Apache DataFusion SQL Query Engine

fix: preserve NULL semantics when simplifying col ~ '.*' (#24380)

Liang-Chi Hsieh16b08db00ddf
diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rs
index f04d9476c4..7dccb5b1b9 100644
--- a/datafusion/optimizer/src/simplify_expressions/regex.rs
+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs
@@ -38,7 +38,7 @@ const ANY_CHAR_REGEX_PATTERN: &str = ".*";
/// - full anchored regex patterns (e.g. `^foo$`) to `= 'foo'`
/// - partial anchored regex patterns (e.g. `^foo`) to `LIKE 'foo%'`
/// - combinations (alternatives) of the above, will be concatenated with `OR` or `AND`
-/// - `EQ .*` to NotNull
+/// - `EQ .*` to `col IS NOT NULL OR NULL` (true for any string, NULL if col is NULL)
/// - `NE .*` to col IS NULL AND Boolean(NULL) (false for any string, or NULL if col is NULL)
///
/// Dev note: unit tests of this function are in `expr_simplifier.rs`, case `test_simplify_regex`.
@@ -75,8 +75,11 @@ pub fn simplify_regex_expr(
right: Box::new(null_bool),
})
} else {
- // not null
- left.is_not_null()
+ // `col ~ '.*'` matches every non-NULL string and yields NULL for a
+ // NULL input (three-valued logic). Mirror the `not` branch so the
+ // NULL row is preserved (`NULL`) instead of collapsing to `false`.
+ let null_bool = lit(ScalarValue::Boolean(None));
+ left.is_not_null().or(null_bool)
};
return Ok(Transformed::yes(new_expr));
}
diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
index 0e72a17abc..1c5a4a1869 100644
--- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
+++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs
@@ -1026,7 +1026,7 @@ mod tests {
]);
let table_scan = table_scan(Some("test"), &schema, None)?.build()?;
- // Test `~ ".*"` transforms to true for any non-NULL string
+ // Test `~ ".*"` is TRUE for any non-NULL string and NULL for a NULL input
let plan = LogicalPlanBuilder::from(table_scan.clone())
.filter(binary_expr(col("a"), Operator::RegexMatch, lit(".*")))?
.build()?;
@@ -1034,7 +1034,7 @@ mod tests {
assert_optimized_plan_equal!(
plan,
@ r"
- Filter: test.a IS NOT NULL
+ Filter: test.a IS NOT NULL OR Boolean(NULL)
TableScan: test
"
)?;
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt
index 57dc440407..158096328e 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -34,10 +34,10 @@ query TT
explain select b from t where b ~ '.*'
----
logical_plan
-01)Filter: t.b IS NOT NULL
+01)Filter: t.b IS NOT NULL OR Boolean(NULL)
02)--TableScan: t projection=[b]
physical_plan
-01)FilterExec: b@0 IS NOT NULL
+01)FilterExec: b@0 IS NOT NULL OR NULL
02)--DataSourceExec: partitions=1, partition_sizes=[1]
query TT
@@ -50,6 +50,24 @@ physical_plan
01)FilterExec: b@0 IS NULL AND NULL
02)--DataSourceExec: partitions=1, partition_sizes=[1]
+# `col ~ '.*'` is TRUE for any non-NULL string and NULL for a NULL input.
+# The `.*` -> IS NOT NULL simplification must preserve that NULL (returning
+# `false` for the NULL row would be wrong in a projection context).
+query TB
+WITH vals(id, col) AS (
+ VALUES
+ (1, 'foo'::text),
+ (2, ''::text),
+ (3, NULL::text)
+)
+SELECT col, col ~ '.*'
+FROM vals
+ORDER BY id
+----
+foo true
+(empty) true
+NULL NULL
+
query TB
WITH vals(id, col) AS (
VALUES