Apache DataFusion SQL Query Engine
fix: `Operator::returns_null_on_null()` should include string concat (`||`) (#22458)
Neil Conway51b51e8d8a09
diff --git a/datafusion/expr-common/src/operator.rs b/datafusion/expr-common/src/operator.rsindex 7b10d9bfae..b15e770802 100644--- a/datafusion/expr-common/src/operator.rs+++ b/datafusion/expr-common/src/operator.rs@@ -36,15 +36,15 @@ pub enum Operator {Plus,/// SubtractionMinus,- /// Multiplication operator, like `*`+ /// MultiplicationMultiply,- /// Division operator, like `/`+ /// DivisionDivide,- /// Remainder operator, like `%`+ /// RemainderModulo,- /// Logical AND, like `&&`+ /// Logical ANDAnd,- /// Logical OR, like `||`+ /// Logical OROr,/// `IS DISTINCT FROM` (see [`distinct`])///@@ -80,20 +80,20 @@ pub enum Operator {BitwiseShiftRight,/// Bitwise left, like `<<`BitwiseShiftLeft,- /// String concat+ /// String concatenation, like `||`StringConcat,/// At arrow, like `@>`.////// Currently only supported to be used with lists:/// ```sql- /// select [1,3] <@ [1,2,3]+ /// select [1,2,3] @> [1,3]/// ```AtArrow,/// Arrow at, like `<@`.////// Currently only supported to be used with lists:/// ```sql- /// select [1,2,3] @> [1,3]+ /// select [1,3] <@ [1,2,3]/// ```ArrowAt,/// Arrow, like `->`.@@ -120,7 +120,7 @@ pub enum Operator {////// Not implemented in DataFusion yet.IntegerDivide,- /// Hash Minis, like `#-`+ /// Hash Minus, like `#-`////// Not implemented in DataFusion yet.HashMinus,@@ -163,6 +163,10 @@ impl Operator {Operator::ILikeMatch => Some(Operator::NotILikeMatch),Operator::NotLikeMatch => Some(Operator::LikeMatch),Operator::NotILikeMatch => Some(Operator::ILikeMatch),+ Operator::RegexMatch => Some(Operator::RegexNotMatch),+ Operator::RegexIMatch => Some(Operator::RegexNotIMatch),+ Operator::RegexNotMatch => Some(Operator::RegexMatch),+ Operator::RegexNotIMatch => Some(Operator::RegexIMatch),Operator::Plus| Operator::Minus| Operator::Multiply@@ -170,10 +174,6 @@ impl Operator {| Operator::Modulo| Operator::And| Operator::Or- | Operator::RegexMatch- | Operator::RegexIMatch- | Operator::RegexNotMatch- | Operator::RegexNotIMatch| Operator::BitwiseAnd| Operator::BitwiseOr| Operator::BitwiseXor@@ -377,7 +377,8 @@ impl Operator {| Operator::Question| Operator::QuestionAnd| Operator::QuestionPipe- | Operator::Colon => true,+ | Operator::Colon+ | Operator::StringConcat => true,// E.g. `TRUE OR NULL` is `TRUE`Operator::Or@@ -385,9 +386,7 @@ impl Operator {| Operator::And// IS DISTINCT FROM and IS NOT DISTINCT FROM always return a TRUE/FALSE value, never NULL| Operator::IsDistinctFrom- | Operator::IsNotDistinctFrom- // DataFusion string concatenation operator treats NULL as an empty string- | Operator::StringConcat => false,+ | Operator::IsNotDistinctFrom => false,}}diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 143d8eae69..39c8541b51 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -2919,6 +2919,21 @@ mod tests {}}+ #[test]+ fn test_simplify_concat_by_null() {+ let null = Expr::Literal(ScalarValue::Utf8(None), None);+ // A || null --> null+ {+ let expr = binary_expr(col("c1"), Operator::StringConcat, null.clone());+ assert_eq!(simplify(expr), null);+ }+ // null || A --> null+ {+ let expr = binary_expr(null.clone(), Operator::StringConcat, col("c1"));+ assert_eq!(simplify(expr), null);+ }+ }+#[test]fn test_simplify_composed_bitwise_and() {// ((c2 > 5) & (c1 < 6)) & (c2 > 5) --> (c2 > 5) & (c1 < 6)@@ -3538,6 +3553,32 @@ mod tests {assert_no_change(regex_match(col("c1"), lit("foo|bar|baz|blarg|bozo|etc")));}+ #[test]+ fn test_simplify_not_regex_match() {+ let pattern = || lit("foo.*");++ // NOT (c1 ~ pattern) --> c1 !~ pattern+ assert_eq!(+ simplify(regex_match(col("c1"), pattern()).not()),+ regex_not_match(col("c1"), pattern()),+ );+ // NOT (c1 !~ pattern) --> c1 ~ pattern+ assert_eq!(+ simplify(regex_not_match(col("c1"), pattern()).not()),+ regex_match(col("c1"), pattern()),+ );+ // NOT (c1 ~* pattern) --> c1 !~* pattern+ assert_eq!(+ simplify(regex_imatch(col("c1"), pattern()).not()),+ regex_not_imatch(col("c1"), pattern()),+ );+ // NOT (c1 !~* pattern) --> c1 ~* pattern+ assert_eq!(+ simplify(regex_not_imatch(col("c1"), pattern()).not()),+ regex_imatch(col("c1"), pattern()),+ );+ }+#[track_caller]fn assert_no_change(expr: Expr) {let optimized = simplify(expr.clone());