Apache DataFusion SQL Query Engine
fix: preserve NULL semantics in bitwise xor simplification (#24248)
linfeng50cdbde1ed4b
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rsindex 2b606687d4..b7e9624946 100644--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs@@ -1372,7 +1372,7 @@ impl TreeNodeRewriter for Simplifier<'_> {left,op: BitwiseXor,right,- }) if expr_contains(&left, &right, BitwiseXor) => {+ }) if !info.nullable(&right)? && expr_contains(&left, &right, BitwiseXor) => {let expr = delete_xor_in_complex_expr(&left, &right, false);Transformed::yes(if expr == *right {Expr::Literal(@@ -1389,7 +1389,7 @@ impl TreeNodeRewriter for Simplifier<'_> {left,op: BitwiseXor,right,- }) if expr_contains(&right, &left, BitwiseXor) => {+ }) if !info.nullable(&left)? && expr_contains(&right, &left, BitwiseXor) => {let expr = delete_xor_in_complex_expr(&right, &left, true);Transformed::yes(if expr == *left {Expr::Literal(@@ -3020,16 +3020,19 @@ mod tests {// c2 ^ ((c2 ^ (c2 | c1)) ^ (c1 & c2)) --> (c2 | c1) ^ (c1 & c2)let expr = bitwise_xor(- col("c2"),+ col("c2_non_null"),bitwise_xor(- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),- bitwise_and(col("c1"), col("c2")),+ bitwise_xor(+ col("c2_non_null"),+ bitwise_or(col("c2_non_null"), col("c1")),+ ),+ bitwise_and(col("c1"), col("c2_non_null")),),);let expected = bitwise_xor(- bitwise_or(col("c2"), col("c1")),- bitwise_and(col("c1"), col("c2")),+ bitwise_or(col("c2_non_null"), col("c1")),+ bitwise_and(col("c1"), col("c2_non_null")),);assert_eq!(simplify(expr), expected);@@ -3038,18 +3041,24 @@ mod tests {// c2 ^ (c2 ^ (c2 | c1)) ^ ((c1 & c2) ^ c2) --> c2 ^ ((c2 | c1) ^ (c1 & c2))let expr = bitwise_xor(- col("c2"),+ col("c2_non_null"),bitwise_xor(- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),- bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),+ bitwise_xor(+ col("c2_non_null"),+ bitwise_or(col("c2_non_null"), col("c1")),+ ),+ bitwise_xor(+ bitwise_and(col("c1"), col("c2_non_null")),+ col("c2_non_null"),+ ),),);let expected = bitwise_xor(- col("c2"),+ col("c2_non_null"),bitwise_xor(- bitwise_or(col("c2"), col("c1")),- bitwise_and(col("c1"), col("c2")),+ bitwise_or(col("c2_non_null"), col("c1")),+ bitwise_and(col("c1"), col("c2_non_null")),),);@@ -3060,15 +3069,18 @@ mod tests {let expr = bitwise_xor(bitwise_xor(- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),- bitwise_and(col("c1"), col("c2")),+ bitwise_xor(+ col("c2_non_null"),+ bitwise_or(col("c2_non_null"), col("c1")),+ ),+ bitwise_and(col("c1"), col("c2_non_null")),),- col("c2"),+ col("c2_non_null"),);let expected = bitwise_xor(- bitwise_or(col("c2"), col("c1")),- bitwise_and(col("c1"), col("c2")),+ bitwise_or(col("c2_non_null"), col("c1")),+ bitwise_and(col("c1"), col("c2_non_null")),);assert_eq!(simplify(expr), expected);@@ -3078,23 +3090,47 @@ mod tests {let expr = bitwise_xor(bitwise_xor(- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),- bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),+ bitwise_xor(+ col("c2_non_null"),+ bitwise_or(col("c2_non_null"), col("c1")),+ ),+ bitwise_xor(+ bitwise_and(col("c1"), col("c2_non_null")),+ col("c2_non_null"),+ ),),- col("c2"),+ col("c2_non_null"),);let expected = bitwise_xor(bitwise_xor(- bitwise_or(col("c2"), col("c1")),- bitwise_and(col("c1"), col("c2")),+ bitwise_or(col("c2_non_null"), col("c1")),+ bitwise_and(col("c1"), col("c2_non_null")),),- col("c2"),+ col("c2_non_null"),);assert_eq!(simplify(expr), expected);}+ #[test]+ fn test_does_not_cancel_nullable_bitwise_xor() {+ let nullable = col("c3");+ let other = col("c3_non_null");++ let expr = bitwise_xor(nullable.clone(), nullable.clone());+ assert_eq!(simplify(expr.clone()), expr);++ let expr = bitwise_xor(+ bitwise_xor(nullable.clone(), other.clone()),+ nullable.clone(),+ );+ assert_eq!(simplify(expr.clone()), expr);++ let expr = bitwise_xor(nullable.clone(), bitwise_xor(other, nullable));+ assert_eq!(simplify(expr.clone()), expr);+ }+#[test]fn test_simplify_negated_bitwise_and() {// !c3 & c3 --> 0@@ -3184,13 +3220,13 @@ mod tests {#[test]fn test_simplify_simple_bitwise_xor() {// c4 ^ c4 -> 0- let expr = (col("c4")).bitxor(col("c4"));+ let expr = (col("c4_non_null")).bitxor(col("c4_non_null"));let expected = lit(0u32);assert_eq!(simplify(expr), expected);// c3 ^ c3 -> 0- let expr = col("c3").bitxor(col("c3"));+ let expr = col("c3_non_null").bitxor(col("c3_non_null"));let expected = lit(0i64);assert_eq!(simplify(expr), expected);diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.sltindex 7666b680e1..2bedb28a85 100644--- a/datafusion/sqllogictest/test_files/scalar.slt+++ b/datafusion/sqllogictest/test_files/scalar.slt@@ -1444,6 +1444,17 @@ select a ^ b, c ^ d, e ^ f from signed_integers;-998 -133 -16NULL NULL NULL+# repeated nullable operands must not be cancelled+query III rowsort+select+ (a XOR b) XOR a AS left_associative,+ a XOR (b XOR a) AS right_associative,+ a XOR a AS self_xor+from (values (NULL::INT, 7), (3, 7)) as t(a, b);+----+7 7 0+NULL NULL NULL+# bitwise xor with other operatorsquery II rowsortselect 2 * c - 1 ^ 856 + d + 3, d ^ 7 >> 4 from signed_integers;