Skip to content

Apache DataFusion SQL Query Engine

fix: preserve NULL semantics in bitwise xor simplification (#24248)

linfeng50cdbde1ed4b
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 2b606687d4..b7e9624946 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -1372,7 +1372,7 @@ impl TreeNodeRewriter for Simplifier<'_> {
left,
op: BitwiseXor,
right,
- }) if expr_contains(&left, &right, BitwiseXor) => {
+ }) if !info.nullable(&right)? && expr_contains(&left, &right, BitwiseXor) => {
let expr = delete_xor_in_complex_expr(&left, &right, false);
Transformed::yes(if expr == *right {
Expr::Literal(
@@ -1389,7 +1389,7 @@ impl TreeNodeRewriter for Simplifier<'_> {
left,
op: BitwiseXor,
right,
- }) if expr_contains(&right, &left, BitwiseXor) => {
+ }) if !info.nullable(&left)? && expr_contains(&right, &left, BitwiseXor) => {
let expr = delete_xor_in_complex_expr(&right, &left, true);
Transformed::yes(if expr == *left {
Expr::Literal(
@@ -3020,16 +3020,19 @@ mod tests {
// c2 ^ ((c2 ^ (c2 | c1)) ^ (c1 & c2)) --> (c2 | c1) ^ (c1 & c2)
let expr = bitwise_xor(
- col("c2"),
+ col("c2_non_null"),
bitwise_xor(
- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_xor(
+ col("c2_non_null"),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ ),
+ bitwise_and(col("c1"), col("c2_non_null")),
),
);
let expected = bitwise_xor(
- bitwise_or(col("c2"), col("c1")),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ bitwise_and(col("c1"), col("c2_non_null")),
);
assert_eq!(simplify(expr), expected);
@@ -3038,18 +3041,24 @@ mod tests {
// c2 ^ (c2 ^ (c2 | c1)) ^ ((c1 & c2) ^ c2) --> c2 ^ ((c2 | c1) ^ (c1 & c2))
let expr = bitwise_xor(
- col("c2"),
+ col("c2_non_null"),
bitwise_xor(
- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
- bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),
+ bitwise_xor(
+ col("c2_non_null"),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ ),
+ bitwise_xor(
+ bitwise_and(col("c1"), col("c2_non_null")),
+ col("c2_non_null"),
+ ),
),
);
let expected = bitwise_xor(
- col("c2"),
+ col("c2_non_null"),
bitwise_xor(
- bitwise_or(col("c2"), col("c1")),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ bitwise_and(col("c1"), col("c2_non_null")),
),
);
@@ -3060,15 +3069,18 @@ mod tests {
let expr = bitwise_xor(
bitwise_xor(
- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_xor(
+ col("c2_non_null"),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ ),
+ bitwise_and(col("c1"), col("c2_non_null")),
),
- col("c2"),
+ col("c2_non_null"),
);
let expected = bitwise_xor(
- bitwise_or(col("c2"), col("c1")),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ bitwise_and(col("c1"), col("c2_non_null")),
);
assert_eq!(simplify(expr), expected);
@@ -3078,23 +3090,47 @@ mod tests {
let expr = bitwise_xor(
bitwise_xor(
- bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
- bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),
+ bitwise_xor(
+ col("c2_non_null"),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ ),
+ bitwise_xor(
+ bitwise_and(col("c1"), col("c2_non_null")),
+ col("c2_non_null"),
+ ),
),
- col("c2"),
+ col("c2_non_null"),
);
let expected = bitwise_xor(
bitwise_xor(
- bitwise_or(col("c2"), col("c1")),
- bitwise_and(col("c1"), col("c2")),
+ bitwise_or(col("c2_non_null"), col("c1")),
+ bitwise_and(col("c1"), col("c2_non_null")),
),
- col("c2"),
+ col("c2_non_null"),
);
assert_eq!(simplify(expr), expected);
}
+ #[test]
+ fn test_does_not_cancel_nullable_bitwise_xor() {
+ let nullable = col("c3");
+ let other = col("c3_non_null");
+
+ let expr = bitwise_xor(nullable.clone(), nullable.clone());
+ assert_eq!(simplify(expr.clone()), expr);
+
+ let expr = bitwise_xor(
+ bitwise_xor(nullable.clone(), other.clone()),
+ nullable.clone(),
+ );
+ assert_eq!(simplify(expr.clone()), expr);
+
+ let expr = bitwise_xor(nullable.clone(), bitwise_xor(other, nullable));
+ assert_eq!(simplify(expr.clone()), expr);
+ }
+
#[test]
fn test_simplify_negated_bitwise_and() {
// !c3 & c3 --> 0
@@ -3184,13 +3220,13 @@ mod tests {
#[test]
fn test_simplify_simple_bitwise_xor() {
// c4 ^ c4 -> 0
- let expr = (col("c4")).bitxor(col("c4"));
+ let expr = (col("c4_non_null")).bitxor(col("c4_non_null"));
let expected = lit(0u32);
assert_eq!(simplify(expr), expected);
// c3 ^ c3 -> 0
- let expr = col("c3").bitxor(col("c3"));
+ let expr = col("c3_non_null").bitxor(col("c3_non_null"));
let expected = lit(0i64);
assert_eq!(simplify(expr), expected);
diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt
index 7666b680e1..2bedb28a85 100644
--- a/datafusion/sqllogictest/test_files/scalar.slt
+++ b/datafusion/sqllogictest/test_files/scalar.slt
@@ -1444,6 +1444,17 @@ select a ^ b, c ^ d, e ^ f from signed_integers;
-998 -133 -16
NULL NULL NULL
+# repeated nullable operands must not be cancelled
+query III rowsort
+select
+ (a XOR b) XOR a AS left_associative,
+ a XOR (b XOR a) AS right_associative,
+ a XOR a AS self_xor
+from (values (NULL::INT, 7), (3, 7)) as t(a, b);
+----
+7 7 0
+NULL NULL NULL
+
# bitwise xor with other operators
query II rowsort
select 2 * c - 1 ^ 856 + d + 3, d ^ 7 >> 4 from signed_integers;