Skip to content

Apache DataFusion SQL Query Engine

fix: dedup commutative AND/OR operands when canonicalize is disabled (#23615)

Stefan Wangadb071111f43
diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs
index 3b708b4ede..ad491c9782 100644
--- a/datafusion/expr/src/expr.rs
+++ b/datafusion/expr/src/expr.rs
@@ -2514,7 +2514,7 @@ impl NormalizeEq for Expr {
args: other_args,
}),
) => {
- self_func.name() == other_func.name()
+ self_func == other_func
&& self_args.len() == other_args.len()
&& self_args
.iter()
@@ -2545,7 +2545,7 @@ impl NormalizeEq for Expr {
},
}),
) => {
- self_func.name() == other_func.name()
+ self_func == other_func
&& self_distinct == other_distinct
&& self_null_treatment == other_null_treatment
&& self_args.len() == other_args.len()
@@ -2598,7 +2598,7 @@ impl NormalizeEq for Expr {
},
} = other.as_ref();
- self_fun.name() == other_fun.name()
+ self_fun == other_fun
&& self_window_frame == other_window_frame
&& match (self_filter, other_filter) {
(Some(a), Some(b)) => a.normalize_eq(b),
@@ -2611,10 +2611,12 @@ impl NormalizeEq for Expr {
.iter()
.zip(other_args.iter())
.all(|(a, b)| a.normalize_eq(b))
+ && self_partition_by.len() == other_partition_by.len()
&& self_partition_by
.iter()
.zip(other_partition_by.iter())
.all(|(a, b)| a.normalize_eq(b))
+ && self_order_by.len() == other_order_by.len()
&& self_order_by
.iter()
.zip(other_order_by.iter())
@@ -3831,6 +3833,7 @@ pub fn physical_name(expr: &Expr) -> Result<String> {
#[cfg(test)]
mod test {
use crate::expr_fn::col;
+ use crate::test::function_stub::max_udaf;
use crate::{
ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Volatility, case,
lit, placeholder, qualified_wildcard, wildcard, wildcard_with_options,
@@ -4345,6 +4348,32 @@ mod test {
use super::*;
use crate::logical_plan::{EmptyRelation, LogicalPlan};
+ #[test]
+ fn normalize_eq_window_function_over_clause_lengths() {
+ let window = |partition_by: Vec<Expr>, order_by: Vec<Sort>| {
+ let mut window = WindowFunction::new(max_udaf(), vec![col("value")]);
+ window.params.partition_by = partition_by;
+ window.params.order_by = order_by;
+ Expr::from(window)
+ };
+ let base = window(vec![col("a")], vec![Sort::new(col("a"), true, true)]);
+
+ let extra_partition = window(
+ vec![col("a"), col("b")],
+ vec![Sort::new(col("a"), true, true)],
+ );
+ assert!(!base.normalize_eq(&extra_partition));
+
+ let extra_order = window(
+ vec![col("a")],
+ vec![
+ Sort::new(col("a"), true, true),
+ Sort::new(col("b"), true, true),
+ ],
+ );
+ assert!(!base.normalize_eq(&extra_order));
+ }
+
#[test]
fn test_display_wildcard() {
assert_eq!(format!("{}", wildcard()), "*");
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index dadea47848..ec01becb8f 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -2514,6 +2514,77 @@ mod tests {
}
}
+ #[test]
+ fn test_simplify_swapped_operands_in_and_or_no_canonicalize() {
+ // Regression test for https://github.com/apache/datafusion/issues/14943
+ //
+ // `SimplifyExpressions` disables canonicalization for `LogicalPlan::Join`
+ // (see https://github.com/apache/datafusion/pull/8780), so commutative
+ // operands like `A = B` and `B = A` cannot be normalized to a single
+ // form before AND/OR dedup runs. The dedup itself must therefore
+ // recognize commutative equivalence directly.
+
+ // c3 = 5 AND 5 = c3 --> c3 = 5
+ let expr = col("c3_non_null")
+ .eq(lit(5_i64))
+ .and(lit(5_i64).eq(col("c3_non_null")));
+ let expected = col("c3_non_null").eq(lit(5_i64));
+ assert_eq!(simplify_no_canonicalize(expr), expected);
+
+ // 5 = c3 AND c3 = 5 --> 5 = c3
+ let expr = lit(5_i64)
+ .eq(col("c3_non_null"))
+ .and(col("c3_non_null").eq(lit(5_i64)));
+ let expected = lit(5_i64).eq(col("c3_non_null"));
+ assert_eq!(simplify_no_canonicalize(expr), expected);
+
+ // c3 = 5 OR 5 = c3 --> c3 = 5
+ let expr = col("c3_non_null")
+ .eq(lit(5_i64))
+ .or(lit(5_i64).eq(col("c3_non_null")));
+ let expected = col("c3_non_null").eq(lit(5_i64));
+ assert_eq!(simplify_no_canonicalize(expr), expected);
+
+ // (c3 = 5 AND c4 > 0) AND (5 = c3) --> c3 = 5 AND c4 > 0
+ let expr = col("c3_non_null")
+ .eq(lit(5_i64))
+ .and(col("c4_non_null").gt(lit(0_u32)))
+ .and(lit(5_i64).eq(col("c3_non_null")));
+ let expected = col("c3_non_null")
+ .eq(lit(5_i64))
+ .and(col("c4_non_null").gt(lit(0_u32)));
+ assert_eq!(simplify_no_canonicalize(expr), expected);
+ }
+
+ #[test]
+ fn test_simplify_swapped_operands_in_xor_no_canonicalize() {
+ // `expr_contains` guards the XOR rules, so `delete_xor_in_complex_expr` has to
+ // use the same equality relation. Comparing structurally there while the guard
+ // normalizes makes the rule rebuild its input and still report `Transformed::yes`,
+ // which spins the simplifier until it hits the cycle limit.
+ //
+ // Operands are non-nullable so the expected results do not depend on how the XOR
+ // rules treat a NULL operand, which is a separate question from the cycle.
+
+ // (c4_non_null + 1) ^ (1 + c4_non_null) --> 0
+ let expr = bitwise_xor(
+ col("c4_non_null") + lit(1_u32),
+ lit(1_u32) + col("c4_non_null"),
+ );
+ let (simplified, cycles) = simplify_no_canonicalize_with_cycle_count(expr);
+ assert_eq!(simplified, lit(0_u32));
+ assert_eq!(cycles, 2);
+
+ // (c4_non_null + 1) ^ ((1 + c4_non_null) ^ c4_non_null) --> c4_non_null
+ let expr = bitwise_xor(
+ col("c4_non_null") + lit(1_u32),
+ bitwise_xor(lit(1_u32) + col("c4_non_null"), col("c4_non_null")),
+ );
+ let (simplified, cycles) = simplify_no_canonicalize_with_cycle_count(expr);
+ assert_eq!(simplified, col("c4_non_null"));
+ assert_eq!(cycles, 2);
+ }
+
#[test]
fn test_simplify_eq_not_self() {
// `expr_a`: column `c2` is nullable, so `c2 = c2` simplifies to `c2 IS NOT NULL OR NULL`
@@ -3667,6 +3738,20 @@ mod tests {
try_simplify(expr).unwrap()
}
+ fn simplify_no_canonicalize(expr: Expr) -> Expr {
+ simplify_no_canonicalize_with_cycle_count(expr).0
+ }
+
+ fn simplify_no_canonicalize_with_cycle_count(expr: Expr) -> (Expr, u32) {
+ let schema = expr_test_schema();
+ let (transformed, count) =
+ ExprSimplifier::new(SimplifyContext::builder().with_schema(schema).build())
+ .with_canonicalize(false)
+ .simplify_with_cycle_count_transformed(expr)
+ .unwrap();
+ (transformed.data, count)
+ }
+
fn try_simplify_with_cycle_count(expr: Expr) -> Result<(Expr, u32)> {
let schema = expr_test_schema();
let simplifier =
diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs
index 89bb762d59..78d801630c 100644
--- a/datafusion/optimizer/src/simplify_expressions/utils.rs
+++ b/datafusion/optimizer/src/simplify_expressions/utils.rs
@@ -17,6 +17,7 @@
//! Utility functions for expression simplification
+use datafusion_common::cse::NormalizeEq;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
Case, Expr, Like, Operator,
@@ -26,13 +27,18 @@ use datafusion_expr::{
/// returns true if `needle` is found in a chain of search_op
/// expressions. Such as: (A AND B) AND C
+///
+/// Equality uses [`NormalizeEq`] so commutative operands (`A = B` vs `B = A`)
+/// are recognized as the same predicate even when the canonicalizer is
+/// disabled, notably for `LogicalPlan::Join` filters (see
+/// <https://github.com/apache/datafusion/pull/8780>).
fn expr_contains_inner(expr: &Expr, needle: &Expr, search_op: Operator) -> bool {
match expr {
Expr::BinaryExpr(BinaryExpr { left, op, right }) if *op == search_op => {
expr_contains_inner(left, needle, search_op)
|| expr_contains_inner(right, needle, search_op)
}
- _ => expr == needle,
+ _ => expr.normalize_eq(needle),
}
}
@@ -43,6 +49,12 @@ pub fn expr_contains(expr: &Expr, needle: &Expr, search_op: Operator) -> bool {
/// Deletes all 'needles' or remains one 'needle' that are found in a chain of xor
/// expressions. Such as: A ^ (A ^ (B ^ A))
+///
+/// Matching uses [`NormalizeEq`] to stay consistent with the [`expr_contains`]
+/// guard on the XOR rules. A structural comparison here would let the guard fire
+/// on operands this function then fails to delete, so the rule would rebuild its
+/// input and still report a transformation, spinning the simplifier until it hits
+/// the cycle limit.
pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) -> Expr {
/// Deletes recursively 'needles' in a chain of xor expressions
fn recursive_delete_xor_in_expr(
@@ -56,10 +68,10 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) ->
{
let left_expr = recursive_delete_xor_in_expr(left, needle, xor_counter);
let right_expr = recursive_delete_xor_in_expr(right, needle, xor_counter);
- if left_expr == *needle {
+ if left_expr.normalize_eq(needle) {
*xor_counter += 1;
return right_expr;
- } else if right_expr == *needle {
+ } else if right_expr.normalize_eq(needle) {
*xor_counter += 1;
return left_expr;
}
@@ -76,7 +88,7 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) ->
let mut xor_counter: i32 = 0;
let result_expr = recursive_delete_xor_in_expr(expr, needle, &mut xor_counter);
- if result_expr == *needle {
+ if result_expr.normalize_eq(needle) {
return needle.clone();
} else if xor_counter % 2 == 0 {
if is_left {
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt b/datafusion/sqllogictest/test_files/simplify_expr.slt
index 158096328e..b24a1a6cff 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -165,6 +165,60 @@ physical_plan
01)ProjectionExec: expr=[column1@0 = 1 as opt1, column1@0 = 2 AND column1@0 != 2 as noopt1, column1@0 = 4 as opt2, column1@0 != 5 AND column1@0 = 5 as noopt2]
02)--DataSourceExec: partitions=1, partition_sizes=[1]
+# Dedup commutative AND/OR operands in join conditions
+# (canonicalization is disabled for joins, so `A = B` vs `B = A` must be
+# recognized directly). See https://github.com/apache/datafusion/issues/14943
+
+statement ok
+create table t1(a int) as values (1), (2);
+
+statement ok
+create table t2(b int) as values (1), (3);
+
+# `t1.a = t2.b OR t2.b = t1.a` simplifies to `t1.a = t2.b`, allowing a
+# HashJoin instead of a NestedLoopJoin
+query TT
+explain select * from t1 join t2 on t1.a = t2.b or t2.b = t1.a;
+----
+logical_plan
+01)Inner Join: t1.a = t2.b
+02)--TableScan: t1 projection=[a]
+03)--TableScan: t2 projection=[b]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, b@0)]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+select * from t1 join t2 on t1.a = t2.b or t2.b = t1.a;
+----
+1 1
+
+# swapped commutative operands (`t1.a + t2.b` vs `t2.b + t1.a`) dedup within
+# a non-equi join filter
+query TT
+explain select * from t1 join t2 on t1.a > t2.b and (t1.a + t2.b > 1 or t2.b + t1.a > 1);
+----
+logical_plan
+01)Inner Join: Filter: t1.a > t2.b AND t1.a + t2.b > Int32(1)
+02)--TableScan: t1 projection=[a]
+03)--TableScan: t2 projection=[b]
+physical_plan
+01)NestedLoopJoinExec: join_type=Inner, filter=a@0 > b@1 AND a@0 + b@1 > 1
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+select * from t1 join t2 on t1.a > t2.b and (t1.a + t2.b > 1 or t2.b + t1.a > 1);
+----
+2 1
+
+statement ok
+drop table t1;
+
+statement ok
+drop table t2;
+
# Identity Date cast in a comparison predicate.
# `cast(d AS date)` where `d` is already Date32 is an identity cast and should
# fold away, so the predicate compares against the bare column `d`. This enables