Skip to content

Apache DataFusion SQL Query Engine

fix: do not extract '=' predicate into 'IS NOT DISTINCT FROM' join (#27215) (#27222)

Sanghoon095cdc6a3e08
diff --git a/datafusion/optimizer/src/extract_equijoin_predicate.rs b/datafusion/optimizer/src/extract_equijoin_predicate.rs
index ff12675b36..cfa718417e 100644
--- a/datafusion/optimizer/src/extract_equijoin_predicate.rs
+++ b/datafusion/optimizer/src/extract_equijoin_predicate.rs
@@ -64,8 +64,17 @@ impl OptimizerRule for ExtractEquijoinPredicate {
}) => {
let left_schema = left.schema();
let right_schema = right.schema();
+ let op = match null_equality {
+ NullEquality::NullEqualsNull => Operator::IsNotDistinctFrom,
+ NullEquality::NullEqualsNothing => Operator::Eq,
+ };
let (equijoin_predicates, non_equijoin_expr) =
- split_eq_and_noneq_join_predicate(expr, left_schema, right_schema)?;
+ split_op_and_other_join_predicates(
+ expr,
+ left_schema,
+ right_schema,
+ op,
+ )?;
// Equi-join operators like HashJoin support a special behavior
// that evaluates `NULL = NULL` as true instead of NULL. Therefore,
@@ -75,7 +84,10 @@ impl OptimizerRule for ExtractEquijoinPredicate {
// Nested Loop Join, resulting in better performance.
//
// Only convert when there are NO equijoin predicates, to be conservative.
- if on.is_empty()
+ // A `NullEqualsNull` join already moved its `IS NOT DISTINCT FROM`
+ // predicates into `on` in the split above, so nothing is left to convert.
+ if null_equality == NullEquality::NullEqualsNothing
+ && on.is_empty()
&& equijoin_predicates.is_empty()
&& non_equijoin_expr.is_some()
{
@@ -260,11 +272,13 @@ fn split_op_and_other_join_predicates(
#[cfg(test)]
mod tests {
use super::*;
+ use crate::OptimizerContext;
use crate::assert_optimized_plan_eq_display_indent_snapshot;
use crate::test::*;
use arrow::datatypes::DataType;
use datafusion_expr::{
- JoinType, col, lit, logical_plan::builder::LogicalPlanBuilder,
+ JoinType, Operator, binary_expr, col, lit,
+ logical_plan::builder::LogicalPlanBuilder,
};
use std::sync::Arc;
@@ -517,4 +531,92 @@ mod tests {
"
)
}
+
+ #[test]
+ fn is_not_distinct_join_ignores_eq_filter() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ // Initial pass: extract 't1.a IS NOT DISTINCT FROM t2.a'
+ let initial_filter =
+ binary_expr(col("t1.a"), Operator::IsNotDistinctFrom, col("t2.a"));
+ let plan = LogicalPlanBuilder::from(t1)
+ .join_on(t2, JoinType::Inner, Some(initial_filter))?
+ .build()?;
+
+ let rule = ExtractEquijoinPredicate {};
+ let config = OptimizerContext::new();
+ let pass1 = rule.rewrite(plan, &config)?.data;
+ // on: [(t1.a, t2.a)], IsNotDistinctFrom
+ // filter: None
+ let LogicalPlan::Join(mut join) = pass1 else {
+ panic!("Expected Join");
+ };
+ assert_eq!(join.null_equality, NullEquality::NullEqualsNull);
+ assert_eq!(join.on.len(), 1);
+ assert!(join.filter.is_none());
+
+ // Second pass: 't1.b = t2.b' must NOT be extracted into on
+ join.filter = Some(col("t1.b").eq(col("t2.b")));
+ let plan_with_pushed_filter = LogicalPlan::Join(join);
+
+ let pass2 = rule.rewrite(plan_with_pushed_filter, &config)?;
+ // on: [(t1.a, t2.a)], IsNotDistinctFrom
+ // filter: (t1.b = t2.b)
+ assert!(!pass2.transformed);
+ let LogicalPlan::Join(join2) = pass2.data else {
+ panic!("Expected Join");
+ };
+ assert_eq!(join2.null_equality, NullEquality::NullEqualsNull);
+ assert_eq!(join2.on.len(), 1);
+ assert_eq!(join2.filter, Some(col("t1.b").eq(col("t2.b"))));
+
+ Ok(())
+ }
+
+ #[test]
+ fn is_not_distinct_join_extracts_same_filter() -> Result<()> {
+ let t1 = test_table_scan_with_name("t1")?;
+ let t2 = test_table_scan_with_name("t2")?;
+
+ // Initial pass: extract 't1.a IS NOT DISTINCT FROM t2.a'
+ let initial_filter =
+ binary_expr(col("t1.a"), Operator::IsNotDistinctFrom, col("t2.a"));
+ let plan = LogicalPlanBuilder::from(t1)
+ .join_on(t2, JoinType::Inner, Some(initial_filter))?
+ .build()?;
+
+ let rule = ExtractEquijoinPredicate {};
+ let config = OptimizerContext::new();
+ let pass1 = rule.rewrite(plan, &config)?.data;
+ // on: [(t1.a, t2.a)], IsNotDistinctFrom
+ // filter: None
+ let LogicalPlan::Join(mut join) = pass1 else {
+ panic!("Expected Join");
+ };
+ assert_eq!(join.null_equality, NullEquality::NullEqualsNull);
+ assert_eq!(join.on.len(), 1);
+ assert!(join.filter.is_none());
+
+ // Second pass: both keys merged into on
+ join.filter = Some(binary_expr(
+ col("t1.b"),
+ Operator::IsNotDistinctFrom,
+ col("t2.b"),
+ ));
+ let plan_with_pushed_filter = LogicalPlan::Join(join);
+
+ let pass2 = rule.rewrite(plan_with_pushed_filter, &config)?;
+ // on: [(t1.a, t2.a), (t1.b, t2.b)], IsNotDistinctFrom
+ // filter: None
+ assert!(pass2.transformed);
+ let LogicalPlan::Join(join2) = pass2.data else {
+ panic!("Expected Join");
+ };
+ assert_eq!(join2.null_equality, NullEquality::NullEqualsNull);
+ assert_eq!(join2.on.len(), 2);
+ assert!(join2.filter.is_none());
+
+ Ok(())
+ }
}
diff --git a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
index 39324a4c16..c4660c0e41 100644
--- a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
+++ b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt
@@ -322,3 +322,48 @@ drop table t3;
statement ok
drop table t4;
+
+# An ordinary '=' predicate pushed down into an 'IS NOT DISTINCT FROM' join
+# must NOT evaluate NULL = NULL as true (issue #27215).
+statement ok
+CREATE TABLE t1(a INT, b INT) AS VALUES (NULL, 1), (5, 2), (6, NULL);
+
+statement ok
+CREATE TABLE t2(a INT, b INT) AS VALUES (NULL, 1), (5, 2), (6, NULL);
+
+query TT
+EXPLAIN SELECT *
+FROM (SELECT t1.a a1, t1.b b1, t2.b b2
+ FROM t1 JOIN t2 ON t1.a IS NOT DISTINCT FROM t2.a) s
+WHERE s.b1 = s.b2
+ORDER BY a1;
+----
+logical_plan
+01)Sort: s.a1 ASC NULLS LAST
+02)--SubqueryAlias: s
+03)----Projection: t1.a AS a1, t1.b AS b1, t2.b AS b2
+04)------Inner Join: t1.a = t2.a Filter: t2.b = t1.b
+05)--------TableScan: t1 projection=[a, b]
+06)--------TableScan: t2 projection=[a, b]
+physical_plan
+01)SortExec: expr=[a1@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--ProjectionExec: expr=[a@0 as a1, b@1 as b1, b@2 as b2]
+03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, a@0)], filter=b@1 = b@0, projection=[a@0, b@1, b@3], NullsEqual: true
+04)------DataSourceExec: partitions=1, partition_sizes=[1]
+05)------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query III rowsort
+SELECT *
+FROM (SELECT t1.a a1, t1.b b1, t2.b b2
+ FROM t1 JOIN t2 ON t1.a IS NOT DISTINCT FROM t2.a) s
+WHERE s.b1 = s.b2
+ORDER BY a1;
+----
+5 2 2
+NULL 1 1
+
+statement ok
+DROP TABLE t1;
+
+statement ok
+DROP TABLE t2;