Apache DataFusion SQL Query Engine
fix: don't infer join predicates for null-aware joins in push_down_filter (#23901)
Liang-Chi Hsieh0fcf6284fd81
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex f30b1187b7..cf54ae2547 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -576,6 +576,17 @@ fn infer_join_predicates(predicates: &[Expr],on_filters: &[Expr],) -> Result<Vec<Expr>> {+ // Null-aware joins (e.g. `NOT IN` with a nullable subquery) rely on SQL+ // three-valued logic: a NULL join key on the right/subquery side makes the+ // predicate UNKNOWN and empties the result, so those NULLs must reach the+ // join. Inferring an equi-key predicate here would rewrite a left-side+ // predicate onto the right side and, because the inferred predicate must be+ // null-rejecting, drop the subquery's NULL rows and produce wrong results.+ // Skip inference entirely for null-aware joins.+ if join.null_aware {+ return Ok(vec![]);+ }+// Only allow both side key is column.let join_col_keys = join.on@@ -3826,6 +3837,51 @@ mod tests {)}+ /// Regression test: for a null-aware LeftAnti join (the shape produced by+ /// `NOT IN` with a nullable subquery), a right-side predicate must NOT be+ /// inferred onto the join. Inference would push a null-rejecting predicate+ /// to the subquery side, dropping its NULL rows and breaking the+ /// three-valued `NOT IN` semantics.+ #[test]+ fn null_aware_left_anti_join_no_inferred_pushdown() -> Result<()> {+ let table_scan = test_table_scan_with_name("test1")?;+ let left = LogicalPlanBuilder::from(table_scan)+ .project(vec![col("a"), col("b")])?+ .build()?;+ let right_table_scan = test_table_scan_with_name("test2")?;+ let right = LogicalPlanBuilder::from(right_table_scan)+ .project(vec![col("a"), col("b")])?+ .build()?;+ let plan = LogicalPlanBuilder::from(left)+ .join_detailed_with_options(+ right,+ JoinType::LeftAnti,+ (+ vec![Column::from_qualified_name("test1.a")],+ vec![Column::from_qualified_name("test2.a")],+ ),+ None,+ datafusion_common::NullEquality::NullEqualsNothing,+ true,+ )?+ .filter(col("test1.a").gt(lit(2u32)))?+ .build()?;++ // The left-side filter is pushed to the left input, but — unlike the+ // non-null-aware `left_anti_join` test — no `test2.a > 2` predicate is+ // inferred onto the right/subquery side.+ assert_optimized_plan_equal!(+ plan,+ @r"+ LeftAnti Join: test1.a = test2.a null_aware+ Projection: test1.a, test1.b+ TableScan: test1, full_filters=[test1.a > UInt32(2)]+ Projection: test2.a, test2.b+ TableScan: test2+ "+ )+ }+#[test]fn left_anti_join_with_filters() -> Result<()> {let table_scan = test_table_scan_with_name("test1")?;diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex 6061458b13..bdb56cf220 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -530,3 +530,48 @@ RESET datafusion.execution.parquet.pushdown_filters;statement okRESET datafusion.optimizer.enable_join_dynamic_filter_pushdown;++#############+## Regression: null-aware NOT IN with an outer predicate on the join key+##+## `push_down_filter` used to infer the outer predicate `id > 5` onto the+## subquery side (as `eid > 5`), dropping the subquery's NULL row and wrongly+## returning outer rows. The subquery NULL must reach the join so that+## `NOT IN` stays UNKNOWN for every row.+#############++statement ok+CREATE TABLE nai_outer(id INT) AS VALUES (3), (7);++statement ok+CREATE TABLE nai_inner(id INT) AS VALUES (NULL);++# Expected: zero rows (subquery contains NULL => NOT IN is UNKNOWN for all).+query I+SELECT id FROM nai_outer WHERE id > 5 AND id NOT IN (SELECT id FROM nai_inner) ORDER BY id;+----++# Same query under SortMergeJoin + multiple partitions: null-aware joins must+# be planned as a CollectLeft HashJoin, not a plain anti SortMergeJoin.+statement ok+SET datafusion.optimizer.prefer_hash_join = false;++statement ok+SET datafusion.execution.target_partitions = 4;++query I+SELECT id FROM nai_outer WHERE id NOT IN (SELECT id FROM nai_inner) ORDER BY id;+----++statement ok+SET datafusion.optimizer.prefer_hash_join = true;++# The SLT runner sets target_partitions to 4, so restore that value explicitly.+statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+DROP TABLE nai_outer;++statement ok+DROP TABLE nai_inner;