Skip to content

Apache DataFusion SQL Query Engine

fix: don't infer join predicates for null-aware joins in push_down_filter (#23901)

Liang-Chi Hsieh0fcf6284fd81
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index f30b1187b7..cf54ae2547 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -576,6 +576,17 @@ fn infer_join_predicates(
predicates: &[Expr],
on_filters: &[Expr],
) -> Result<Vec<Expr>> {
+ // Null-aware joins (e.g. `NOT IN` with a nullable subquery) rely on SQL
+ // three-valued logic: a NULL join key on the right/subquery side makes the
+ // predicate UNKNOWN and empties the result, so those NULLs must reach the
+ // join. Inferring an equi-key predicate here would rewrite a left-side
+ // predicate onto the right side and, because the inferred predicate must be
+ // null-rejecting, drop the subquery's NULL rows and produce wrong results.
+ // Skip inference entirely for null-aware joins.
+ if join.null_aware {
+ return Ok(vec![]);
+ }
+
// Only allow both side key is column.
let join_col_keys = join
.on
@@ -3826,6 +3837,51 @@ mod tests {
)
}
+ /// Regression test: for a null-aware LeftAnti join (the shape produced by
+ /// `NOT IN` with a nullable subquery), a right-side predicate must NOT be
+ /// inferred onto the join. Inference would push a null-rejecting predicate
+ /// to the subquery side, dropping its NULL rows and breaking the
+ /// three-valued `NOT IN` semantics.
+ #[test]
+ fn null_aware_left_anti_join_no_inferred_pushdown() -> Result<()> {
+ let table_scan = test_table_scan_with_name("test1")?;
+ let left = LogicalPlanBuilder::from(table_scan)
+ .project(vec![col("a"), col("b")])?
+ .build()?;
+ let right_table_scan = test_table_scan_with_name("test2")?;
+ let right = LogicalPlanBuilder::from(right_table_scan)
+ .project(vec![col("a"), col("b")])?
+ .build()?;
+ let plan = LogicalPlanBuilder::from(left)
+ .join_detailed_with_options(
+ right,
+ JoinType::LeftAnti,
+ (
+ vec![Column::from_qualified_name("test1.a")],
+ vec![Column::from_qualified_name("test2.a")],
+ ),
+ None,
+ datafusion_common::NullEquality::NullEqualsNothing,
+ true,
+ )?
+ .filter(col("test1.a").gt(lit(2u32)))?
+ .build()?;
+
+ // The left-side filter is pushed to the left input, but — unlike the
+ // non-null-aware `left_anti_join` test — no `test2.a > 2` predicate is
+ // inferred onto the right/subquery side.
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ LeftAnti Join: test1.a = test2.a null_aware
+ Projection: test1.a, test1.b
+ TableScan: test1, full_filters=[test1.a > UInt32(2)]
+ Projection: test2.a, test2.b
+ TableScan: test2
+ "
+ )
+ }
+
#[test]
fn left_anti_join_with_filters() -> Result<()> {
let table_scan = test_table_scan_with_name("test1")?;
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index 6061458b13..bdb56cf220 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -530,3 +530,48 @@ RESET datafusion.execution.parquet.pushdown_filters;
statement ok
RESET datafusion.optimizer.enable_join_dynamic_filter_pushdown;
+
+#############
+## Regression: null-aware NOT IN with an outer predicate on the join key
+##
+## `push_down_filter` used to infer the outer predicate `id > 5` onto the
+## subquery side (as `eid > 5`), dropping the subquery's NULL row and wrongly
+## returning outer rows. The subquery NULL must reach the join so that
+## `NOT IN` stays UNKNOWN for every row.
+#############
+
+statement ok
+CREATE TABLE nai_outer(id INT) AS VALUES (3), (7);
+
+statement ok
+CREATE TABLE nai_inner(id INT) AS VALUES (NULL);
+
+# Expected: zero rows (subquery contains NULL => NOT IN is UNKNOWN for all).
+query I
+SELECT id FROM nai_outer WHERE id > 5 AND id NOT IN (SELECT id FROM nai_inner) ORDER BY id;
+----
+
+# Same query under SortMergeJoin + multiple partitions: null-aware joins must
+# be planned as a CollectLeft HashJoin, not a plain anti SortMergeJoin.
+statement ok
+SET datafusion.optimizer.prefer_hash_join = false;
+
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+query I
+SELECT id FROM nai_outer WHERE id NOT IN (SELECT id FROM nai_inner) ORDER BY id;
+----
+
+statement ok
+SET datafusion.optimizer.prefer_hash_join = true;
+
+# The SLT runner sets target_partitions to 4, so restore that value explicitly.
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+statement ok
+DROP TABLE nai_outer;
+
+statement ok
+DROP TABLE nai_inner;