Apache DataFusion SQL Query Engine
fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)
Adam Gutglick6d764823764b
diff --git a/datafusion/optimizer/src/filter_null_join_keys.rs b/datafusion/optimizer/src/filter_null_join_keys.rsindex c8f419d3e5..e3de8048a8 100644--- a/datafusion/optimizer/src/filter_null_join_keys.rs+++ b/datafusion/optimizer/src/filter_null_join_keys.rs@@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys {match plan {LogicalPlan::Join(mut join)if !join.on.is_empty()+ && !join.null_aware&& join.null_equality == NullEquality::NullEqualsNothing =>{let (left_preserved, right_preserved) =@@ -359,4 +360,50 @@ mod tests {let t2 = table_scan(Some("t2"), &schema, None)?.build()?;Ok((t1, t2))}++ #[test]+ fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> {+ let (t1, t2) = test_tables()?;+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?;++ assert_optimized_plan_equal!(plan, @r"+ LeftMark Join: t1.id = t2.optional_id null_aware+ TableScan: t1+ TableScan: t2+ ")+ }++ #[test]+ fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> {+ let (t1, t2) = test_tables()?;+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?;++ assert_optimized_plan_equal!(plan, @r"+ LeftAnti Join: t1.id = t2.optional_id null_aware+ TableScan: t1+ TableScan: t2+ ")+ }++ /// A join whose nullable right key would get an `IS NOT NULL` filter if it+ /// were not null-aware.+ fn build_null_aware_plan(+ left_table: LogicalPlan,+ right_table: LogicalPlan,+ join_type: JoinType,+ ) -> Result<LogicalPlan> {+ LogicalPlanBuilder::from(left_table)+ .join_detailed_with_options(+ right_table,+ join_type,+ (+ vec![Column::from_qualified_name("t1.id")],+ vec![Column::from_qualified_name("t2.optional_id")],+ ),+ None,+ NullEquality::NullEqualsNothing,+ true,+ )?+ .build()+ }}diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex 1d12fc33c9..6061458b13 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -70,6 +70,20 @@ query IT rowsortSELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);----+# Regression test++statement ok+set datafusion.optimizer.filter_null_join_keys = true;++# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or+# FALSE, so the result stays empty.+query IT rowsort+SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);+----++statement ok+reset datafusion.optimizer.filter_null_join_keys;+# Verify the result is empty even though there are rows in outer_table# that don't match the non-NULL value (2) in the subquery.# This is correct null-aware behavior: if subquery contains NULL, result is unknown.