Skip to content

Apache DataFusion SQL Query Engine

fix: Handle null-aware joins correctly in `FilterNullJoinKeys` when its enabled (#23848)

Adam Gutglick6d764823764b
diff --git a/datafusion/optimizer/src/filter_null_join_keys.rs b/datafusion/optimizer/src/filter_null_join_keys.rs
index c8f419d3e5..e3de8048a8 100644
--- a/datafusion/optimizer/src/filter_null_join_keys.rs
+++ b/datafusion/optimizer/src/filter_null_join_keys.rs
@@ -52,6 +52,7 @@ impl OptimizerRule for FilterNullJoinKeys {
match plan {
LogicalPlan::Join(mut join)
if !join.on.is_empty()
+ && !join.null_aware
&& join.null_equality == NullEquality::NullEqualsNothing =>
{
let (left_preserved, right_preserved) =
@@ -359,4 +360,50 @@ mod tests {
let t2 = table_scan(Some("t2"), &schema, None)?.build()?;
Ok((t1, t2))
}
+
+ #[test]
+ fn null_aware_left_mark_join_keys_not_filtered() -> Result<()> {
+ let (t1, t2) = test_tables()?;
+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftMark)?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ LeftMark Join: t1.id = t2.optional_id null_aware
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ #[test]
+ fn null_aware_left_anti_join_keys_not_filtered() -> Result<()> {
+ let (t1, t2) = test_tables()?;
+ let plan = build_null_aware_plan(t1, t2, JoinType::LeftAnti)?;
+
+ assert_optimized_plan_equal!(plan, @r"
+ LeftAnti Join: t1.id = t2.optional_id null_aware
+ TableScan: t1
+ TableScan: t2
+ ")
+ }
+
+ /// A join whose nullable right key would get an `IS NOT NULL` filter if it
+ /// were not null-aware.
+ fn build_null_aware_plan(
+ left_table: LogicalPlan,
+ right_table: LogicalPlan,
+ join_type: JoinType,
+ ) -> Result<LogicalPlan> {
+ LogicalPlanBuilder::from(left_table)
+ .join_detailed_with_options(
+ right_table,
+ join_type,
+ (
+ vec![Column::from_qualified_name("t1.id")],
+ vec![Column::from_qualified_name("t2.optional_id")],
+ ),
+ None,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?
+ .build()
+ }
}
diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
index 1d12fc33c9..6061458b13 100644
--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt
@@ -70,6 +70,20 @@ query IT rowsort
SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
----
+# Regression test
+
+statement ok
+set datafusion.optimizer.filter_null_join_keys = true;
+
+# The subquery NULL must reach the join: every row's NOT IN is UNKNOWN or
+# FALSE, so the result stays empty.
+query IT rowsort
+SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);
+----
+
+statement ok
+reset datafusion.optimizer.filter_null_join_keys;
+
# Verify the result is empty even though there are rows in outer_table
# that don't match the non-NULL value (2) in the subquery.
# This is correct null-aware behavior: if subquery contains NULL, result is unknown.