Apache DataFusion SQL Query Engine
fix: skip dynamic filter pushdown for null-aware anti joins with a nullable build key (#23173)
Moe5975a2d57cdf
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex ccdb050d16..9d9c867c27 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -871,6 +871,18 @@ impl HashJoinExec {return false;}+ // A null-aware anti join emits a build-side NULL only when the probe+ // is truly empty. The pushed filter can empty the probe by pruning+ // every row, which would surface that NULL wrongly. A NOT NULL build+ // key cannot produce such a NULL, so the filter stays there.+ if self.null_aware+ && self.on.iter().any(|(build_key, _)| {+ build_key.nullable(&self.left.schema()).unwrap_or(true)+ })+ {+ return false;+ }+// `preserve_file_partitions` can report Hash partitioning for Hive-style// file groups, but those partitions are not actually hash-distributed.// Partitioned dynamic filters rely on hash routing, so disable them in@@ -6949,6 +6961,79 @@ mod tests {Ok(())}+ #[test]+ fn test_dynamic_filter_pushdown_rejects_null_aware_nullable_build_key() -> Result<()>+ {+ let left = build_table_two_cols(+ ("a1", &vec![Some(1), None]),+ ("b1", &vec![Some(1), Some(2)]),+ );+ let right = build_table_two_cols(+ ("a2", &vec![Some(2), Some(3)]),+ ("b2", &vec![Some(1), Some(2)]),+ );+ let on = vec![(+ Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,+ )];++ let mut session_config = SessionConfig::default();+ session_config+ .options_mut()+ .optimizer+ .enable_join_dynamic_filter_pushdown = true;++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options()));++ Ok(())+ }++ #[test]+ fn test_dynamic_filter_pushdown_allows_null_aware_non_null_build_key() -> Result<()> {+ // A NOT NULL build key cannot surface a build-side NULL, so the+ // pushdown must stay enabled.+ let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));+ let right = build_table(("a2", &vec![2]), ("b2", &vec![2]), ("c2", &vec![2]));+ let on = vec![(+ Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,+ )];++ let mut session_config = SessionConfig::default();+ session_config+ .options_mut()+ .optimizer+ .enable_join_dynamic_filter_pushdown = true;++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ assert!(join.allow_join_dynamic_filter_pushdown(session_config.options()));++ Ok(())+ }+#[test]fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()>{diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex c58047c4ab..c51a127986 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -383,7 +383,7 @@ logical_planphysical_plan01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet# LEFT MARK JOIN: the OR prevents decorrelation to LeftSemi, so the optimizer# uses LeftMark. Self-generated dynamic filter pushes to the probe side.@@ -479,7 +479,7 @@ physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Correctness checkquery ITdiff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.sltindex 8588c0e7ba..4e0397bb41 100644--- a/datafusion/sqllogictest/test_files/explain_tree.slt+++ b/datafusion/sqllogictest/test_files/explain_tree.slt@@ -1120,10 +1120,7 @@ physical_plan13)│ -------------------- ││ -------------------- │14)│ files: 1 ││ files: 1 │15)│ format: csv ││ format: parquet │-16)│ ││ │-17)│ ││ predicate: │-18)│ ││ DynamicFilter [ empty ] │-19)└───────────────────────────┘└───────────────────────────┘+16)└───────────────────────────┘└───────────────────────────┘# Query with nested loop join.query TTdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex e879947e32..f1e787441d 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -1066,6 +1066,72 @@ statement okdrop table nej_probe;+########+# Regression test for build-NULL + emptied-probe interaction in null-aware LeftAnti joins.+#+# `x NOT IN (subquery)` plans as a null-aware LeftAnti hash join where `x` is+# the build (left) side. The dynamic-filter pushdown derives a bounds/membership+# filter from the build keys and pushes it onto the probe scan. When the build+# contains a NULL key and the filter prunes every probe row, the probe looks+# empty to the join. A null-aware LeftAnti treats an empty probe as a genuinely-+# absent subquery, so it emits the build-side NULL as a matching row. That is+# wrong: `NULL NOT IN (non-empty set)` must be UNKNOWN, not TRUE.+#+# The fix: suppress dynamic-filter pushdown whenever the build key is nullable+# and the join is null-aware, so the probe is never artificially emptied.+########++statement ok+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = true;++statement ok+set datafusion.execution.parquet.pushdown_filters = true;++# Build side: `ao` has a nullable `id` column; the NULL row is the one that+# must NOT appear in the output.+query I+COPY (SELECT * FROM (VALUES (5), (NULL)) v(id))+TO 'test_files/scratch/push_down_filter_parquet/ao_p.parquet'+STORED AS PARQUET;+----+2++# Probe / subquery side: `i_disj` has two non-NULL values that don't match 5,+# and no NULLs. The subquery is non-empty, so `NULL NOT IN (...)` is UNKNOWN.+query I+COPY (SELECT * FROM (VALUES (2), (3)) v(eid))+TO 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet'+STORED AS PARQUET;+----+2++statement ok+CREATE EXTERNAL TABLE ao_p (id INT) STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/ao_p.parquet';++statement ok+CREATE EXTERNAL TABLE i_disj_p (eid INT) STORED AS PARQUET+LOCATION 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet';++# Must return only `5`. `NULL NOT IN (2, 3)` is UNKNOWN, so that row is dropped.+query I+SELECT id FROM ao_p WHERE id NOT IN (SELECT eid FROM i_disj_p) ORDER BY id;+----+5++statement ok+drop table ao_p;++statement ok+drop table i_disj_p;++statement ok+RESET datafusion.optimizer.enable_join_dynamic_filter_pushdown;++statement ok+RESET datafusion.execution.parquet.pushdown_filters;++# Config resetstatement okRESET datafusion.explain.physical_plan_only;