Skip to content

Apache DataFusion SQL Query Engine

fix: skip dynamic filter pushdown for null-aware anti joins with a nullable build key (#23173)

Moe5975a2d57cdf
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index ccdb050d16..9d9c867c27 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -871,6 +871,18 @@ impl HashJoinExec {
return false;
}
+ // A null-aware anti join emits a build-side NULL only when the probe
+ // is truly empty. The pushed filter can empty the probe by pruning
+ // every row, which would surface that NULL wrongly. A NOT NULL build
+ // key cannot produce such a NULL, so the filter stays there.
+ if self.null_aware
+ && self.on.iter().any(|(build_key, _)| {
+ build_key.nullable(&self.left.schema()).unwrap_or(true)
+ })
+ {
+ return false;
+ }
+
// `preserve_file_partitions` can report Hash partitioning for Hive-style
// file groups, but those partitions are not actually hash-distributed.
// Partitioned dynamic filters rely on hash routing, so disable them in
@@ -6949,6 +6961,79 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_dynamic_filter_pushdown_rejects_null_aware_nullable_build_key() -> Result<()>
+ {
+ let left = build_table_two_cols(
+ ("a1", &vec![Some(1), None]),
+ ("b1", &vec![Some(1), Some(2)]),
+ );
+ let right = build_table_two_cols(
+ ("a2", &vec![Some(2), Some(3)]),
+ ("b2", &vec![Some(1), Some(2)]),
+ );
+ let on = vec![(
+ Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
+ )];
+
+ let mut session_config = SessionConfig::default();
+ session_config
+ .options_mut()
+ .optimizer
+ .enable_join_dynamic_filter_pushdown = true;
+
+ let join = HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::LeftAnti,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?;
+
+ assert!(!join.allow_join_dynamic_filter_pushdown(session_config.options()));
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dynamic_filter_pushdown_allows_null_aware_non_null_build_key() -> Result<()> {
+ // A NOT NULL build key cannot surface a build-side NULL, so the
+ // pushdown must stay enabled.
+ let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));
+ let right = build_table(("a2", &vec![2]), ("b2", &vec![2]), ("c2", &vec![2]));
+ let on = vec![(
+ Arc::new(Column::new_with_schema("a1", &left.schema())?) as _,
+ Arc::new(Column::new_with_schema("a2", &right.schema())?) as _,
+ )];
+
+ let mut session_config = SessionConfig::default();
+ session_config
+ .options_mut()
+ .optimizer
+ .enable_join_dynamic_filter_pushdown = true;
+
+ let join = HashJoinExec::try_new(
+ left,
+ right,
+ on,
+ None,
+ &JoinType::LeftAnti,
+ None,
+ PartitionMode::CollectLeft,
+ NullEquality::NullEqualsNothing,
+ true,
+ )?;
+
+ assert!(join.allow_join_dynamic_filter_pushdown(session_config.options()));
+
+ Ok(())
+ }
+
#[test]
fn test_partitioned_dynamic_filter_pushdown_rejects_range_partitioning() -> Result<()>
{
diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
index c58047c4ab..c51a127986 100644
--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
@@ -383,7 +383,7 @@ logical_plan
physical_plan
01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet
-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet
# LEFT MARK JOIN: the OR prevents decorrelation to LeftSemi, so the optimizer
# uses LeftMark. Self-generated dynamic filter pushes to the probe side.
@@ -479,7 +479,7 @@ physical_plan
01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible
+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
# Correctness check
query IT
diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt
index 8588c0e7ba..4e0397bb41 100644
--- a/datafusion/sqllogictest/test_files/explain_tree.slt
+++ b/datafusion/sqllogictest/test_files/explain_tree.slt
@@ -1120,10 +1120,7 @@ physical_plan
13)│ -------------------- ││ -------------------- │
14)│ files: 1 ││ files: 1 │
15)│ format: csv ││ format: parquet │
-16)│ ││ │
-17)│ ││ predicate: │
-18)│ ││ DynamicFilter [ empty ] │
-19)└───────────────────────────┘└───────────────────────────┘
+16)└───────────────────────────┘└───────────────────────────┘
# Query with nested loop join.
query TT
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index e879947e32..f1e787441d 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -1066,6 +1066,72 @@ statement ok
drop table nej_probe;
+########
+# Regression test for build-NULL + emptied-probe interaction in null-aware LeftAnti joins.
+#
+# `x NOT IN (subquery)` plans as a null-aware LeftAnti hash join where `x` is
+# the build (left) side. The dynamic-filter pushdown derives a bounds/membership
+# filter from the build keys and pushes it onto the probe scan. When the build
+# contains a NULL key and the filter prunes every probe row, the probe looks
+# empty to the join. A null-aware LeftAnti treats an empty probe as a genuinely-
+# absent subquery, so it emits the build-side NULL as a matching row. That is
+# wrong: `NULL NOT IN (non-empty set)` must be UNKNOWN, not TRUE.
+#
+# The fix: suppress dynamic-filter pushdown whenever the build key is nullable
+# and the join is null-aware, so the probe is never artificially emptied.
+########
+
+statement ok
+set datafusion.optimizer.enable_join_dynamic_filter_pushdown = true;
+
+statement ok
+set datafusion.execution.parquet.pushdown_filters = true;
+
+# Build side: `ao` has a nullable `id` column; the NULL row is the one that
+# must NOT appear in the output.
+query I
+COPY (SELECT * FROM (VALUES (5), (NULL)) v(id))
+TO 'test_files/scratch/push_down_filter_parquet/ao_p.parquet'
+STORED AS PARQUET;
+----
+2
+
+# Probe / subquery side: `i_disj` has two non-NULL values that don't match 5,
+# and no NULLs. The subquery is non-empty, so `NULL NOT IN (...)` is UNKNOWN.
+query I
+COPY (SELECT * FROM (VALUES (2), (3)) v(eid))
+TO 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet'
+STORED AS PARQUET;
+----
+2
+
+statement ok
+CREATE EXTERNAL TABLE ao_p (id INT) STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_parquet/ao_p.parquet';
+
+statement ok
+CREATE EXTERNAL TABLE i_disj_p (eid INT) STORED AS PARQUET
+LOCATION 'test_files/scratch/push_down_filter_parquet/i_disj_p.parquet';
+
+# Must return only `5`. `NULL NOT IN (2, 3)` is UNKNOWN, so that row is dropped.
+query I
+SELECT id FROM ao_p WHERE id NOT IN (SELECT eid FROM i_disj_p) ORDER BY id;
+----
+5
+
+statement ok
+drop table ao_p;
+
+statement ok
+drop table i_disj_p;
+
+statement ok
+RESET datafusion.optimizer.enable_join_dynamic_filter_pushdown;
+
+statement ok
+RESET datafusion.execution.parquet.pushdown_filters;
+
+
# Config reset
statement ok
RESET datafusion.explain.physical_plan_only;