Apache DataFusion SQL Query Engine
feat: null aware RightAnti hash join execution + planning support (#23957)
Saad Tajwar0a4e93052267
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 265279a8ca..63654ae048 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -444,6 +444,192 @@ async fn test_join_with_swap_semi() {}}+#[rstest]+#[case(PartitionMode::CollectLeft)]+#[case(PartitionMode::Auto)]+#[case(PartitionMode::Partitioned)]+#[tokio::test]+async fn test_null_aware_left_anti_swaps_to_right_anti(+ #[case] partition_mode: PartitionMode,+) -> Result<()> {+ let (big, small) = create_big_and_small();+ let join = HashJoinExec::try_new(+ Arc::clone(&big),+ Arc::clone(&small),+ vec![(+ Arc::new(Column::new_with_schema("big_col", &big.schema())?),+ Arc::new(Column::new_with_schema("small_col", &small.schema())?),+ )],+ None,+ &JoinType::LeftAnti,+ None,+ partition_mode,+ NullEquality::NullEqualsNothing,+ true,+ )?;+ let original_schema = join.schema();++ let optimized_join =+ JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;+ let swapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("anti join swap should not require a projection");++ assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);+ assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);+ assert!(swapped_join.null_aware);+ assert_eq!(swapped_join.left().schema().field(0).name(), "small_col");+ assert_eq!(swapped_join.right().schema().field(0).name(), "big_col");+ assert_eq!(swapped_join.schema(), original_schema);++ Ok(())+}++#[tokio::test]+async fn test_null_aware_auto_large_inputs_swaps_to_collect_left() -> Result<()> {+ let bigger: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(+ bigger_statistics(),+ Schema::new(vec![Field::new("bigger_col", DataType::Int32, false)]),+ ));+ let big: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(+ big_statistics(),+ Schema::new(vec![Field::new("big_col", DataType::Int32, false)]),+ ));+ let join = HashJoinExec::try_new(+ Arc::clone(&bigger),+ Arc::clone(&big),+ vec![(+ Arc::new(Column::new_with_schema("bigger_col", &bigger.schema())?),+ Arc::new(Column::new_with_schema("big_col", &big.schema())?),+ )],+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::Auto,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let optimized_join =+ JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;+ let swapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("anti join swap should not require a projection");++ assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);+ assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);+ assert!(swapped_join.null_aware);+ assert_eq!(swapped_join.left().schema().field(0).name(), "big_col");+ assert_eq!(swapped_join.right().schema().field(0).name(), "bigger_col");++ Ok(())+}++#[tokio::test]+async fn test_null_aware_left_anti_does_not_swap_when_left_is_smaller() -> Result<()> {+ let (big, small) = create_big_and_small();+ let join = HashJoinExec::try_new(+ Arc::clone(&small),+ Arc::clone(&big),+ vec![(+ Arc::new(Column::new_with_schema("small_col", &small.schema())?),+ Arc::new(Column::new_with_schema("big_col", &big.schema())?),+ )],+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let optimized_join =+ JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;+ let unswapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("join type should remain unchanged");++ assert_eq!(*unswapped_join.join_type(), JoinType::LeftAnti);+ assert_eq!(*unswapped_join.partition_mode(), PartitionMode::CollectLeft);+ assert!(unswapped_join.null_aware);+ assert_eq!(unswapped_join.left().schema().field(0).name(), "small_col");+ assert_eq!(unswapped_join.right().schema().field(0).name(), "big_col");++ Ok(())+}++#[tokio::test]+async fn test_null_aware_left_anti_respects_disabled_join_reordering() -> Result<()> {+ let (big, small) = create_big_and_small();+ let join = HashJoinExec::try_new(+ Arc::clone(&big),+ Arc::clone(&small),+ vec![(+ Arc::new(Column::new_with_schema("big_col", &big.schema())?),+ Arc::new(Column::new_with_schema("small_col", &small.schema())?),+ )],+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;+ let mut config = ConfigOptions::new();+ config.optimizer.join_reordering = false;++ let optimized_join = JoinSelection::new().optimize(Arc::new(join), &config)?;+ let unswapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("join type should remain unchanged");++ assert_eq!(*unswapped_join.join_type(), JoinType::LeftAnti);+ assert_eq!(*unswapped_join.partition_mode(), PartitionMode::CollectLeft);+ assert!(unswapped_join.null_aware);+ assert_eq!(unswapped_join.left().schema().field(0).name(), "big_col");+ assert_eq!(unswapped_join.right().schema().field(0).name(), "small_col");++ Ok(())+}++/// A filtered null-aware `LeftAnti` (produced by decorrelating a correlated+/// `NOT IN`) must not be swapped to `RightAnti`: the swap is only valid for+/// unfiltered null-aware joins, since a filtered `RightAnti` would apply its+/// build-side NULL short-circuit before the filter.+#[tokio::test]+async fn test_null_aware_left_anti_with_filter_does_not_swap() -> Result<()> {+ let (big, small) = create_big_and_small();+ let join = HashJoinExec::try_new(+ Arc::clone(&big),+ Arc::clone(&small),+ vec![(+ Arc::new(Column::new_with_schema("big_col", &big.schema())?),+ Arc::new(Column::new_with_schema("small_col", &small.schema())?),+ )],+ nl_join_filter(),+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let optimized_join =+ JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;+ let unswapped_join = optimized_join+ .downcast_ref::<HashJoinExec>()+ .expect("filtered null-aware anti join should not swap");++ assert_eq!(*unswapped_join.join_type(), JoinType::LeftAnti);+ assert_eq!(*unswapped_join.partition_mode(), PartitionMode::CollectLeft);+ assert!(unswapped_join.null_aware);+ assert!(unswapped_join.filter().is_some());+ assert_eq!(unswapped_join.left().schema().field(0).name(), "big_col");+ assert_eq!(unswapped_join.right().schema().field(0).name(), "small_col");++ Ok(())+}+#[tokio::test]async fn test_join_with_swap_mark() {let join_types = [JoinType::LeftMark, JoinType::RightMark];diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex 42736f8205..4ce6329f32 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -186,6 +186,14 @@ impl PhysicalOptimizerRule for JoinSelection {}}+/// Determines whether it is possible to swap inputs of a hash join - for null-aware joins, we can only swap `LeftAnti` with no filters+fn can_swap_hash_join(hash_join: &HashJoinExec) -> bool {+ hash_join.join_type().supports_swap()+ && (!hash_join.null_aware+ || (*hash_join.join_type() == JoinType::LeftAnti+ && hash_join.filter().is_none()))+}+/// Tries to create a [`HashJoinExec`] in [`PartitionMode::CollectLeft`] when possible.////// This function will first consider the given join type and check whether the@@ -224,9 +232,8 @@ pub(crate) fn try_collect_left(match (left_can_collect, right_can_collect) {(true, true) => {- // Don't swap null-aware anti joins as they have specific side requirements- if hash_join.join_type().supports_swap()- && !hash_join.null_aware+ // For null-aware joins, we only swap `LeftAnti` joins where the left side is > right side+ if can_swap_hash_join(hash_join)&& should_swap_join_order(&**left, &**right, config, registry)?{Ok(Some(hash_join.swap_inputs(PartitionMode::CollectLeft)?))@@ -246,11 +253,7 @@ pub(crate) fn try_collect_left(.build()?,))),(false, true) => {- // Don't swap null-aware anti joins as they have specific side requirements- if optimizer_config.join_reordering- && hash_join.join_type().supports_swap()- && !hash_join.null_aware- {+ if optimizer_config.join_reordering && can_swap_hash_join(hash_join) {hash_join.swap_inputs(PartitionMode::CollectLeft).map(Some)} else {Ok(None)@@ -275,22 +278,20 @@ pub(crate) fn partitioned_hash_join() -> Result<Arc<dyn ExecutionPlan>> {let left = hash_join.left();let right = hash_join.right();- // Don't swap null-aware anti joins as they have specific side requirements- if hash_join.join_type().supports_swap()- && !hash_join.null_aware+ let partition_mode = if hash_join.null_aware {+ PartitionMode::CollectLeft+ } else {+ PartitionMode::Partitioned+ };+ if can_swap_hash_join(hash_join)&& should_swap_join_order(&**left, &**right, config, registry)?{- hash_join.swap_inputs(PartitionMode::Partitioned)+ hash_join.swap_inputs(partition_mode)} else {// Null-aware anti joins must use CollectLeft mode because they track probe-side state// (probe_side_non_empty, probe_side_has_null) per-partition, but need global knowledge// for correct null handling. With partitioning, a partition might not see probe rows// even if the probe side is globally non-empty, leading to incorrect NULL row handling.- let partition_mode = if hash_join.null_aware {- PartitionMode::CollectLeft- } else {- PartitionMode::Partitioned- };Ok(Arc::new(hash_join@@ -330,14 +331,16 @@ fn statistical_join_selection_subrule(PartitionMode::Partitioned => {let left = hash_join.left();let right = hash_join.right();- // Don't swap null-aware anti joins as they have specific side requirements- if hash_join.join_type().supports_swap()- && !hash_join.null_aware+ if can_swap_hash_join(hash_join)&& should_swap_join_order(&**left, &**right, config, registry)?{- hash_join- .swap_inputs(PartitionMode::Partitioned)- .map(Some)?+ // Null-aware RightAnti only supports CollectLeft+ let partition_mode = if hash_join.null_aware {+ PartitionMode::CollectLeft+ } else {+ PartitionMode::Partitioned+ };+ hash_join.swap_inputs(partition_mode).map(Some)?} else {None}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 08d209003a..34e901b9a3 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -221,6 +221,9 @@ pub(super) struct JoinLeftData {pub(super) probe_side_non_empty: AtomicBool,/// Shared atomic flag indicating if any probe partition saw NULL in join keys (for null-aware anti joins)pub(super) probe_side_has_null: AtomicBool,++ // For RightAnti joins, where the build side is a smaller subquery, truthy if has null for the single join key+ pub(super) build_side_has_null: bool,}impl JoinLeftData {@@ -426,9 +429,14 @@ impl HashJoinExecBuilder {// Validate null_aware flagif exec.null_aware {let join_type = exec.join_type();- if !matches!(join_type, JoinType::LeftAnti) {+ let partition_mode = exec.partition_mode();+ if !matches!(+ (join_type, partition_mode),+ (JoinType::LeftAnti, _)+ | (JoinType::RightAnti, PartitionMode::CollectLeft) // `PartitionMode::CollectLeft` is safe because `RightAnti` is probe-driven+ ) {return plan_err!(- "null_aware can only be true for LeftAnti joins, got {join_type}"+ "null_aware can only be true for LeftAnti joins and RightAnti joins with `CollectLeft` `PartitionMode`, got {join_type} with {partition_mode}");}let on = exec.on();@@ -438,6 +446,11 @@ impl HashJoinExecBuilder {on.len());}+ if *join_type == JoinType::RightAnti && exec.filter.is_some() {+ return plan_err!(+ "null_aware RightAnti join does not support a join filter"+ );+ }}if preserve_properties {@@ -1504,6 +1517,7 @@ impl ExecutionPlan for HashJoinExec {enable_dynamic_filter_pushdown,Arc::clone(context.session_config().options()),self.null_equality,+ self.null_aware && self.join_type == JoinType::RightAnti,array_map_created_count,))})?,@@ -1524,6 +1538,7 @@ impl ExecutionPlan for HashJoinExec {enable_dynamic_filter_pushdown,Arc::clone(context.session_config().options()),self.null_equality,+ false,array_map_created_count,))}@@ -2238,6 +2253,7 @@ async fn collect_left_input(should_compute_dynamic_filters: bool,config: Arc<ConfigOptions>,null_equality: NullEquality,+ compute_build_side_has_null: bool,array_map_created_count: Count,) -> Result<JoinLeftData> {let schema = left_stream.schema();@@ -2414,6 +2430,10 @@ async fn collect_left_input(bounds = None;}+ let build_has_null = compute_build_side_has_null+ && !left_values.is_empty()+ && left_values[0].logical_null_count() > 0;+let data = JoinLeftData {map,batch,@@ -2425,6 +2445,7 @@ async fn collect_left_input(membership,probe_side_non_empty: AtomicBool::new(false),probe_side_has_null: AtomicBool::new(false),+ build_side_has_null: build_has_null,};Ok(data)@@ -2482,7 +2503,8 @@ mod tests {};use arrow::array::{- Date32Array, Int32Array, Int64Array, StructArray, UInt32Array, UInt64Array,+ Array, ArrayRef, Date32Array, DictionaryArray, Int32Array, Int64Array,+ StructArray, UInt32Array, UInt64Array,};use arrow::buffer::NullBuffer;use arrow::datatypes::{DataType, Field};@@ -2499,6 +2521,7 @@ mod tests {use datafusion_physical_expr::{EquivalenceProperties, PhysicalSortExpr, RangePartitioning, SplitPoint,};+ use futures::StreamExt;use hashbrown::HashTable;use insta::{allow_duplicates, assert_snapshot};use rstest::*;@@ -2645,6 +2668,36 @@ mod tests {TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()}+ /// Build a two-column table whose join key is a dictionary.+ ///+ /// `dict_values` are the dictionary entries; `keys` are all physically+ /// valid indices into that dictionary. When `dict_values` contains NULL,+ /// the resulting array can have `null_count() == 0` but+ /// `logical_null_count() > 0`.+ fn build_table_dict_key(+ key_name: &str,+ dict_values: Vec<Option<i32>>,+ keys: Vec<i32>,+ dummy_name: &str,+ dummy: Vec<Option<i32>>,+ ) -> Arc<dyn ExecutionPlan> {+ let dict_type =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32));+ let schema = Arc::new(Schema::new(vec![+ Field::new(key_name, dict_type, true),+ Field::new(dummy_name, DataType::Int32, true),+ ]));+ let values = Int32Array::from(dict_values);+ let key_array = Int32Array::from(keys);+ let dict: ArrayRef = Arc::new(DictionaryArray::new(key_array, Arc::new(values)));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![dict, Arc::new(Int32Array::from(dummy))],+ )+ .unwrap();+ TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap()+ }+fn join(left: Arc<dyn ExecutionPlan>,right: Arc<dyn ExecutionPlan>,@@ -6871,6 +6924,337 @@ mod tests {Ok(())}+ /// Test null-aware RightAnti when build side (subquery) contains NULL+ /// Expected: no rows should be output+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_build_null(batch_size: usize) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ // Build left table (subquery with NULL)+ let left = build_table_two_cols(+ ("c1", &vec![Some(1), Some(2), Some(3), None]),+ ("dummy", &vec![Some(100), Some(200), Some(300), Some(400)]),+ );++ // Build right table (outer rows to potentially output)+ let right = build_table_two_cols(+ ("c2", &vec![Some(1), Some(2), Some(3), Some(4)]),+ ("dummy", &vec![Some(10), Some(20), Some(30), Some(40)]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +++ +++ ");+ }+ Ok(())+ }++ /// Test null-aware RightAnti when probe side (outer) contains NULL keys+ /// Expected: rows with NULL keys should not be output+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_probe_null(batch_size: usize) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ // Build left table (subquery, no NULL)+ let left = build_table_two_cols(+ ("c1", &vec![Some(1), Some(2), Some(3)]),+ ("dummy", &vec![Some(100), Some(200), Some(300)]),+ );++ // Build right table with NULL key (this row should not be output)+ let right = build_table_two_cols(+ ("c2", &vec![Some(1), Some(4), None]),+ ("dummy", &vec![Some(10), Some(40), Some(0)]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ // Expected: only c2=4 (not c2=1 which matches, not c2=NULL)+ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | c2 | dummy |+ +----+-------++ | 4 | 40 |+ +----+-------++ ");+ }+ Ok(())+ }++ /// Test null-aware RightAnti with no NULLs (should work like regular RightAnti)+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_no_nulls(batch_size: usize) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ // Build left table (subquery, no NULLs)+ let left = build_table_two_cols(+ ("c1", &vec![Some(1), Some(2), Some(3)]),+ ("dummy", &vec![Some(100), Some(200), Some(300)]),+ );++ // Build right table (outer, no NULLs)+ let right = build_table_two_cols(+ ("c2", &vec![Some(1), Some(2), Some(4), Some(5)]),+ ("dummy", &vec![Some(10), Some(20), Some(40), Some(50)]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ // Expected: c2=4 and c2=5 (they don't match anything in left)+ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | c2 | dummy |+ +----+-------++ | 4 | 40 |+ | 5 | 50 |+ +----+-------++ ");+ }+ Ok(())+ }++ /// Test null-aware RightAnti when build side (subquery) is empty+ /// Expected: all outer rows should be output, including NULL keys+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_empty_build(batch_size: usize) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ // Build left table (empty subquery)+ let left = build_table_two_cols(("c1", &vec![]), ("dummy", &vec![]));++ // Build right table (outer)+ let right = build_table_two_cols(+ ("c2", &vec![Some(1), None, Some(4)]),+ ("dummy", &vec![Some(10), Some(0), Some(40)]),+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | c2 | dummy |+ +----+-------++ | | 0 |+ | 1 | 10 |+ | 4 | 40 |+ +----+-------++ ");+ }+ Ok(())+ }++ /// Null-aware RightAnti must treat dictionary keys that point at a null+ /// dictionary value as build-side NULLs, even when the key bitmap has no+ /// physical nulls (`null_count() == 0` but `logical_null_count() > 0`).+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_build_logical_null(+ batch_size: usize,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ // Dictionary values: [1, NULL, 2]; keys: [0, 1, 2] => logical [1, NULL, 2]+ let left = build_table_dict_key(+ "c1",+ vec![Some(1), None, Some(2)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(100), Some(200), Some(300)],+ );+ let left_key = left+ .execute(0, Arc::clone(&task_ctx))?+ .next()+ .await+ .unwrap()?;+ assert_eq!(left_key.column(0).null_count(), 0);+ assert_eq!(left_key.column(0).logical_null_count(), 1);++ let right = build_table_dict_key(+ "c2",+ vec![Some(1), Some(2), Some(3), Some(4)],+ vec![0, 1, 2, 3],+ "dummy",+ vec![Some(10), Some(20), Some(30), Some(40)],+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +++ +++ ");+ }+ Ok(())+ }++ /// Null-aware RightAnti must drop outer rows whose dictionary key is only+ /// logically NULL (key points at a null dictionary value).+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_right_anti_probe_logical_null(+ batch_size: usize,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ let left = build_table_dict_key(+ "c1",+ vec![Some(1), Some(2), Some(3)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(100), Some(200), Some(300)],+ );++ // Dictionary values: [1, NULL, 4]; keys: [0, 1, 2] => logical [1, NULL, 4]+ let right = build_table_dict_key(+ "c2",+ vec![Some(1), None, Some(4)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(10), Some(0), Some(40)],+ );+ let right_key = right+ .execute(0, Arc::clone(&task_ctx))?+ .next()+ .await+ .unwrap()?;+ assert_eq!(right_key.column(0).null_count(), 0);+ assert_eq!(right_key.column(0).logical_null_count(), 1);++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ // Expected: only c2=4 (not c2=1 which matches, not logically NULL c2)+ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | c2 | dummy |+ +----+-------++ | 4 | 40 |+ +----+-------++ ");+ }+ Ok(())+ }+/// Test that null_aware validation rejects non-LeftAnti join types#[tokio::test]async fn test_null_aware_validation_wrong_join_type() {@@ -6898,12 +7282,40 @@ mod tests {);assert!(result.is_err());- assert!(- result- .unwrap_err()- .to_string()- .contains("null_aware can only be true for LeftAnti joins")+ assert!(result.unwrap_err().to_string().contains(+ "null_aware can only be true for LeftAnti joins and RightAnti joins"+ ));+ }++ /// Test that null_aware RightAnti rejects Partitioned mode+ #[tokio::test]+ async fn test_null_aware_validation_right_anti_partitioned() {+ let left =+ build_table_two_cols(("c1", &vec![Some(1)]), ("dummy", &vec![Some(10)]));+ let right =+ build_table_two_cols(("c2", &vec![Some(1)]), ("dummy", &vec![Some(100)]));++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema()).unwrap()) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema()).unwrap()) as _,+ )];++ let result = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::RightAnti,+ None,+ PartitionMode::Partitioned,+ NullEquality::NullEqualsNothing,+ true,);++ assert!(result.is_err());+ assert!(result.unwrap_err().to_string().contains(+ "null_aware can only be true for LeftAnti joins and RightAnti joins"+ ));}/// Test that null_aware validation rejects multi-column joins@@ -6946,6 +7358,121 @@ mod tests {);}+ /// A null-aware `RightAnti` short-circuits on the build-side NULL before any+ /// filter runs, so the combination must be rejected at construction (and thus+ /// via protobuf decoding, which routes through the same builder). A filtered+ /// null-aware `LeftAnti`, by contrast, is a valid decorrelated correlated+ /// `NOT IN` plan and must still be accepted.+ #[tokio::test]+ async fn test_null_aware_filter_rejected_only_for_right_anti() {+ let make_inputs = || {+ let left =+ build_table_two_cols(("c1", &vec![Some(1)]), ("dummy", &vec![Some(10)]));+ let right =+ build_table_two_cols(("c2", &vec![Some(1)]), ("dummy", &vec![Some(100)]));+ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema()).unwrap()) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema()).unwrap()) as _,+ )];+ (left, right, on)+ };++ // RightAnti + filter is rejected.+ let (left, right, on) = make_inputs();+ let result = HashJoinExec::try_new(+ left,+ right,+ on,+ Some(prepare_join_filter()),+ &JoinType::RightAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ );+ assert!(result.is_err());+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("null_aware RightAnti join does not support a join filter")+ );++ // LeftAnti + filter is accepted (correlated NOT IN decorrelates to this).+ let (left, right, on) = make_inputs();+ let result = HashJoinExec::try_new(+ left,+ right,+ on,+ Some(prepare_join_filter()),+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ );+ assert!(result.is_ok());+ }++ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_left_anti_probe_logical_null(+ batch_size: usize,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ let left = build_table_dict_key(+ "c1",+ vec![Some(1), Some(2), Some(3)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(100), Some(200), Some(300)],+ );++ let right = build_table_dict_key(+ "c2",+ vec![Some(1), None, Some(4)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(10), Some(0), Some(40)],+ );+ let right_key = right+ .execute(0, Arc::clone(&task_ctx))?+ .next()+ .await+ .unwrap()?;+ assert_eq!(right_key.column(0).null_count(), 0);+ assert_eq!(right_key.column(0).logical_null_count(), 1);++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +++ +++ ");+ }+ Ok(())+ }+#[test]fn test_lr_is_preserved() {assert_eq!(lr_is_preserved(JoinType::Inner), (true, true));diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 686939537e..3bb2d6d989 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -750,40 +750,63 @@ impl HashJoinStream {let timer = self.join_metrics.join_time.timer();// Null-aware anti join semantics:+// For LeftAnti: output LEFT (build) rows where LEFT.key NOT IN RIGHT.key// 1. If RIGHT (probe) contains NULL in any batch, no LEFT rows should be output// 2. LEFT rows with NULL keys should not be output (handled in final stage)++ // For RightAnti: output RIGHT (probe) rows where RIGHT.key NOT IN LEFT.key+ // 1. If LEFT (build) contains NULL, no RIGHT rows should be output+ // 2. RIGHT rows with NULL keys should not be output+ // 3. If LEFT (build) is empty, all RIGHT rows should be outputif self.null_aware {- // Mark that we've seen a probe batch with actual rows (probe side is non-empty)- // Only set this if batch has rows - empty batches don't count- // Use shared atomic state so all partitions can see this global information- if state.batch.num_rows() > 0 {- build_side- .left_data- .probe_side_non_empty- .store(true, Ordering::Relaxed);- }+ match self.join_type {+ JoinType::RightAnti => {+ if build_side.left_data.build_side_has_null {+ timer.done();+ self.state = HashJoinStreamState::FetchProbeBatch;+ return Ok(StatefulStreamResult::Continue);+ }+ }+ JoinType::LeftAnti => {+ // Mark that we've seen a probe batch with actual rows (probe side is non-empty)+ // Only set this if batch has rows - empty batches don't count+ // Use shared atomic state so all partitions can see this global information+ if state.batch.num_rows() > 0 {+ build_side+ .left_data+ .probe_side_non_empty+ .store(true, Ordering::Relaxed);+ }- // Check if probe side (RIGHT) contains NULL- // Since null_aware validation ensures single column join, we only check the first column- let probe_key_column = &state.values[0];- if probe_key_column.null_count() > 0 {- // Found NULL in probe side - set shared flag to prevent any output- build_side- .left_data- .probe_side_has_null- .store(true, Ordering::Relaxed);- }+ // Check if probe side (RIGHT) contains NULL+ // Since null_aware validation ensures single column join, we only check the first column+ let probe_key_column = &state.values[0];+ let probe_has_null = if self.filter.is_some() {+ probe_key_column.null_count() > 0+ } else {+ probe_key_column.logical_null_count() > 0+ };+ if probe_has_null {+ // Found NULL in probe side - set shared flag to prevent any output+ build_side+ .left_data+ .probe_side_has_null+ .store(true, Ordering::Relaxed);+ }- // If probe side has NULL (detected in this or any other partition), return empty result- if build_side- .left_data- .probe_side_has_null- .load(Ordering::Relaxed)- {- timer.done();- self.state = HashJoinStreamState::FetchProbeBatch;- return Ok(StatefulStreamResult::Continue);+ // If probe side has NULL (detected in this or any other partition), return empty result+ if build_side+ .left_data+ .probe_side_has_null+ .load(Ordering::Relaxed)+ {+ timer.done();+ self.state = HashJoinStreamState::FetchProbeBatch;+ return Ok(StatefulStreamResult::Continue);+ }+ }+ _ => {}}}@@ -902,7 +925,7 @@ impl HashJoinStream {last_joined_right_idx.map_or(0, |v| v + 1)};- let (left_indices, right_indices) = adjust_indices_by_join_type(+ let (left_indices, mut right_indices) = adjust_indices_by_join_type(left_indices,right_indices,index_alignment_range_start..index_alignment_range_end,@@ -910,6 +933,26 @@ impl HashJoinStream {self.right_side_ordered,)?;+ // If null-aware RightAnti join, we don't want to emit NULL probe keys+ if self.join_type == JoinType::RightAnti && self.null_aware {+ let probe_key = &state.values[0];+ // if the valid_keys mask is available, use that, else use the probe key+ let mask = state+ .valid_keys+ .clone()+ .or_else(|| probe_key.logical_nulls());+ // we only need this copy if there are NULLs+ if let Some(mask) = mask.filter(|m| m.null_count() > 0) {+ let filtered_right_indices = right_indices+ .values()+ .iter()+ .copied()+ .filter(|idx| !mask.is_null(*idx as usize))+ .collect::<Vec<_>>();+ right_indices = UInt32Array::from(filtered_right_indices);+ }+ }+// Build output batch and push to coalescerlet (build_batch, probe_batch, join_side) =if self.join_type == JoinType::RightMark {diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex e4f7e2e123..fc16571b79 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -17,6 +17,9 @@//! DataFusion Join implementations+use core::fmt;+use std::fmt::{Display, Formatter};+use arrow::array::BooleanBufferBuilder;pub use cross_join::CrossJoinExec;use datafusion_physical_expr::PhysicalExprRef;@@ -104,6 +107,17 @@ pub enum PartitionMode {Auto,}+impl Display for PartitionMode {+ fn fmt(&self, f: &mut Formatter) -> fmt::Result {+ let partition_mode = match self {+ PartitionMode::Partitioned => "Partitioned",+ PartitionMode::CollectLeft => "CollectLeft",+ PartitionMode::Auto => "Auto",+ };+ write!(f, "{partition_mode}")+ }+}+/// Partitioning mode to use for symmetric hash join#[derive(Hash, Clone, Copy, Debug, PartialEq, Eq)]pub enum StreamJoinPartitionMode {diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex eec6e5ae17..41d259e88c 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -366,8 +366,8 @@ physical_planstatement count 0RESET datafusion.execution.parquet.pushdown_filters;-# LEFT ANTI JOIN: both self generated and parent filters can push to the-# preserved (left/build) side.+# NULL-AWARE ANTI JOIN: JoinSelection swaps the physical join to RightAnti so+# the subquery is the build side and the preserved outer input is the probe side.query TTEXPLAIN SELECT l.*FROM left_parquet l@@ -381,9 +381,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet-03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)], null_aware+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet# LEFT MARK JOIN: the OR prevents decorrelation to LeftSemi, so the optimizer# uses LeftMark. Self-generated dynamic filter pushes to the probe side.@@ -457,9 +457,10 @@ ORDER BY l.id LIMIT 2;1 left13 left3-# ANTI JOIN with TopK parent: the TopK dynamic filter on `id` is pushed only-# to the preserved output side. Filtering the non-output side can create-# anti-join output.+# ANTI JOIN with TopK parent: JoinSelection swaps the physical join to+# RightAnti (build = subquery, probe = preserved outer input). The TopK dynamic+# filter on `id` is pushed only to the preserved output (probe) side; filtering+# the non-output build side can create anti-join output.query TTEXPLAIN SELECT l.*FROM left_parquet l@@ -476,9 +477,9 @@ logical_plan07)--------TableScan: right_parquet projection=[id]physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet+02)--HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)], null_aware+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible# Correctness checkquery ITdiff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex bdb56cf220..4beab6998d 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -48,7 +48,8 @@ SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_no_null);1 a3 c-# Verify the plan uses LeftAnti join+# The logical plan remains LeftAnti, while JoinSelection swaps the physical+# join to RightAnti so the smaller subquery is the build side.query TTEXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_no_null);----@@ -58,7 +59,7 @@ logical_plan03)--SubqueryAlias: __correlated_sq_104)----TableScan: inner_table_no_null projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]@@ -198,11 +199,10 @@ WHERE id NOT IN (SELECT id FROM inner_table_with_null WHERE value = 'x');# so this behaves like regular anti join (not null-aware)#############-## Test 11: Verify NULL-aware flag is set for LeftAnti joins+## Test 11: Verify the null-aware LeftAnti logical join is physically swapped#############-# Check that the physical plan shows null-aware anti join-# Note: The exact format may vary, but we should see LeftAnti join type+# Check that JoinSelection builds on the smaller subquery using RightAnti.query TTEXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);----@@ -212,7 +212,7 @@ logical_plan03)--SubqueryAlias: __correlated_sq_104)----TableScan: inner_table_with_null projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 594964778b..c38fb4c7ee 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -2626,6 +2626,7 @@ SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null) ORDER# The null-aware anti join must be planned as a CollectLeft HashJoinExec even with# prefer_hash_join = false: SortMergeJoinExec is not null-aware and must not be used.+# This test has no usable size statistics, so JoinSelection keeps LeftAnti.query TTEXPLAIN SELECT x FROM nia_left WHERE x NOT IN (SELECT y FROM nia_right_with_null);----