Apache DataFusion SQL Query Engine
fix: drop build rows with a dictionary NULL key in null-aware anti joins (#26578)
Alejandro Borregoa79a694887a1
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 9f77d782b2..178a55af6e 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -357,11 +357,12 @@ struct JoinBuildData {/// `build_side_has_null` were computed from the original keys in/// `collect_left_input`.values: Vec<ArrayRef>,- /// Logical nulls of the scalar key for uncorrelated null-aware `LeftMark`.+ /// Logical nulls of the scalar key for uncorrelated null-aware `LeftMark`+ /// and `LeftAnti`.////// Computed once while collecting the build side because dictionary/// logical-null discovery can scan nested dictionary values.- left_mark_build_key_nulls: Option<NullBuffer>,+ null_aware_build_key_nulls: Option<NullBuffer>,/// Bounds computed from the build side; absent for an empty partition.bounds: Option<PartitionBounds>,/// IN-list values or a hash-table reference used for filter pushdown.@@ -438,8 +439,8 @@ impl JoinLeftData {&self.build.values}- pub(super) fn left_mark_build_key_nulls(&self) -> Option<&NullBuffer> {- self.build.left_mark_build_key_nulls.as_ref()+ pub(super) fn null_aware_build_key_nulls(&self) -> Option<&NullBuffer> {+ self.build.null_aware_build_key_nulls.as_ref()}/// returns a reference to the visited indices bitmap@@ -3647,12 +3648,17 @@ fn build_in_memory(&& !left_values.is_empty()&& left_values[0].logical_null_count() > 0;- let left_mark_build_key_nulls = if matches!(+ let null_aware_build_key_nulls = if matches!(null_aware,- Some(NullAwareMode::LeftMark {- correlated: false,- ..- })+ Some(+ NullAwareMode::LeftMark {+ correlated: false,+ ..+ } | NullAwareMode::LeftAnti {+ correlated: false,+ ..+ }+ )) {left_values[0].logical_nulls()@@ -3660,7 +3666,7 @@ fn build_in_memory(} else {None};- if let Some(nulls) = &left_mark_build_key_nulls {+ if let Some(nulls) = &null_aware_build_key_nulls {// `logical_nulls` reuses an array's physical null buffer when it can.// Only charge newly materialized buffers, such as dictionary logical// nulls, because the build batch's buffers were reserved above.@@ -3692,7 +3698,7 @@ fn build_in_memory(map,batch,values: normalized_values,- left_mark_build_key_nulls,+ null_aware_build_key_nulls,bounds,membership,reservation,@@ -10549,6 +10555,73 @@ mod tests {Ok(())}+ /// The build side counterpart of+ /// [`test_null_aware_left_anti_probe_logical_null`]: `NULL NOT IN (5, 6)`+ /// is UNKNOWN, so the build row whose key points at a NULL dictionary+ /// value is dropped.+ #[apply(hash_join_exec_configs)]+ #[tokio::test]+ async fn test_null_aware_left_anti_build_logical_null(+ batch_size: usize,+ ) -> Result<()> {+ let task_ctx = prepare_task_ctx(batch_size, false);++ let left = build_table_dict_key(+ "c1",+ vec![Some(1), None, Some(2)],+ vec![0, 1, 2],+ "dummy",+ vec![Some(10), Some(0), Some(20)],+ );+ let left_key = left+ .execute(0, Arc::clone(&task_ctx))?+ .next()+ .await+ .unwrap()?;+ assert_eq!(left_key.column(0).null_count(), 0);+ assert_eq!(left_key.column(0).logical_null_count(), 1);++ let right = build_table_dict_key(+ "c2",+ vec![Some(5), Some(6)],+ vec![0, 1],+ "dummy",+ vec![Some(50), Some(60)],+ );++ let on = vec![(+ Arc::new(Column::new_with_schema("c1", &left.schema())?) as _,+ Arc::new(Column::new_with_schema("c2", &right.schema())?) as _,+ )];++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::LeftAnti,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ true,+ )?;++ let stream = join.execute(0, task_ctx)?;+ let batches = common::collect(stream).await?;++ allow_duplicates! {+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +----+-------++ | c1 | dummy |+ +----+-------++ | 1 | 10 |+ | 2 | 20 |+ +----+-------++ ");+ }+ Ok(())+ }+/// A dictionary key that points to a NULL dictionary value is a NULL build/// key even when the dictionary key bitmap itself has no NULLs.#[apply(hash_join_exec_configs)]diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex db12f65bb0..00506d9c6c 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -1379,12 +1379,15 @@ fn null_aware_left_anti_final_indices(if !probe_summary.non_empty {return (left_side, right_side);}- // null_aware validation ensures a single join key- let build_key_column = &left_data.values()[0];+ // Logical nulls, so that a dictionary key pointing at a NULL value counts+ // as a NULL build key. Computed once with the build side.+ let Some(build_key_nulls) = left_data.null_aware_build_key_nulls() else {+ return (left_side, right_side);+ };let left_side = UInt64Array::from_iter_values(left_side.iter().filter_map(|idx| {let idx =idx.expect("LeftAnti final indices should always contain build-side rows");- (!build_key_column.is_null(idx as usize)).then_some(idx)+ (!build_key_nulls.is_null(idx as usize)).then_some(idx)}));let right_side = UInt32Array::new_null(left_side.len());(left_side, right_side)@@ -1406,7 +1409,7 @@ fn null_aware_left_mark_column(let build_key_nulls = if correlated {None} else {- left_data.left_mark_build_key_nulls()+ left_data.null_aware_build_key_nulls()};build_null_aware_left_mark_column(left_side,