Apache DataFusion SQL Query Engine
feat: build multi-column null-aware NOT IN on the subquery side (#26213)
Liang-Chi Hsiehc6869995c11c
diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmarkindex 1857ef64d2..3ed9ff477c 100644--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q10.benchmark@@ -23,9 +23,9 @@ expect_plan HashJoinExecrun-- Q10: uncorrelated multi-column NOT IN over nullable keys that hold no NULL.--- A tuple NOT IN always decides UNKNOWN per build row (the outer side), but--- without NULLs there is no candidate pair to check, so this is the floor for--- the multi-column shape.+-- A tuple NOT IN decides UNKNOWN per row (here per outer row, as the join is+-- swapped to build on the subquery), but without NULLs there is no candidate+-- pair to check, so this is the floor for the multi-column shape.SELECT count(*)FROM small_outer oWHERE (o.id_n0, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);diff --git a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmarkindex c126c9c161..8197a47698 100644--- a/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark+++ b/benchmarks/sql_benchmarks/null_aware_join/benchmarks/q12.benchmark@@ -26,10 +26,10 @@ expect_plan HashJoinExecrun-- Q12: uncorrelated multi-column NOT IN with 1% NULL in one outer element,--- the mirror of Q11. The outer side is the build side, so each NULL-valued--- build row is paired with every subquery row and kept as UNKNOWN only when--- the other element matches: the cost grows with (NULL-valued outer rows) x--- (subquery rows). Compare with Q10 and Q11.+-- the mirror of Q11. Each NULL-valued outer row is paired with every subquery+-- row and is UNKNOWN as soon as the other element of one of them matches: the+-- cost grows with (NULL-valued outer rows) x (subquery rows). Compare with Q10+-- and Q11.SELECT count(*)FROM small_outer oWHERE (o.id_n1, o.z) NOT IN (SELECT i.id_n0, i.z FROM small_inner i);diff --git a/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rsindex 4bf7171771..c76e4d1a83 100644--- a/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs+++ b/datafusion/core/tests/fuzz_cases/null_aware_join_fuzz.rs@@ -21,23 +21,34 @@//! null-aware hash joins, are compared with `NOT EXISTS` / `EXISTS`//! formulations of the same three-valued logic, which do not. Random tables//! with NULLs on both sides are swept across batch sizes and partition counts.+//!+//! The subquery table is sometimes much smaller than the outer table, so+//! `JoinSelection` also swaps `NOT IN` into a null-aware `RightAnti` that+//! builds on the subquery side; the test checks that both plan shapes ran.use std::sync::Arc;use arrow::array::{Int32Array, RecordBatch};use arrow::datatypes::{DataType, Field, Schema};use arrow::util::pretty::pretty_format_batches;+use datafusion::common::JoinType;use datafusion::datasource::MemTable;+use datafusion::physical_plan::joins::HashJoinExec;+use datafusion::physical_plan::{ExecutionPlan, collect};use datafusion::prelude::{SessionConfig, SessionContext};use rand::rngs::StdRng;use rand::{Rng, SeedableRng};#[tokio::test(flavor = "multi_thread")]async fn null_aware_join_matches_reference() {+ let mut plans = NullAwarePlans::default();for seed in 0..20 {let mut rng = StdRng::seed_from_u64(seed);- let outer = random_table(&mut rng, &["id", "k", "a", "b", "c"]);- let inner = random_table(&mut rng, &["k", "x", "y", "z"]);+ let outer = random_table(&mut rng, &["id", "k", "a", "b", "c"], 6);+ // Alternate between a subquery table of similar size and a much+ // smaller one that `JoinSelection` moves to the build side.+ let inner_rows = if seed % 2 == 0 { 6 } else { 2 };+ let inner = random_table(&mut rng, &["k", "x", "y", "z"], inner_rows);for target_partitions in [1, 4] {for batch_size in [1, 3, 8192] {let config = SessionConfig::new()@@ -57,8 +68,9 @@ async fn null_aware_join_matches_reference() {ctx.register_table("o", Arc::clone(&outer) as _).unwrap();ctx.register_table("i", Arc::clone(&inner) as _).unwrap();for (query, reference) in cases() {- let actual = run(&ctx, &query).await;- let expected = run(&ctx, &reference).await;+ let (actual, plan) = run(&ctx, &query).await;+ plans.record(plan.as_ref());+ let (expected, _) = run(&ctx, &reference).await;assert_eq!(actual, expected,"seed={seed} target_partitions={target_partitions} \@@ -68,6 +80,43 @@ async fn null_aware_join_matches_reference() {}}}+ // A fuzz run that never reaches one of the plan shapes would be vacuous.+ assert!(plans.left_anti > 0, "{plans:?}");+ assert!(plans.right_anti > 0, "{plans:?}");+ assert!(plans.correlated_right_anti > 0, "{plans:?}");+}++/// Counts the null-aware anti join plans the fuzz queries ran.+#[derive(Debug, Default)]+struct NullAwarePlans {+ left_anti: usize,+ right_anti: usize,+ /// `RightAnti` joins with more than one key or a join filter, which+ /// decide `NOT IN` per probe row.+ correlated_right_anti: usize,+}++impl NullAwarePlans {+ /// Counts the null-aware anti joins in `plan` and its descendants.+ fn record(&mut self, plan: &dyn ExecutionPlan) {+ if let Some(join) = plan.downcast_ref::<HashJoinExec>()+ && join.null_aware+ {+ match join.join_type() {+ JoinType::LeftAnti => self.left_anti += 1,+ JoinType::RightAnti => {+ self.right_anti += 1;+ if join.on().len() > 1 || join.filter().is_some() {+ self.correlated_right_anti += 1;+ }+ }+ _ => {}+ }+ }+ for child in plan.children() {+ self.record(child.as_ref());+ }+ }}/// Pairs of (query, reference) that must return the same rows.@@ -131,8 +180,9 @@ fn cases() -> Vec<(String, String)> {}/// A table of nullable `Int32` columns with values in `0..3`, split into up-/// to three partitions. A column named `id` holds unique row ids instead.-fn random_table(rng: &mut StdRng, columns: &[&str]) -> Arc<MemTable> {+/// to three partitions of fewer than `max_rows` rows each. A column named `id`+/// holds unique row ids instead.+fn random_table(rng: &mut StdRng, columns: &[&str], max_rows: i32) -> Arc<MemTable> {let schema = Arc::new(Schema::new(columns.iter()@@ -143,7 +193,7 @@ fn random_table(rng: &mut StdRng, columns: &[&str]) -> Arc<MemTable> {let mut next_id = 0;let partitions = (0..rng.random_range(1..4)).map(|_| {- let rows = rng.random_range(0..6);+ let rows = rng.random_range(0..max_rows);let arrays = columns.iter().map(|name| {@@ -167,9 +217,16 @@ fn random_table(rng: &mut StdRng, columns: &[&str]) -> Arc<MemTable> {Arc::new(MemTable::try_new(schema, partitions).unwrap())}-/// The rows of `sql`, sorted.-async fn run(ctx: &SessionContext, sql: &str) -> Vec<String> {- let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap();+/// The rows of `sql`, sorted, and the physical plan that produced them.+async fn run(ctx: &SessionContext, sql: &str) -> (Vec<String>, Arc<dyn ExecutionPlan>) {+ let plan = ctx+ .sql(sql)+ .await+ .unwrap()+ .create_physical_plan()+ .await+ .unwrap();+ let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await.unwrap();let formatted = pretty_format_batches(&batches).unwrap().to_string();let mut rows: Vec<String> = formatted.lines()@@ -178,5 +235,5 @@ async fn run(ctx: &SessionContext, sql: &str) -> Vec<String> {.map(String::from).collect();rows.sort();- rows+ (rows, plan)}diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 9f23432990..3b2241b87b 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -50,7 +50,9 @@ use datafusion_physical_plan::collect;use datafusion_physical_plan::displayable;use datafusion_physical_plan::joins::utils::ColumnIndex;use datafusion_physical_plan::joins::utils::JoinFilter;-use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode};+use datafusion_physical_plan::joins::{+ HashJoinExec, HashJoinExecBuilder, NestedLoopJoinExec, PartitionMode,+};use datafusion_physical_plan::operator_statistics::{ClosureStatisticsProvider, StatisticsRegistry, StatisticsResult,};@@ -469,32 +471,45 @@ async fn test_join_with_swap_semi() {}}+/// A null-aware `LeftAnti` swaps to a null-aware `RightAnti` that builds on+/// the smaller subquery side. Besides the scalar `NOT IN`, this covers a+/// multi-key one (a multi-column `NOT IN`, correlation scope keys, or both),+/// which keeps the number of `NOT IN` value keys and the key order, and a+/// filtered one (a decorrelated correlated `NOT IN`), whose filter sides are+/// swapped with the inputs.+////// `PartitionMode::Partitioned` is deliberately not a case here: a null-aware/// `LeftAnti` cannot be constructed in that mode at all, so there is no input/// plan to optimize. See/// `HashJoinExec::null_aware_mode`/`NullAwareMode::try_new`.#[rstest]-#[case(PartitionMode::CollectLeft)]-#[case(PartitionMode::Auto)]+#[case::collect_left(PartitionMode::CollectLeft, 1, 1, false)]+#[case::auto(PartitionMode::Auto, 1, 1, false)]+#[case::filter(PartitionMode::CollectLeft, 1, 1, true)]+#[case::scope_key(PartitionMode::CollectLeft, 2, 1, false)]+#[case::multi_column(PartitionMode::CollectLeft, 2, 2, false)]#[tokio::test]async fn test_null_aware_left_anti_swaps_to_right_anti(#[case] partition_mode: PartitionMode,+ #[case] num_keys: usize,+ #[case] value_keys: usize,+ #[case] with_filter: bool,) -> Result<()> {let (big, small) = create_big_and_small();- let join = HashJoinExec::try_new(+ let big_col = Arc::new(Column::new_with_schema("big_col", &big.schema())?) as _;+ let small_col = Arc::new(Column::new_with_schema("small_col", &small.schema())?) as _;+ let on = vec![(big_col, small_col); num_keys];+ let join = HashJoinExecBuilder::new(Arc::clone(&big),Arc::clone(&small),- vec![(- Arc::new(Column::new_with_schema("big_col", &big.schema())?),- Arc::new(Column::new_with_schema("small_col", &small.schema())?),- )],- None,- &JoinType::LeftAnti,- None,- partition_mode,- NullEquality::NullEqualsNothing,- true,- )?;+ on,+ JoinType::LeftAnti,+ )+ .with_filter(if with_filter { nl_join_filter() } else { None })+ .with_partition_mode(partition_mode)+ .with_null_aware(true)+ .with_null_aware_value_keys(value_keys)+ .build()?;let original_schema = join.schema();let optimized_join =@@ -506,9 +521,18 @@ async fn test_null_aware_left_anti_swaps_to_right_anti(assert_eq!(*swapped_join.join_type(), JoinType::RightAnti);assert_eq!(*swapped_join.partition_mode(), PartitionMode::CollectLeft);assert!(swapped_join.null_aware);+ assert_eq!(swapped_join.null_aware_value_keys, value_keys);+ assert_eq!(swapped_join.on().len(), num_keys);assert_eq!(swapped_join.left().schema().field(0).name(), "small_col");assert_eq!(swapped_join.right().schema().field(0).name(), "big_col");assert_eq!(swapped_join.schema(), original_schema);+ let filter_sides = swapped_join+ .filter()+ .map(|filter| filter.column_indices().iter().map(|c| c.side).collect());+ assert_eq!(+ filter_sides,+ with_filter.then(|| vec![JoinSide::Right, JoinSide::Left])+ );Ok(())}@@ -620,44 +644,6 @@ async fn test_null_aware_left_anti_respects_disabled_join_reordering() -> ResultOk(())}-/// A filtered null-aware `LeftAnti` (produced by decorrelating a correlated-/// `NOT IN`) must not be swapped to `RightAnti`: the swap is only valid for-/// unfiltered null-aware joins, since a filtered `RightAnti` would apply its-/// build-side NULL short-circuit before the filter.-#[tokio::test]-async fn test_null_aware_left_anti_with_filter_does_not_swap() -> Result<()> {- let (big, small) = create_big_and_small();- let join = HashJoinExec::try_new(- Arc::clone(&big),- Arc::clone(&small),- vec![(- Arc::new(Column::new_with_schema("big_col", &big.schema())?),- Arc::new(Column::new_with_schema("small_col", &small.schema())?),- )],- nl_join_filter(),- &JoinType::LeftAnti,- None,- PartitionMode::CollectLeft,- NullEquality::NullEqualsNothing,- true,- )?;-- let optimized_join =- JoinSelection::new().optimize(Arc::new(join), &ConfigOptions::new())?;- let unswapped_join = optimized_join- .downcast_ref::<HashJoinExec>()- .expect("filtered null-aware anti join should not swap");-- assert_eq!(*unswapped_join.join_type(), JoinType::LeftAnti);- assert_eq!(*unswapped_join.partition_mode(), PartitionMode::CollectLeft);- assert!(unswapped_join.null_aware);- assert!(unswapped_join.filter().is_some());- assert_eq!(unswapped_join.left().schema().field(0).name(), "big_col");- assert_eq!(unswapped_join.right().schema().field(0).name(), "small_col");-- Ok(())-}-#[tokio::test]async fn test_join_swap_uses_context_statistics_registry() -> Result<()> {let left: Arc<dyn ExecutionPlan> = Arc::new(StatisticsExec::new(diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rsindex 11975973ed..3559e44977 100644--- a/datafusion/physical-optimizer/src/join_selection.rs+++ b/datafusion/physical-optimizer/src/join_selection.rs@@ -168,14 +168,14 @@ impl PhysicalOptimizerRule for JoinSelection {}}-/// Determines whether it is possible to swap inputs of a hash join - for null-aware joins, we can only swap an uncorrelated `LeftAnti`-/// (a single join key and no filter), because the swapped `RightAnti` has no per-row NULL handling+/// Determines whether it is possible to swap inputs of a hash join. Of the+/// null-aware joins only a `LeftAnti` can be swapped: the resulting `RightAnti`+/// decides `NOT IN` per probe row, including multi-column and correlated+/// (scope keys or a join filter) shapes. A null-aware `LeftMark` has no+/// null-aware `RightMark` counterpart.fn can_swap_hash_join(hash_join: &HashJoinExec) -> bool {hash_join.join_type().supports_swap()- && (!hash_join.null_aware- || (*hash_join.join_type() == JoinType::LeftAnti- && hash_join.on().len() == 1- && hash_join.filter().is_none()))+ && (!hash_join.null_aware || *hash_join.join_type() == JoinType::LeftAnti)}/// Tries to create a [`HashJoinExec`] in [`PartitionMode::CollectLeft`] when possible.diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 178a55af6e..d3d0c55ee0 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -206,8 +206,9 @@ fn array_map_key_range(/// multi-column key), used by correlated null-aware joins (see/// [`NullAwareMode`]).///-/// Such rows are UNKNOWN whenever *any* probe row in their correlation scope-/// passes the join filter, so every probe row must be tested against them.+/// Such rows make `NOT IN` UNKNOWN whenever *any* probe row in their+/// correlation scope passes the join filter (unless another element is a+/// definite mismatch), so every probe row must be tested against them./// Restricting this lookup to the NULL-valued build rows keeps it/// proportional to the number of NULLs instead of enumerating every scope/// match of every probe row.@@ -241,9 +242,10 @@ pub(super) fn null_value_key_mask(value_keys: &[ArrayRef]) -> Option<BooleanArra/// A `correlated` join has more than one key — correlation scope keys/// (`on[V..]`, see [`HashJoinExec::null_aware`]), a multi-column value key, or/// both — or a join filter. A NULL then makes `NOT IN` UNKNOWN only for the-/// build rows whose scope, filter and other tuple elements keep that NULL, so-/// the join records the decision per build row in the null-indices bitmap-/// instead of in shared probe-side flags.+/// rows whose scope, filter and other tuple elements keep that NULL. `LeftAnti`+/// and `LeftMark` record the decision per build row in the null-indices bitmap+/// instead of in shared probe-side flags; `RightAnti` decides it per probe row+/// within each probe batch.#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(super) enum NullAwareMode {/// `build.key NOT IN (probe.key)`: emits build rows. When uncorrelated,@@ -252,9 +254,12 @@ pub(super) enum NullAwareMode {/// `value_keys` is the number of `NOT IN` value keys, see/// [`HashJoinExec::null_aware_value_keys`].LeftAnti { correlated: bool, value_keys: usize },- /// Uncorrelated `probe.key NOT IN (build.key)`: emits probe rows, and- /// none of them once any build key is NULL.- RightAnti,+ /// `probe.key NOT IN (build.key)`: emits probe rows. When uncorrelated,+ /// none of them are emitted once any build key is NULL. When correlated,+ /// a probe row is dropped when some build row in its correlation scope+ /// matches it exactly or has no definite mismatch with it; `value_keys`+ /// as for `LeftAnti`.+ RightAnti { correlated: bool, value_keys: usize },/// `NOT IN` as a nullable mark column on the build rows; `value_keys` as/// for `LeftAnti`.LeftMark { correlated: bool, value_keys: usize },@@ -284,8 +289,12 @@ impl NullAwareMode {correlated,value_keys: num_value_keys,},- // `PartitionMode::CollectLeft` is safe because `RightAnti` is probe-driven- (JoinType::RightAnti, PartitionMode::CollectLeft) => Self::RightAnti,+ // `PartitionMode::CollectLeft` is safe because `RightAnti` is+ // probe-driven: every probe row sees the whole build side.+ (JoinType::RightAnti, PartitionMode::CollectLeft) => Self::RightAnti {+ correlated,+ value_keys: num_value_keys,+ },(JoinType::LeftMark, _) => Self::LeftMark {correlated,value_keys: num_value_keys,@@ -297,9 +306,6 @@ impl NullAwareMode {}};match mode {- Self::RightAnti if num_keys != 1 => plan_err!(- "null_aware {join_type} joins only support single column join key, got {num_keys} columns"- ),// Build-driven joins decide in the final stage from state shared// across the probe partitions: the per-build-row null bitmap for// the correlated forms, and the probe-side NULL flag for the@@ -312,35 +318,32 @@ impl NullAwareMode {"null_aware joins require PartitionMode::CollectLeft, got PartitionMode::Partitioned")}- Self::RightAnti if has_filter => {- plan_err!("null_aware RightAnti join does not support a join filter")- }_ => Ok(mode),}}- /// Whether this join decides UNKNOWN per build row (see [`NullAwareMode`]).+ /// Whether this join decides UNKNOWN per row (see [`NullAwareMode`]).pub(super) fn is_correlated(self) -> bool {- matches!(- self,- Self::LeftAnti {- correlated: true,- ..- } | Self::LeftMark {- correlated: true,- ..- }- )+ match self {+ Self::LeftAnti { correlated, .. }+ | Self::RightAnti { correlated, .. }+ | Self::LeftMark { correlated, .. } => correlated,+ }+ }++ /// Whether this join records UNKNOWN per build row in the shared+ /// null-indices bitmap: a correlated `LeftAnti` or `LeftMark`.+ pub(super) fn marks_build_rows(self) -> bool {+ self.is_correlated() && !matches!(self, Self::RightAnti { .. })}/// Number of leading keys that are `NOT IN` value keys; the keys after/// them are correlation scope keys.pub(super) fn value_keys(self) -> usize {match self {- Self::LeftAnti { value_keys, .. } | Self::LeftMark { value_keys, .. } => {- value_keys- }- Self::RightAnti => 1,+ Self::LeftAnti { value_keys, .. }+ | Self::RightAnti { value_keys, .. }+ | Self::LeftMark { value_keys, .. } => value_keys,}}}@@ -390,7 +393,8 @@ pub(super) struct JoinLeftData {/// Tracks the final probe partition and what every partition observed./// See [`ProbeCompletion`] for the completion invariant.probe_completion: ProbeCompletion,- /// Whether the smaller RightAnti subquery contains a NULL key.+ /// Whether the subquery of an uncorrelated null-aware `RightAnti` join+ /// contains a NULL key.pub(super) build_side_has_null: bool,// Drop the reservation after the mutable state it also accounts for.build: Arc<JoinBuildData>,@@ -1011,10 +1015,14 @@ pub struct HashJoinExec {/// A join filter holds the non-equality part of a correlated subquery, and/// only the probe rows that pass it take part in the three-valued logic.///+ /// `JoinSelection` may swap a null-aware `LeftAnti` into a `RightAnti`+ /// join that builds on the subquery side and decides each outer (probe)+ /// row as it streams.+ ////// Key-ordering convention (relied on positionally, not enforced): for a- /// null-aware `LeftAnti` or `LeftMark` join, `on[..V]` are the `NOT IN`- /// value keys and `on[V..]` the correlated equality scope keys, where `V`- /// is [`Self::null_aware_value_keys`].+ /// null-aware join, `on[..V]` are the `NOT IN` value keys and `on[V..]`+ /// the correlated equality scope keys, where `V` is+ /// [`Self::null_aware_value_keys`]./// Reordering these keys would silently produce wrong results, which is why/// such joins are pinned to `PartitionMode::CollectLeft` (the only key/// reorderer acts solely on `PartitionMode::Partitioned`).@@ -1029,13 +1037,10 @@ pub struct HashJoinExec {/// one element does not make every comparison UNKNOWN the way a scalar/// NULL does.///- /// Such a join always decides UNKNOWN per build row, like a correlated- /// scalar `NOT IN`, which has two costs the uncorrelated scalar join does- /// not: the outer side stays the build side, since only a single-key join- /// is swapped, and without correlation keys every NULL-valued row is paired- /// with every row on the other side to check its other elements. See- /// <https://github.com/apache/datafusion/issues/26088> and- /// <https://github.com/apache/datafusion/issues/26089> for the follow-ups.+ /// Such a join decides UNKNOWN per row, like a correlated scalar `NOT IN`:+ /// without correlation keys every NULL-valued row is paired with every row+ /// on the other side to check its other elements. See+ /// <https://github.com/apache/datafusion/issues/26088> for the follow-up.pub null_aware_value_keys: usize,/// Cache holding plan properties like equivalences, output partitioning etc.cache: Arc<PlanProperties>,@@ -3372,9 +3377,14 @@ fn build_in_memory(array_map_created_count: &Count,prepared: bool,) -> Result<JoinLeftData> {- // The extra scope maps + null bitmap are only built for correlated- // null-aware joins (see `NullAwareMode`).- let with_null_aware_row_state = null_aware.is_some_and(NullAwareMode::is_correlated);+ // Correlated null-aware joins (see `NullAwareMode`), multi-column value+ // keys without scope keys included, pair NULL-valued rows with the other+ // side to find UNKNOWN rows: they need the NULL-valued build rows and, with+ // scope keys, the scope map. Of those, only the ones that decide build rows+ // also need the per-build-row UNKNOWN bitmap.+ let with_null_candidate_state = null_aware.is_some_and(NullAwareMode::is_correlated);+ let with_build_unknown_bitmap =+ null_aware.is_some_and(NullAwareMode::marks_build_rows);let null_aware_value_keys = null_aware.map_or(1, NullAwareMode::value_keys);let is_phj_candidate = is_perfect_hash_join_candidate(on_left, schema)?;@@ -3502,13 +3512,13 @@ fn build_in_memory(BooleanBufferBuilder::new(0)};- let null_indices_bitmap = if with_null_aware_row_state {+ let null_indices_bitmap = if with_build_unknown_bitmap {allocate_bitmap()?} else {BooleanBufferBuilder::new(0)};- let (null_aware_scope_map, null_value_build_rows) = if with_null_aware_row_state {+ let (null_aware_scope_map, null_value_build_rows) = if with_null_candidate_state {// Null-aware convention: `on_left[..V]` are the value keys and// `on_left[V..]` the (possibly empty) correlation scope keys.let scope_keys = &on_left[null_aware_value_keys..];@@ -3644,8 +3654,13 @@ fn build_in_memory(bounds = None;}- let build_has_null = null_aware == Some(NullAwareMode::RightAnti)- && !left_values.is_empty()+ let build_has_null = matches!(+ null_aware,+ Some(NullAwareMode::RightAnti {+ correlated: false,+ ..+ })+ ) && !left_values.is_empty()&& left_values[0].logical_null_count() > 0;let null_aware_build_key_nulls = if matches!(@@ -10318,46 +10333,6 @@ mod tests {));}- /// Test that null_aware validation rejects multi-column joins- #[tokio::test]- async fn test_null_aware_validation_multi_column() {- let left = build_table(("a", &vec![1]), ("b", &vec![2]), ("c", &vec![3]));- let right = build_table(("x", &vec![1]), ("y", &vec![2]), ("z", &vec![3]));-- // Try multi-column join- let on = vec![- (- Arc::new(Column::new_with_schema("a", &left.schema()).unwrap()) as _,- Arc::new(Column::new_with_schema("x", &right.schema()).unwrap()) as _,- ),- (- Arc::new(Column::new_with_schema("b", &left.schema()).unwrap()) as _,- Arc::new(Column::new_with_schema("y", &right.schema()).unwrap()) as _,- ),- ];-- // Try to create null-aware right anti join with 2 columns (should fail).- // A multi-column `LeftAnti` is a correlated `NOT IN` and is accepted.- let result = HashJoinExec::try_new(- left,- right,- on,- None,- &JoinType::RightAnti,- None,- PartitionMode::CollectLeft,- NullEquality::NullEqualsNothing,- true, // null_aware = true (invalid for multi-column)- );-- assert!(result.is_err());- assert!(- result.unwrap_err().to_string().contains(- "null_aware RightAnti joins only support single column join key"- )- );- }-/// An uncorrelated null-aware `LeftAnti` must reject/// `PartitionMode::Partitioned` just like the correlated forms do.///@@ -10418,7 +10393,7 @@ mod tests {),];- // LeftMark (not LeftAnti) so the single-key check is skipped and the partition-mode check is reached.+ // A correlated (multi-key) LeftMark requires CollectLeft.let result = HashJoinExec::try_new(left,right,@@ -10440,60 +10415,55 @@ mod tests {);}- /// A null-aware `RightAnti` short-circuits on the build-side NULL before any- /// filter runs, so the combination must be rejected at construction (and thus- /// via protobuf decoding, which routes through the same builder). A filtered- /// null-aware `LeftAnti`, by contrast, is a valid decorrelated correlated- /// `NOT IN` plan and must still be accepted.- #[tokio::test]- async fn test_null_aware_filter_rejected_only_for_right_anti() {- let make_inputs = || {- let left =- build_table_two_cols(("c1", &vec![Some(1)]), ("dummy", &vec![Some(10)]));- let right =- build_table_two_cols(("c2", &vec![Some(1)]), ("dummy", &vec![Some(100)]));- let on = vec![(- Arc::new(Column::new_with_schema("c1", &left.schema()).unwrap()) as _,- Arc::new(Column::new_with_schema("c2", &right.schema()).unwrap()) as _,- )];- (left, right, on)+ /// A correlated null-aware anti join, with more than one key or a join+ /// filter (a decorrelated correlated `NOT IN`), is accepted both as+ /// `LeftAnti` and as the `RightAnti` that `JoinSelection` swaps it into.+ #[rstest]+ #[case::right_anti_multi_key(JoinType::RightAnti, true, false)]+ #[case::right_anti_filter(JoinType::RightAnti, false, true)]+ #[case::left_anti_filter(JoinType::LeftAnti, false, true)]+ fn test_null_aware_correlated_anti_join_accepted(+ #[case] join_type: JoinType,+ #[case] multi_key: bool,+ #[case] with_filter: bool,+ ) -> Result<()> {+ let left =+ build_table_two_cols(("c1", &vec![Some(1)]), ("dummy", &vec![Some(10)]));+ let right =+ build_table_two_cols(("c2", &vec![Some(1)]), ("dummy", &vec![Some(100)]));+ let key = |l: &str, r: &str| -> Result<(PhysicalExprRef, PhysicalExprRef)> {+ Ok((+ Arc::new(Column::new_with_schema(l, &left.schema())?) as _,+ Arc::new(Column::new_with_schema(r, &right.schema())?) as _,+ ))};-- // RightAnti + filter is rejected.- let (left, right, on) = make_inputs();- let result = HashJoinExec::try_new(- left,- right,- on,- Some(prepare_join_filter()),- &JoinType::RightAnti,- None,- PartitionMode::CollectLeft,- NullEquality::NullEqualsNothing,- true,- );- assert!(result.is_err());- assert!(- result- .unwrap_err()- .to_string()- .contains("null_aware RightAnti join does not support a join filter")- );-- // LeftAnti + filter is accepted (correlated NOT IN decorrelates to this).- let (left, right, on) = make_inputs();- let result = HashJoinExec::try_new(- left,- right,+ let mut on = vec![key("c1", "c2")?];+ if multi_key {+ on.push(key("dummy", "dummy")?);+ }+ let join = HashJoinExec::try_new(+ Arc::clone(&left),+ Arc::clone(&right),on,- Some(prepare_join_filter()),- &JoinType::LeftAnti,+ with_filter.then(prepare_join_filter),+ &join_type,None,PartitionMode::CollectLeft,NullEquality::NullEqualsNothing,true,- );- assert!(result.is_ok());+ )?;+ let expected = match join_type {+ JoinType::RightAnti => NullAwareMode::RightAnti {+ correlated: true,+ value_keys: 1,+ },+ _ => NullAwareMode::LeftAnti {+ correlated: true,+ value_keys: 1,+ },+ };+ assert_eq!(join.null_aware_mode()?, Some(expected));+ Ok(())}#[apply(hash_join_exec_configs)]@@ -11188,10 +11158,12 @@ mod tests {}).collect::<Result<JoinOn>>()?;- let join = HashJoinExec::try_new(- left,- right,- on,+ // The outer side `left` as the build side (`LeftAnti`), and swapped to+ // the probe side (`RightAnti`), must keep the same rows.+ let left_anti = HashJoinExec::try_new(+ Arc::clone(&left),+ Arc::clone(&right),+ on.clone(),None,&JoinType::LeftAnti,None,@@ -11199,9 +11171,14 @@ mod tests {NullEquality::NullEqualsNothing,true,)?;-- let stream = join.execute(0, task_ctx)?;- let batches = common::collect(stream).await?;+ let right_anti = swap_for_separate_execution(&left_anti)?;+ let batches =+ common::collect(left_anti.execute(0, Arc::clone(&task_ctx))?).await?;+ let swapped = common::collect(right_anti.execute(0, task_ctx)?).await?;+ assert_eq!(+ batches_to_sort_string(&batches),+ batches_to_sort_string(&swapped)+ );// Scope (1, 1) holds 2, 3, 4 and 5: `(1, 1, 1)` is kept, `(2, 1, 1)`// matches, and both NULL-valued build rows there are UNKNOWN. Scope@@ -11223,6 +11200,16 @@ mod tests {Ok(())}+ /// Swaps a null-aware `LeftAnti` into the `RightAnti` that `JoinSelection`+ /// plans, with its own build state so that both can be executed.+ fn swap_for_separate_execution(+ left_anti: &HashJoinExec,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ let swapped = left_anti.swap_inputs(PartitionMode::CollectLeft)?;+ let swapped = swapped.downcast_ref::<HashJoinExec>().unwrap();+ Ok(Arc::new(swapped.builder().reset_state().build()?))+ }+/// Correlated null-aware `LeftAnti` whose two scope keys are Float64, with/// `-0.0` on the build side and `0.0` on the probe side. Both scopes must/// still match in every direction: the full key lookup, NULL probe valuesdiff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rsindex 00506d9c6c..f7e5aba2cb 100644--- a/datafusion/physical-plan/src/joins/hash_join/stream.rs+++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs@@ -26,6 +26,7 @@ use std::task::Poll;use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};use crate::joins::Map;use crate::joins::PartitionMode;+use crate::joins::SharedBitmapBuilder;use crate::joins::hash_join::exec::{JoinLeftData, NullAwareMode, null_value_key_mask};use crate::joins::hash_join::probe_completion::ProbeSideSummary;use crate::joins::hash_join::shared_bounds::{@@ -48,9 +49,12 @@ use crate::{},};-use arrow::array::{Array, ArrayRef, AsArray, BooleanArray, UInt32Array, UInt64Array};+use arrow::array::{+ Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, UInt32Array,+ UInt64Array,+};use arrow::buffer::{BooleanBuffer, NullBuffer};-use arrow::compute::{filter, take};+use arrow::compute::{filter, filter_record_batch, take};use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::{@@ -201,6 +205,11 @@ pub(super) struct ProcessProbeBatchState {/// Key comparator for this batch, built on first use and reused by every/// chunkkey_comparator: Option<JoinKeyComparator>,+ /// For a correlated null-aware `RightAnti` join, the probe rows whose+ /// `NOT IN` is UNKNOWN because of a NULL (see+ /// `mark_null_candidates_for_probe_batch`); they are never emitted. `None`+ /// when there is no such row; boxed to keep this state small.+ unknown_probe_rows: Option<Box<BooleanBuffer>>,}impl ProcessProbeBatchState {@@ -832,6 +841,7 @@ impl HashJoinStream {joined_probe_idx: None,matched_probe_idx: None,key_comparator: None,+ unknown_probe_rows: None,});}Some(Err(err)) => return Poll::Ready(Err(err)),@@ -873,31 +883,69 @@ impl HashJoinStream {// Must precede the empty-build-map return below: an all-NULL-key// build side has an empty full-key map but still needs UNKNOWN marks.if mode.is_correlated() && state.is_first_chunk() {- mark_null_candidates_for_probe_batch(- build_side,- state,- mode.value_keys(),- self.filter.as_ref(),- self.join_type,- &self.random_state,- self.batch_size,- &mut self.null_mark_hashes_buffer,- &mut self.null_mark_probe_indices_buffer,- &mut self.null_mark_build_indices_buffer,- )?;+ let mut mark = |marks| {+ mark_null_candidates_for_probe_batch(+ build_side,+ state,+ mode.value_keys(),+ self.filter.as_ref(),+ self.join_type,+ marks,+ &self.random_state,+ self.batch_size,+ &mut self.null_mark_hashes_buffer,+ &mut self.null_mark_probe_indices_buffer,+ &mut self.null_mark_build_indices_buffer,+ )+ };+ match mode {+ // `RightAnti` decides the rows of this probe batch. Without+ // a NULL-valued row on either side none of them can be+ // UNKNOWN, so the bitmap is only allocated when one exists.+ NullAwareMode::RightAnti { value_keys, .. } => {+ let may_be_unknown =+ build_side.left_data.null_value_build_rows().is_some()+ || state.values[..value_keys]+ .iter()+ .any(|values| values.logical_null_count() > 0);+ if may_be_unknown {+ let num_rows = state.batch.num_rows();+ let mut unknown = BooleanBufferBuilder::new(num_rows);+ unknown.append_n(num_rows, false);+ let unknown = SharedBitmapBuilder::new(unknown);+ mark(UnknownMarks::Probe(&unknown))?;+ let unknown = unknown.into_inner().finish();+ state.unknown_probe_rows =+ (unknown.count_set_bits() > 0).then(|| Box::new(unknown));+ }+ }+ // The others decide the build rows shared by all probe+ // partitions.+ NullAwareMode::LeftAnti { .. } | NullAwareMode::LeftMark { .. } => {+ mark(UnknownMarks::Build(+ build_side.left_data.null_indices_bitmap(),+ ))?;+ }+ }}}let is_empty = !build_side.left_data.has_matchable_build_rows();if is_empty {- let result = build_batch_empty_build_side(+ let mut result = build_batch_empty_build_side(&self.schema,build_side.left_data.batch(),&state.batch,&self.column_indices,self.join_type,)?;+ if let Some(unknown) = &state.unknown_probe_rows {+ result = filter_record_batch(+ &result,+ &BooleanArray::new(!unknown.as_ref(), None),+ )?;+ }timer.done();self.output_buffer.push_batch(result)?;self.state = HashJoinStreamState::FetchProbeBatch;@@ -1019,8 +1067,22 @@ impl HashJoinStream {self.right_side_ordered,)?;- if self.null_aware == Some(NullAwareMode::RightAnti) {- right_indices = drop_null_probe_keys(state, right_indices);+ match self.null_aware {+ Some(NullAwareMode::RightAnti {+ correlated: false, ..+ }) => {+ right_indices = drop_null_probe_keys(state, right_indices);+ }+ Some(NullAwareMode::RightAnti {+ correlated: true, ..+ }) => {+ if let Some(unknown) = &state.unknown_probe_rows {+ right_indices = retain_probe_rows(&right_indices, |idx| {+ !unknown.value(idx as usize)+ });+ }+ }+ _ => {}}// Build output batch and push to coalescer@@ -1270,8 +1332,8 @@ fn next_final_indices_chunk(/// Applies the pre-lookup bookkeeping of a null-aware join to a probe batch/// and returns `true` if the batch cannot contribute any output row.///-/// Under SQL three-valued logic a NULL on either side of `NOT IN` makes the-/// predicate UNKNOWN for every row of the other side:+/// Under SQL three-valued logic a NULL on either side of an uncorrelated scalar+/// `NOT IN` makes the predicate UNKNOWN for every row of the other side:/// - `RightAnti` emits probe rows, so a NULL build key silences every batch./// - `LeftAnti` emits build rows in the final stage, so a NULL probe key seen/// by any partition silences everything; the shared flags carry this to the@@ -1284,10 +1346,15 @@ fn null_aware_skip_probe_batch(left_data: &JoinLeftData,) -> bool {match mode {- NullAwareMode::RightAnti => left_data.build_side_has_null,- // Correlated joins decide UNKNOWN per build row instead, in+ NullAwareMode::RightAnti {+ correlated: false, ..+ } => left_data.build_side_has_null,+ // Correlated joins decide UNKNOWN per row instead, in// `mark_null_candidates_for_probe_batch`.- NullAwareMode::LeftAnti {+ NullAwareMode::RightAnti {+ correlated: true, ..+ }+ | NullAwareMode::LeftAnti {correlated: true, ..}| NullAwareMode::LeftMark {@@ -1331,17 +1398,27 @@ fn drop_null_probe_keys(.or_else(|| probe_key.logical_nulls());// we only need this copy if there are NULLsmatch mask.filter(|m| m.null_count() > 0) {- Some(mask) => UInt32Array::from_iter_values(- right_indices- .values()- .iter()- .copied()- .filter(|idx| !mask.is_null(*idx as usize)),- ),+ Some(mask) => {+ retain_probe_rows(&right_indices, |idx| !mask.is_null(idx as usize))+ }None => right_indices,}}+/// Keeps the probe rows of `right_indices` for which `keep` returns true.+fn retain_probe_rows(+ right_indices: &UInt32Array,+ keep: impl Fn(u32) -> bool,+) -> UInt32Array {+ UInt32Array::from_iter_values(+ right_indices+ .values()+ .iter()+ .copied()+ .filter(|idx| keep(*idx)),+ )+}+/// Final-stage rules of a null-aware `LeftAnti` join, evaluated by the last/// probe partition.///@@ -1421,21 +1498,53 @@ fn null_aware_left_mark_column()}-/// Records which build rows of a correlated null-aware join are UNKNOWN-/// candidates for this probe batch.+/// Where a correlated null-aware join records the rows whose `NOT IN` is+/// UNKNOWN, see [`mark_null_candidates_for_probe_batch`].+#[derive(Clone, Copy)]+enum UnknownMarks<'a> {+ /// `LeftAnti` and `LeftMark` mark build rows, in the null-indices bitmap+ /// shared by all probe partitions.+ Build(&'a SharedBitmapBuilder),+ /// `RightAnti` marks the rows of the current probe batch.+ Probe(&'a SharedBitmapBuilder),+}++impl<'a> UnknownMarks<'a> {+ fn bitmap(self) -> &'a SharedBitmapBuilder {+ match self {+ Self::Build(bitmap) | Self::Probe(bitmap) => bitmap,+ }+ }++ /// The row of a (build, probe) candidate pair whose `NOT IN` the pair+ /// decides, and that is marked when the pair makes it UNKNOWN.+ fn unknown_row(self, build_idx: u64, probe_idx: u32) -> usize {+ match self {+ Self::Build(_) => build_idx as usize,+ Self::Probe(_) => probe_idx as usize,+ }+ }+}++/// Records which rows of a correlated null-aware join are UNKNOWN candidates+/// for this probe batch: the build rows for `LeftAnti` and `LeftMark`, the+/// probe rows for `RightAnti` (see [`UnknownMarks`]).////// Key layout: `on[..V]` are the `NOT IN` value keys, `on[V..]` the (possibly/// empty) correlation scope keys (see `HashJoinExec::null_aware`). A row is-/// NULL-valued when any of its value keys is NULL. An unmatched build row's-/// `NOT IN` is UNKNOWN instead of TRUE (its mark is NULL instead of FALSE)-/// when either:-/// 1. it is NULL-valued and a probe row in its correlation scope passes the-/// join filter, or-/// 2. a NULL-valued probe row in its correlation scope passes the join filter,+/// NULL-valued when any of its value keys is NULL. Each (build, probe) pair+/// decides one of its rows (see [`UnknownMarks::unknown_row`]), and makes that+/// row's `NOT IN` UNKNOWN instead of TRUE (a `LeftMark` mark NULL instead of+/// FALSE) when either:+/// 1. the build row is NULL-valued, or+/// 2. the probe row is NULL-valued,///-/// and, for a multi-column value key, the two rows' value tuples are not a-/// definite mismatch: every element pair is equal or involves a NULL. With a-/// single value key a NULL on either side already rules out a mismatch.+/// the two rows are in the same correlation scope, the pair passes the join+/// filter, and, for a multi-column value key, the two rows' value tuples are+/// not a definite mismatch: every element pair is equal or involves a NULL.+/// With a single value key a NULL on either side already rules out a mismatch.+/// An exact match, which makes `NOT IN` FALSE, is found by the main hash lookup+/// instead and takes precedence over these marks.////// Case 1 pairs the NULL-valued build rows with all probe rows; case 2 pairs/// all build rows with the NULL-valued probe rows. Scope keys narrow these@@ -1443,10 +1552,10 @@ fn null_aware_left_mark_column(/// The value tuples and then the join filter, if any, decide which candidates/// count.///-/// A build row stays UNKNOWN once it is marked, so candidates whose build row-/// is already marked are skipped, and the join filter is not evaluated for-/// them. Without scope keys this also ends the pairing as soon as no unmarked-/// build row is left.+/// A row stays UNKNOWN once it is marked, so pairs that decide a row already+/// known to be UNKNOWN are skipped, and the join filter is not evaluated for+/// them. Without scope keys this also ends the pairing as soon as every row+/// that can be decided is UNKNOWN.#[expect(clippy::too_many_arguments)]fn mark_null_candidates_for_probe_batch(build_side: &BuildSideReadyState,@@ -1454,6 +1563,7 @@ fn mark_null_candidates_for_probe_batch(num_value_keys: usize,filter: Option<&JoinFilter>,join_type: JoinType,+ marks: UnknownMarks<'_>,random_state: &RandomState,batch_size: usize,hashes_buffer: &mut Vec<u64>,@@ -1477,10 +1587,10 @@ fn mark_null_candidates_for_probe_batch(let (probe_value_keys, probe_scope_values) = state.values.split_at(num_value_keys);// Keeps the candidate pairs whose value tuples are not a definite mismatch- // and that pass the join filter, and marks their build rows as UNKNOWN.+ // and that pass the join filter, and marks their rows as UNKNOWN.let mut mark = |build_indices: UInt64Array, probe_indices: UInt32Array| {let (build_indices, probe_indices) =- retain_unmarked(left_data, build_indices, probe_indices);+ retain_unmarked(marks, build_indices, probe_indices);let (build_indices, probe_indices) = if num_value_keys > 1 {retain_value_mismatch_free(build_value_keys,@@ -1494,33 +1604,32 @@ fn mark_null_candidates_for_probe_batch(if build_indices.is_empty() {return Ok(());}- let build_indices = match filter {- Some(filter) => {- apply_join_filter_to_indices(- left_data.batch(),- &state.batch,- build_indices,- probe_indices,- filter,- JoinSide::Left,- None,- join_type,- )?- .0- }- None => build_indices,+ let (build_indices, probe_indices) = match filter {+ Some(filter) => apply_join_filter_to_indices(+ left_data.batch(),+ &state.batch,+ build_indices,+ probe_indices,+ filter,+ JoinSide::Left,+ None,+ join_type,+ )?,+ None => (build_indices, probe_indices),};if !build_indices.is_empty() {- let mut null_bitmap = left_data.null_indices_bitmap().lock();- for build_idx in build_indices.values() {- null_bitmap.set_bit(*build_idx as usize, true);+ let mut bitmap = marks.bitmap().lock();+ for (build_idx, probe_idx) in+ build_indices.values().iter().zip(probe_indices.values())+ {+ bitmap.set_bit(marks.unknown_row(*build_idx, *probe_idx), true);}}Ok(())};- // Case 1: NULL-valued build rows are UNKNOWN as soon as a probe row in- // their correlation scope passes the checks in `mark`.+ // Case 1: NULL-valued build rows paired with the probe rows in their+ // correlation scope.if let Some(null_rows) = null_value_build_rows {match &null_rows.scope_map {Some(scope_map) => {@@ -1551,7 +1660,7 @@ fn mark_null_candidates_for_probe_batch(}None => {for_each_unmarked_cross_product(- left_data,+ marks,null_rows.build_indices.values().iter().copied(),0..state.batch.num_rows() as u32,batch_size,@@ -1561,8 +1670,8 @@ fn mark_null_candidates_for_probe_batch(}}- // Case 2: NULL-valued probe rows make every build row in their correlation- // scope that passes the checks in `mark` UNKNOWN.+ // Case 2: NULL-valued probe rows paired with the build rows in their+ // correlation scope.if let Some(null_mask) = probe_null_value_mask {let null_probe_rows = UInt32Array::from_iter_values(null_mask.values().set_indices().map(|i| i as u32),@@ -1604,7 +1713,7 @@ fn mark_null_candidates_for_probe_batch(}None => {for_each_unmarked_cross_product(- left_data,+ marks,0..left_data.batch().num_rows() as u64,null_probe_rows.values().iter().copied(),batch_size,@@ -1715,75 +1824,108 @@ fn for_each_scope_match(Ok(())}-/// Removes the candidate pairs whose build row is already marked UNKNOWN.+/// Removes the candidate pairs that decide a row already marked UNKNOWN.fn retain_unmarked(- left_data: &JoinLeftData,+ marks: UnknownMarks<'_>,build_indices: UInt64Array,probe_indices: UInt32Array,) -> (UInt64Array, UInt32Array) {- let bitmap = left_data.null_indices_bitmap().lock();- let is_unmarked = |build_idx: &u64| !bitmap.get_bit(*build_idx as usize);- if build_indices.values().iter().all(is_unmarked) {+ let bitmap = marks.bitmap().lock();+ let is_unmarked = |(build_idx, probe_idx): (&u64, &u32)| {+ !bitmap.get_bit(marks.unknown_row(*build_idx, *probe_idx))+ };+ if build_indices+ .values()+ .iter()+ .zip(probe_indices.values().iter())+ .all(is_unmarked)+ {return (build_indices, probe_indices);}let (build, probe): (Vec<u64>, Vec<u32>) = build_indices.values().iter().zip(probe_indices.values().iter())- .filter(|(build_idx, _)| is_unmarked(build_idx))+ .filter(|pair| is_unmarked(*pair)).unzip();(build.into(), probe.into())}-/// Calls `f` with the pairs of `build_rows` x `probe_rows` whose build row is-/// not marked UNKNOWN, as chunks of at most `batch_size` pairs.-///-/// `f` marks build rows, so the unmarked build rows are found again after each-/// chunk. The pairing stops when no unmarked build row is left.+/// Calls `f` with the pairs of `build_rows` x `probe_rows` that decide a row+/// (see [`UnknownMarks::unknown_row`]) not yet marked UNKNOWN, as chunks of at+/// most `batch_size` pairs.fn for_each_unmarked_cross_product(- left_data: &JoinLeftData,+ marks: UnknownMarks<'_>,build_rows: impl Iterator<Item = u64>,probe_rows: impl Iterator<Item = u32>,batch_size: usize,mut f: impl FnMut(UInt64Array, UInt32Array) -> Result<()>,) -> Result<()> {- let retain_unmarked_rows = |rows: &mut Vec<u64>| {- let bitmap = left_data.null_indices_bitmap().lock();- rows.retain(|idx| !bitmap.get_bit(*idx as usize));+ match marks {+ UnknownMarks::Build(bitmap) => for_each_unmarked_pair(+ bitmap,+ build_rows.collect(),+ probe_rows,+ batch_size,+ |build, probe| f(build.into(), probe.into()),+ ),+ UnknownMarks::Probe(bitmap) => for_each_unmarked_pair(+ bitmap,+ probe_rows.collect(),+ build_rows,+ batch_size,+ |probe, build| f(build.into(), probe.into()),+ ),+ }+}++/// Calls `f` with the pairs of `decided_rows` x `other_rows` whose row in+/// `decided_rows` is not set in `bitmap`, as chunks of at most `batch_size`+/// pairs.+///+/// `f` sets bits in `bitmap`, so the unset rows are found again after each+/// chunk. The pairing stops when every row of `decided_rows` is set.+fn for_each_unmarked_pair<D: Copy + Into<u64>, O: Copy>(+ bitmap: &SharedBitmapBuilder,+ mut decided_rows: Vec<D>,+ other_rows: impl Iterator<Item = O>,+ batch_size: usize,+ mut f: impl FnMut(Vec<D>, Vec<O>) -> Result<()>,+) -> Result<()> {+ let retain_undecided_rows = |rows: &mut Vec<D>| {+ let bitmap = bitmap.lock();+ rows.retain(|idx| !bitmap.get_bit((*idx).into() as usize));};- let mut build_rows: Vec<u64> = build_rows.collect();- retain_unmarked_rows(&mut build_rows);+ retain_undecided_rows(&mut decided_rows);- let mut build_chunk = Vec::with_capacity(batch_size);- let mut probe_chunk = Vec::with_capacity(batch_size);+ let mut decided_chunk = Vec::with_capacity(batch_size);+ let mut other_chunk = Vec::with_capacity(batch_size);let mut marks_since_refresh = false;- for probe_row in probe_rows {+ for other_row in other_rows {// Refresh only after a chunk was sent, so the cost of the refresh// stays proportional to the pairs already evaluated.if marks_since_refresh {- retain_unmarked_rows(&mut build_rows);+ retain_undecided_rows(&mut decided_rows);marks_since_refresh = false;}- if build_rows.is_empty() {+ if decided_rows.is_empty() {break;}- for build_row in &build_rows {- build_chunk.push(*build_row);- probe_chunk.push(probe_row);- if build_chunk.len() == batch_size {+ for decided_row in &decided_rows {+ decided_chunk.push(*decided_row);+ other_chunk.push(other_row);+ if decided_chunk.len() == batch_size {f(- std::mem::replace(&mut build_chunk, Vec::with_capacity(batch_size))- .into(),- std::mem::replace(&mut probe_chunk, Vec::with_capacity(batch_size))- .into(),+ std::mem::replace(&mut decided_chunk, Vec::with_capacity(batch_size)),+ std::mem::replace(&mut other_chunk, Vec::with_capacity(batch_size)),)?;marks_since_refresh = true;}}}- if !build_chunk.is_empty() {- f(build_chunk.into(), probe_chunk.into())?;+ if !decided_chunk.is_empty() {+ f(decided_chunk, other_chunk)?;}Ok(())}diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex 916819fbb9..29159f2577 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -411,7 +411,9 @@ NULL 0# Results under a batch size sweep are in null_aware_multi_column.slt. A tuple# NOT IN is a null-aware join whose leading keys are the tuple elements-# (`value_keys`), followed by any correlation keys.+# (`value_keys`), followed by any correlation keys. Like the scalar NOT IN,+# JoinSelection swaps it into a null-aware RightAnti that builds on the+# subquery side when that side is not larger, keeping the key order.statement okCREATE TABLE multi_col_outer(k INT, a INT, b INT) AS VALUES@@ -436,7 +438,7 @@ logical_plan06)------SubqueryAlias: i07)--------TableScan: multi_col_inner projection=[k, x, y]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@1, x@0), (b@2, y@1), (k@0, k@2)], null_aware(value_keys=2)+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (y@1, b@2), (k@2, k@0)], null_aware(value_keys=2)02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]@@ -455,10 +457,10 @@ logical_plan06)------Projection: multi_col_inner.x, CAST(multi_col_inner.y AS Int64)07)--------TableScan: multi_col_inner projection=[x, y]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(a@1, x@0), (__correlated_sq_1_value_1@3, multi_col_inner.y@1)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)-02)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, 2 as __correlated_sq_1_value_1]+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(x@0, a@1), (multi_col_inner.y@1, __correlated_sq_1_value_1@3)], projection=[k@0, a@1, b@2], null_aware(value_keys=2)+02)--ProjectionExec: expr=[x@0 as x, CAST(y@1 AS Int64) as multi_col_inner.y]03)----DataSourceExec: partitions=1, partition_sizes=[1]-04)--ProjectionExec: expr=[x@0 as x, CAST(y@1 AS Int64) as multi_col_inner.y]+04)--ProjectionExec: expr=[k@0 as k, a@1 as a, b@2 as b, 2 as __correlated_sq_1_value_1]05)----DataSourceExec: partitions=1, partition_sizes=[1]# As a nullable value, the tuple NOT IN is a null-aware mark join.@@ -990,7 +992,8 @@ NULL 0# The shape must stay a single anti join that carries the correlation as a# filter. The alternative lowering materializes the mark of three joins, which-# is correct but far slower.+# is correct but far slower. JoinSelection may still swap it into a null-aware+# RightAnti, which applies the filter before deciding each outer row.query TTEXPLAIN SELECT id, g FROM nai_res_og WHERE nai_res_og.id NOT IN (SELECT i.id FROM nai_res_ig i WHERE nai_res_og.g > 0);----@@ -1001,7 +1004,7 @@ logical_plan04)----SubqueryAlias: i05)------TableScan: nai_res_ig projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], filter=g@0 > 0, null_aware+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)], filter=g@0 > 0, null_aware02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]diff --git a/datafusion/sqllogictest/test_files/null_aware_memory_limit.slt b/datafusion/sqllogictest/test_files/null_aware_memory_limit.sltnew file mode 100644index 0000000000..ec0f16f271--- /dev/null+++ b/datafusion/sqllogictest/test_files/null_aware_memory_limit.slt@@ -0,0 +1,67 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# A multi-column NOT IN over a small subquery builds its null-aware hash join+# on the subquery, so a large outer table runs under a memory limit that only+# fits the subquery side. The join used to stay a LeftAnti that builds on the+# outer side and ran out of memory where the scalar NOT IN ran.++statement ok+CREATE TABLE na_mem_big AS+SELECT v AS a, v % 1000 AS b FROM generate_series(1, 200000) AS t(v);++# A NULL-valued subquery row makes NOT IN UNKNOWN for every outer row with+# `b = 7`.+statement ok+CREATE TABLE na_mem_small AS+SELECT v AS a, v % 1000 AS b FROM generate_series(1, 100) AS t(v)+UNION ALL+SELECT NULL, 7;++# Far below the outer side's build state, far above the subquery's.+statement ok+SET datafusion.runtime.memory_limit = '2M'++# Rows 1..=100 match exactly; of the rest, those with `b = 7` are UNKNOWN: 199+# of the 200 values `v % 1000 = 7`, all but `v = 7`.+query I+SELECT count(*) FROM na_mem_big+WHERE (a, b) NOT IN (SELECT a, b FROM na_mem_small);+----+199701++# Building on the outer side does not fit, which shows the limit binds.+statement ok+SET datafusion.optimizer.join_reordering = false++query error Resources exhausted+SELECT count(*) FROM na_mem_big+WHERE (a, b) NOT IN (SELECT a, b FROM na_mem_small);++statement ok+RESET datafusion.optimizer.join_reordering++statement ok+DROP TABLE na_mem_big;++statement ok+DROP TABLE na_mem_small;++# Setting a runtime option rebuilds the session state from the existing one,+# which turns this option off because the default catalog already exists.+statement ok+RESET datafusion.catalog.create_default_catalog_and_schemadiff --git a/datafusion/sqllogictest/test_files/null_aware_multi_column.slt b/datafusion/sqllogictest/test_files/null_aware_multi_column.sltindex 917d1caec9..ebf33f1991 100644--- a/datafusion/sqllogictest/test_files/null_aware_multi_column.slt+++ b/datafusion/sqllogictest/test_files/null_aware_multi_column.slt@@ -20,11 +20,16 @@# mismatch and at least one of them involves a NULL: `(NULL, 8) = (1, 2)` is# `UNKNOWN AND FALSE`, which is FALSE, while `(NULL, 2) = (1, 2)` is UNKNOWN.# The batch size sweep exercises the chunked UNKNOWN candidate pairing of the-# null-aware hash join. Plans are checked in null_aware_anti_join.slt.+# null-aware hash join. With join reordering on, JoinSelection swaps a NOT IN+# whose subquery is not larger into a null-aware RightAnti that builds on the+# subquery and decides each outer row as it streams; with it off, the join+# stays a LeftAnti that builds on the outer side and decides per build row.+# Plans are checked in null_aware_anti_join.slt.# Matrix rules: no EXPLAIN, no in-file SET of a swept knob, rowsort every# multi-row query.# configMatrix: datafusion.execution.batch_size=1,2,100,8192+# configMatrix: datafusion.optimizer.join_reordering=true,falsestatement okCREATE TABLE mc_outer(a INT, b INT, value TEXT) AS VALUES@@ -47,6 +52,11 @@ CREATE TABLE mc_inner_no_null(x INT, y INT) AS VALUESstatement okCREATE TABLE mc_empty(x INT, y INT);+statement ok+CREATE TABLE mc_inner_all_null(x INT, y INT) AS VALUES+(NULL, 4),+(1, NULL);+############### NULL in the subquery#############@@ -61,6 +71,15 @@ WHERE (a, b) NOT IN (SELECT x, y FROM mc_inner_null);5 6 zNULL 8 w+# Every subquery row holds a NULL, so no row can match a whole tuple and only+# the NULL-aware checks decide. Only (5, 6) has a definite mismatch with both+# (NULL, 4) and (1, NULL).+query IIT rowsort+SELECT * FROM mc_outer+WHERE (a, b) NOT IN (SELECT x, y FROM mc_inner_all_null);+----+5 6 z+############### NULL only in the outer tuple#############@@ -543,6 +562,9 @@ DROP TABLE mc_inner_no_null;statement okDROP TABLE mc_empty;+statement ok+DROP TABLE mc_inner_all_null;+statement okDROP TABLE mc3_outer;diff --git a/datafusion/sqllogictest/test_files/subquery_projection.slt b/datafusion/sqllogictest/test_files/subquery_projection.sltindex f86de505fe..f2845c3878 100644--- a/datafusion/sqllogictest/test_files/subquery_projection.slt+++ b/datafusion/sqllogictest/test_files/subquery_projection.slt@@ -403,8 +403,9 @@ statement okDROP TABLE t2;# A non-equality correlation leaves one key and a residual join filter. The-# null-aware `LeftAnti` hash join applies the residual filter when it looks-# for UNKNOWN rows. For `k = 1` the key is NULL and the correlated subquery+# null-aware anti hash join (here swapped to a `RightAnti` that builds on the+# smaller subquery) applies the residual filter when it looks for UNKNOWN+# rows. For `k = 1` the key is NULL and the correlated subquery# result is empty, and `NULL NOT IN (<empty set>)` is TRUE. Both rows are# correct.statement ok@@ -438,10 +439,10 @@ logical_plan05)------Projection: CAST(rb.k AS Int64), rb.z06)--------TableScan: rb projection=[k, z]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(nullif(ra.k,Int64(1))@2, rb.k@0)], filter=z@1 > z@0, projection=[k@0], null_aware-02)--ProjectionExec: expr=[k@0 as k, z@1 as z, nullif(CAST(k@0 AS Int64), 1) as nullif(ra.k,Int64(1))]+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(rb.k@0, nullif(ra.k,Int64(1))@2)], filter=z@1 > z@0, projection=[k@0], null_aware+02)--ProjectionExec: expr=[CAST(k@0 AS Int64) as rb.k, z@1 as z]03)----DataSourceExec: partitions=1, partition_sizes=[1]-04)--ProjectionExec: expr=[CAST(k@0 AS Int64) as rb.k, z@1 as z]+04)--ProjectionExec: expr=[k@0 as k, z@1 as z, nullif(CAST(k@0 AS Int64), 1) as nullif(ra.k,Int64(1))]05)----DataSourceExec: partitions=1, partition_sizes=[1]statement ok