Apache DataFusion SQL Query Engine
Include `null_aware` status in the relevant Join node display implementations (#22913)
Adam Gutglick127731b11487
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex 3608c81878..8dbf41c37f 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -2073,6 +2073,7 @@ impl LogicalPlan {filter,join_constraint,join_type,+ null_aware,..}) => {let join_expr: Vec<String> =@@ -2081,6 +2082,8 @@ impl LogicalPlan {.as_ref().map(|expr| format!(" Filter: {expr}")).unwrap_or_else(|| "".to_string());+ let null_aware_expr =+ if *null_aware { " null_aware" } else { "" };let join_type = if filter.is_none()&& keys.is_empty()&& *join_type == JoinType::Inner@@ -2100,15 +2103,17 @@ impl LogicalPlan {filter_expr)?;}+ write!(f, "{null_aware_expr}")?;Ok(())}JoinConstraint::Using => {write!(f,- "{} Join: Using {}{}",+ "{} Join: Using {}{}{}",join_type,join_expr.join(", "),filter_expr,+ null_aware_expr,)}}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 7cddae276f..6e73c4d2e0 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1140,6 +1140,8 @@ impl DisplayAs for HashJoinExec {let display_fetch = self.fetch.map_or_else(String::new, |f| format!(", fetch={f}"));+ let display_null_aware =+ if self.null_aware { ", null_aware" } else { "" };let on = self.on.iter()@@ -1148,7 +1150,7 @@ impl DisplayAs for HashJoinExec {.join(", ");write!(f,- "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}",+ "HashJoinExec: mode={:?}, join_type={:?}, on=[{}]{}{}{}{}{}",self.mode,self.join_type,on,@@ -1156,6 +1158,7 @@ impl DisplayAs for HashJoinExec {display_projections,display_null_equality,display_fetch,+ display_null_aware,)}DisplayFormatType::TreeRender => {@@ -1178,6 +1181,10 @@ impl DisplayAs for HashJoinExec {writeln!(f, "NullsEqual: true")?;}+ if self.null_aware {+ writeln!(f, "null_aware")?;+ }+if let Some(filter) = self.filter.as_ref() {writeln!(f, "filter={filter}")?;}diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex e779ce2cbf..e436ca7952 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -374,14 +374,14 @@ FROM left_parquet lWHERE l.id NOT IN (SELECT r.id FROM right_parquet r);----logical_plan-01)LeftAnti Join: l.id = __correlated_sq_1.id+01)LeftAnti Join: l.id = __correlated_sq_1.id null_aware02)--SubqueryAlias: l03)----TableScan: left_parquet projection=[id, data]04)--SubqueryAlias: __correlated_sq_105)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ]@@ -469,7 +469,7 @@ ORDER BY l.id LIMIT 2;----logical_plan01)Sort: l.id ASC NULLS LAST, fetch=2-02)--LeftAnti Join: l.id = __correlated_sq_1.id+02)--LeftAnti Join: l.id = __correlated_sq_1.id null_aware03)----SubqueryAlias: l04)------TableScan: left_parquet projection=[id, data]05)----SubqueryAlias: __correlated_sq_1@@ -477,7 +477,7 @@ logical_plan07)--------TableScan: right_parquet projection=[id]physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]+02)--HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ]diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.sltindex d8e90e294f..8588c0e7ba 100644--- a/datafusion/sqllogictest/test_files/explain_tree.slt+++ b/datafusion/sqllogictest/test_files/explain_tree.slt@@ -1100,6 +1100,31 @@ physical_plan24)-----------------------------│ format: csv │25)-----------------------------└───────────────────────────┘+# Query with null-aware anti join (NOT IN subquery).+query TT+explain select int_col from table1 where int_col not in (select int_col from table2);+----+physical_plan+01)┌───────────────────────────┐+02)│ HashJoinExec │+03)│ -------------------- │+04)│ join_type: LeftAnti │+05)│ │+06)│ null_aware ├──────────────┐+07)│ │ │+08)│ on: │ │+09)│ (int_col = int_col) │ │+10)└─────────────┬─────────────┘ │+11)┌─────────────┴─────────────┐┌─────────────┴─────────────┐+12)│ DataSourceExec ││ DataSourceExec │+13)│ -------------------- ││ -------------------- │+14)│ files: 1 ││ files: 1 │+15)│ format: csv ││ format: parquet │+16)│ ││ │+17)│ ││ predicate: │+18)│ ││ DynamicFilter [ empty ] │+19)└───────────────────────────┘└───────────────────────────┘+# Query with nested loop join.query TTexplain select int_col from table1 where exists (select count(*) from table2);diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex 9be1d39d63..082b101672 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -1989,7 +1989,7 @@ where join_t1.t1_id + 12 not in(select join_t2.t2_id + 1 from join_t2 where join_t1.t1_int > 0)----logical_plan-01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0)+01)LeftAnti Join: CAST(join_t1.t1_id AS Int64) + Int64(12) = __correlated_sq_1.join_t2.t2_id + Int64(1) Filter: join_t1.t1_int > UInt32(0) null_aware02)--TableScan: join_t1 projection=[t1_id, t1_name, t1_int]03)--SubqueryAlias: __correlated_sq_104)----Projection: CAST(join_t2.t2_id AS Int64) + Int64(1)diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.sltindex 5907a85a9b..b18f3b3ae7 100644--- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt+++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt@@ -53,12 +53,12 @@ query TTEXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_no_null);----logical_plan-01)LeftAnti Join: outer_table.id = __correlated_sq_1.id+01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware02)--TableScan: outer_table projection=[id, value]03)--SubqueryAlias: __correlated_sq_104)----TableScan: inner_table_no_null projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]@@ -193,12 +193,12 @@ query TTEXPLAIN SELECT * FROM outer_table WHERE id NOT IN (SELECT id FROM inner_table_with_null);----logical_plan-01)LeftAnti Join: outer_table.id = __correlated_sq_1.id+01)LeftAnti Join: outer_table.id = __correlated_sq_1.id null_aware02)--TableScan: outer_table projection=[id, value]03)--SubqueryAlias: __correlated_sq_104)----TableScan: inner_table_with_null projection=[id]physical_plan-01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)]+01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], null_aware02)--DataSourceExec: partitions=1, partition_sizes=[1]03)--DataSourceExec: partitions=1, partition_sizes=[1]diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.partindex 8d8eb0ed11..ab830714b1 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part@@ -54,7 +54,7 @@ logical_plan02)--Projection: part.p_brand, part.p_type, part.p_size, count(alias1) AS supplier_cnt03)----Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size]], aggr=[[count(alias1)]]04)------Aggregate: groupBy=[[part.p_brand, part.p_type, part.p_size, partsupp.ps_suppkey AS alias1]], aggr=[[]]-05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey+05)--------LeftAnti Join: partsupp.ps_suppkey = __correlated_sq_1.s_suppkey null_aware06)----------Projection: partsupp.ps_suppkey, part.p_brand, part.p_type, part.p_size07)------------Inner Join: partsupp.ps_partkey = part.p_partkey08)--------------TableScan: partsupp projection=[ps_partkey, ps_suppkey]@@ -74,7 +74,7 @@ physical_plan07)------------AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[]08)--------------RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 4), input_partitions=409)----------------AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[]-10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)]+10)------------------HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ps_suppkey@0, s_suppkey@0)], null_aware11)--------------------CoalescePartitionsExec12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_partkey@0, p_partkey@0)], projection=[ps_suppkey@1, p_brand@3, p_type@4, p_size@5]13)------------------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4