Apache DataFusion SQL Query Engine
fix: Avoid unnecessary input repartitioning with `ScalarSubqueryExec` (#21986)
Neil Conway80108570e71a
diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsindex b6ad7f91f0..82421d66de 100644--- a/datafusion/physical-plan/src/scalar_subquery.rs+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -239,9 +239,9 @@ impl ExecutionPlan for ScalarSubqueryExec {}fn benefits_from_input_partitioning(&self) -> Vec<bool> {- // Only the main input; subquery children produce at most one row, so- // repartitioning them has no benefit.- self.true_for_input_only()+ // ScalarSubqueryExec is a pass-through coordinator: it does not+ // benefit from repartitioning any child directly below it.+ vec![false; self.subqueries.len() + 1]}fn partition_statistics(&self, partition: Option<usize>) -> Result<Arc<Statistics>> {diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex e70f24303e..2ee21e269a 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -1824,6 +1824,56 @@ SELECT x FROM sq_main ORDER BY x + (SELECT max(v) FROM sq_values);1020+# ScalarSubqueryExec should not add an extra repartition above an ordered main+# input. The main child should be SortPreservingMergeExec directly.+query TT+EXPLAIN SELECT i+FROM generate_series(1, 10) AS t(i)+WHERE i > (+ SELECT avg(j) FROM generate_series(1, 10) AS u(j)+)+ORDER BY i;+----+logical_plan+01)Sort: t.i ASC NULLS LAST+02)--SubqueryAlias: t+03)----Projection: generate_series().value AS i+04)------Filter: CAST(generate_series().value AS Float64) > (<subquery>)+05)--------Subquery:+06)----------Aggregate: groupBy=[[]], aggr=[[avg(CAST(u.j AS Float64))]]+07)------------SubqueryAlias: u+08)--------------Projection: generate_series().value AS j+09)----------------TableScan: generate_series() projection=[value]+10)--------TableScan: generate_series() projection=[value]+physical_plan+01)ScalarSubqueryExec: subqueries=1+02)--SortPreservingMergeExec: [i@0 ASC NULLS LAST]+03)----SortExec: expr=[i@0 ASC NULLS LAST], preserve_partitioning=[true]+04)------ProjectionExec: expr=[value@0 as i]+05)--------FilterExec: CAST(value@0 AS Float64) > scalar_subquery(<pending>)+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=10, batch_size=8192]+08)--AggregateExec: mode=Final, gby=[], aggr=[avg(u.j)]+09)----CoalescePartitionsExec+10)------AggregateExec: mode=Partial, gby=[], aggr=[avg(u.j)]+11)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+12)----------ProjectionExec: expr=[value@0 as j]+13)------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=10, batch_size=8192]++query I+SELECT i+FROM generate_series(1, 10) AS t(i)+WHERE i > (+ SELECT avg(j) FROM generate_series(1, 10) AS u(j)+)+ORDER BY i;+----+6+7+8+9+10+# Aggregate function argument containing uncorrelated scalar subqueryquery ISELECT sum(x + (SELECT max(v) FROM sq_values)) AS s FROM sq_main;diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.partindex 0c5b6d76dc..e8a224867d 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part@@ -74,37 +74,36 @@ logical_plan25)----------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY")]physical_plan01)ScalarSubqueryExec: subqueries=1-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-03)----SortPreservingMergeExec: [value@1 DESC], fetch=10-04)------SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[true]-05)--------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]-06)----------FilterExec: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) > scalar_subquery(<pending>)-07)------------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-08)--------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4-09)----------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-10)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]-11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4-12)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]-13)------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4-14)--------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false-15)------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-16)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false-17)--------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4-18)----------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]-19)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-20)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false-21)--ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]-22)----AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-23)------CoalescePartitionsExec-24)--------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]-25)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]-26)------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4-27)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]-28)----------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4-29)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false-30)----------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-31)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false-32)------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4-33)--------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]-34)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-35)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false+02)--SortPreservingMergeExec: [value@1 DESC], fetch=10+03)----SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[true]+04)------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]+05)--------FilterExec: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) > scalar_subquery(<pending>)+06)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4+08)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[ps_partkey@0, ps_availqty@1, ps_supplycost@2]+10)------------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=4+11)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_availqty@2, ps_supplycost@3, s_nationkey@5]+12)----------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4+13)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false+14)----------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+15)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false+16)------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4+17)--------------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]+18)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+19)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=false+20)--ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)]+21)----AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+22)------CoalescePartitionsExec+23)--------AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]+24)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_availqty@0, ps_supplycost@1]+25)------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4+26)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@0, s_suppkey@0)], projection=[ps_availqty@1, ps_supplycost@2, s_nationkey@4]+27)----------------RepartitionExec: partitioning=Hash([ps_suppkey@0], 4), input_partitions=4+28)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=csv, has_header=false+29)----------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+30)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false+31)------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=4+32)--------------FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0]+33)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+34)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.partindex 3e1aca318b..3de29211bc 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q15.slt.part@@ -71,25 +71,24 @@ logical_plan19)------------------TableScan: lineitem projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("1996-01-01"), lineitem.l_shipdate < Date32("1996-04-01")]physical_plan01)ScalarSubqueryExec: subqueries=1-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-03)----SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST]-04)------SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true]-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5]-06)----------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=csv, has_header=false-08)----------ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]-09)------------FilterExec: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 = scalar_subquery(<pending>)-10)--------------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-11)----------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4-12)------------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-13)--------------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]-14)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false-15)--AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)]-16)----CoalescePartitionsExec-17)------AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)]-18)--------ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]-19)----------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-20)------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4-21)--------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]-22)----------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]-23)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false+02)--SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST]+03)----SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true]+04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5]+05)--------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=csv, has_header=false+07)--------ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]+08)----------FilterExec: sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 = scalar_subquery(<pending>)+09)------------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+10)--------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4+11)----------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+12)------------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]+13)--------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=false+14)--AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)]+15)----CoalescePartitionsExec+16)------AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)]+17)--------ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue]+18)----------AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+19)------------RepartitionExec: partitioning=Hash([l_suppkey@0], 4), input_partitions=4+20)--------------AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]+21)----------------FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2]+22)------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.partindex 3240cbfb69..3e9472e4a8 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part@@ -73,24 +73,23 @@ logical_plan15)--------------TableScan: orders projection=[o_custkey]physical_plan01)ScalarSubqueryExec: subqueries=1-02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true-03)----SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]-04)------SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]-05)--------ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]-06)----------AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]-07)------------RepartitionExec: partitioning=Hash([cntrycode@0], 4), input_partitions=4-08)--------------AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]-09)----------------ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal]-10)------------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2]-11)--------------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4-12)----------------------FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) AND CAST(c_acctbal@2 AS Decimal128(19, 6)) > scalar_subquery(<pending>)-13)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-14)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_custkey, c_phone, c_acctbal], file_type=csv, has_header=false-15)--------------------RepartitionExec: partitioning=Hash([o_custkey@0], 4), input_partitions=4-16)----------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:0..4223281], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:4223281..8446562], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:8446562..12669843], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:12669843..16893122]]}, projection=[o_custkey], file_type=csv, has_header=false-17)--AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)]-18)----CoalescePartitionsExec-19)------AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)]-20)--------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1]-21)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-22)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_phone, c_acctbal], file_type=csv, has_header=false+02)--SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]+03)----SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]+04)------ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]+05)--------AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]+06)----------RepartitionExec: partitioning=Hash([cntrycode@0], 4), input_partitions=4+07)------------AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]+08)--------------ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal]+09)----------------HashJoinExec: mode=Partitioned, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2]+10)------------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4+11)--------------------FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) AND CAST(c_acctbal@2 AS Decimal128(19, 6)) > scalar_subquery(<pending>)+12)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+13)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_custkey, c_phone, c_acctbal], file_type=csv, has_header=false+14)------------------RepartitionExec: partitioning=Hash([o_custkey@0], 4), input_partitions=4+15)--------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:0..4223281], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:4223281..8446562], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:8446562..12669843], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/orders.tbl:12669843..16893122]]}, projection=[o_custkey], file_type=csv, has_header=false+16)--AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)]+17)----CoalescePartitionsExec+18)------AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)]+19)--------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1]+20)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+21)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_phone, c_acctbal], file_type=csv, has_header=false