Skip to content

Apache DataFusion SQL Query Engine

Push sort requirements through simple projections (#23288)

Egor Markov8139397c8a46
diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rs
index f85538b5c3..9b350e5529 100644
--- a/datafusion/core/tests/core_integration.rs
+++ b/datafusion/core/tests/core_integration.rs
@@ -63,6 +63,9 @@ mod tracing;
/// Run all tests that are found in the `extension_types` directory
mod extension_types;
+/// Helper functions for tests.
+mod helper;
+
#[cfg(test)]
#[ctor::ctor(unsafe)]
fn init() {
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 96ffdc9d94..db26413ac9 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -3007,22 +3007,22 @@ async fn test_count_wildcard_on_sort() -> Result<()> {
assert_snapshot!(
pretty_format_batches(&sql_results).unwrap(),
@r"
- +---------------+------------------------------------------------------------------------------------+
- | plan_type | plan |
- +---------------+------------------------------------------------------------------------------------+
- | logical_plan | Sort: count(*) ASC NULLS LAST |
- | | Projection: t1.b, count(Int64(1)) AS count(*) |
- | | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1))]] |
- | | TableScan: t1 projection=[b] |
- | physical_plan | SortPreservingMergeExec: [count(*)@1 ASC NULLS LAST] |
- | | SortExec: expr=[count(*)@1 ASC NULLS LAST], preserve_partitioning=[true] |
- | | ProjectionExec: expr=[b@0 as b, count(Int64(1))@1 as count(*)] |
- | | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[count(Int64(1))] |
- | | RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=1 |
- | | AggregateExec: mode=Partial, gby=[b@0 as b], aggr=[count(Int64(1))] |
- | | DataSourceExec: partitions=1, partition_sizes=[1] |
- | | |
- +---------------+------------------------------------------------------------------------------------+
+ +---------------+-------------------------------------------------------------------------------------+
+ | plan_type | plan |
+ +---------------+-------------------------------------------------------------------------------------+
+ | logical_plan | Sort: count(*) ASC NULLS LAST |
+ | | Projection: t1.b, count(Int64(1)) AS count(*) |
+ | | Aggregate: groupBy=[[t1.b]], aggr=[[count(Int64(1))]] |
+ | | TableScan: t1 projection=[b] |
+ | physical_plan | SortPreservingMergeExec: [count(*)@1 ASC NULLS LAST] |
+ | | ProjectionExec: expr=[b@0 as b, count(Int64(1))@1 as count(*)] |
+ | | SortExec: expr=[count(Int64(1))@1 ASC NULLS LAST], preserve_partitioning=[true] |
+ | | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[count(Int64(1))] |
+ | | RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=1 |
+ | | AggregateExec: mode=Partial, gby=[b@0 as b], aggr=[count(Int64(1))] |
+ | | DataSourceExec: partitions=1, partition_sizes=[1] |
+ | | |
+ +---------------+-------------------------------------------------------------------------------------+
"
);
diff --git a/datafusion/core/tests/helper/mod.rs b/datafusion/core/tests/helper/mod.rs
new file mode 100644
index 0000000000..809f8ca087
--- /dev/null
+++ b/datafusion/core/tests/helper/mod.rs
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared helpers for the `core_integration` test crate.
+//!
+//! Keep cross-cutting test utilities here when they are used by multiple test
+//! modules under `core/tests`. Placing them in this submodule avoids creating
+//! an additional Cargo integration test target for each helper file.
+pub(crate) mod plan_metrics;
diff --git a/datafusion/core/tests/helper/plan_metrics.rs b/datafusion/core/tests/helper/plan_metrics.rs
new file mode 100644
index 0000000000..12d3eaba1a
--- /dev/null
+++ b/datafusion/core/tests/helper/plan_metrics.rs
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Helpers for aggregating execution metrics across a physical plan tree.
+//!
+//! `ExecutionPlan::metrics()` returns metrics for a single plan node only; it
+//! does not include metrics from child operators. These helpers recursively walk
+//! the plan tree so tests can assert on metrics that may move between operators
+//! after optimizer rewrites, such as pushing a `SortExec` below a
+//! `ProjectionExec`.
+
+use datafusion_physical_plan::ExecutionPlan;
+
+/// Returns the total number of spill events recorded by `plan` and all of its
+/// descendants.
+///
+/// Missing `spill_count` metrics are treated as zero.
+pub fn plan_spill_count(plan: &dyn ExecutionPlan) -> usize {
+ let own = plan.metrics().and_then(|m| m.spill_count()).unwrap_or(0);
+
+ own + plan
+ .children()
+ .into_iter()
+ .map(|child| plan_spill_count(child.as_ref()))
+ .sum::<usize>()
+}
+
+/// Returns the total number of spilled bytes recorded by `plan` and all of its
+/// descendants.
+///
+/// Missing `spilled_bytes` metrics are treated as zero.
+pub fn plan_spilled_bytes(plan: &dyn ExecutionPlan) -> usize {
+ let own = plan.metrics().and_then(|m| m.spilled_bytes()).unwrap_or(0);
+
+ own + plan
+ .children()
+ .into_iter()
+ .map(|child| plan_spilled_bytes(child.as_ref()))
+ .sum::<usize>()
+}
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs
index ef9951addd..ebbe4312b1 100644
--- a/datafusion/core/tests/memory_limit/mod.rs
+++ b/datafusion/core/tests/memory_limit/mod.rs
@@ -62,6 +62,8 @@ use async_trait::async_trait;
use futures::StreamExt;
use tokio::fs::File;
+use crate::helper::plan_metrics::{plan_spill_count, plan_spilled_bytes};
+
#[cfg(test)]
#[ctor::ctor(unsafe)]
fn init() {
@@ -546,8 +548,7 @@ async fn test_external_sort_zero_merge_reservation() {
let _result = collect(stream).await;
// Ensures the query spilled during execution
- let metrics = physical_plan.metrics().unwrap();
- let spill_count = metrics.spill_count().unwrap();
+ let spill_count = plan_spill_count(physical_plan.as_ref());
assert!(spill_count > 0);
}
@@ -603,9 +604,8 @@ async fn test_sort_skewed_batches_spill() {
// The query must actually spill, otherwise it never reaches the merge path
// this test is meant to cover.
- let metrics = physical_plan.metrics().unwrap();
assert!(
- metrics.spill_count().unwrap() > 0,
+ plan_spill_count(physical_plan.as_ref()) > 0,
"expected the sort to spill to disk"
);
}
@@ -696,8 +696,8 @@ async fn test_disk_spill_limit_not_reached() -> Result<()> {
.await
.expect("Query execution failed");
- let spill_count = plan.metrics().unwrap().spill_count().unwrap();
- let spilled_bytes = plan.metrics().unwrap().spilled_bytes().unwrap();
+ let spill_count = plan_spill_count(plan.as_ref());
+ let spilled_bytes = plan_spilled_bytes(plan.as_ref());
println!("spill count {spill_count}, spill bytes {spilled_bytes}");
assert!(spill_count > 0);
@@ -732,8 +732,8 @@ async fn test_spill_file_compressed_with_zstd() -> Result<()> {
.await
.expect("Query execution failed");
- let spill_count = plan.metrics().unwrap().spill_count().unwrap();
- let spilled_bytes = plan.metrics().unwrap().spilled_bytes().unwrap();
+ let spill_count = plan_spill_count(plan.as_ref());
+ let spilled_bytes = plan_spilled_bytes(plan.as_ref());
println!("spill count {spill_count}");
assert!(spill_count > 0);
@@ -768,8 +768,8 @@ async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> {
.await
.expect("Query execution failed");
- let spill_count = plan.metrics().unwrap().spill_count().unwrap();
- let spilled_bytes = plan.metrics().unwrap().spilled_bytes().unwrap();
+ let spill_count = plan_spill_count(plan.as_ref());
+ let spilled_bytes = plan_spilled_bytes(plan.as_ref());
println!("spill count {spill_count}");
assert!(spill_count > 0);
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index cab8dc67f9..462807e436 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -1700,8 +1700,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1727,8 +1727,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1750,8 +1750,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1778,8 +1778,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1811,8 +1811,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1831,8 +1831,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1856,8 +1856,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1876,8 +1876,8 @@ fn multi_smj_joins() -> Result<()> {
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=1, maintains_sort_order=true
- SortExec: expr=[b1@1 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
@@ -1951,16 +1951,16 @@ fn smj_join_key_ordering() -> Result<()> {
let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);
assert_plan!(plan_distrib, @r"
SortMergeJoinExec: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]
- SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
- ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ SortExec: expr=[b1@0 ASC, a1@1 ASC], preserve_partitioning=[true]
AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]
RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10
AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]
RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
- SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ SortExec: expr=[b@0 ASC, a@1 ASC], preserve_partitioning=[true]
AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]
RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10
AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]
@@ -1972,16 +1972,16 @@ fn smj_join_key_ordering() -> Result<()> {
let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);
assert_plan!(plan_sort, @r"
SortMergeJoinExec: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]
- SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
- ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ SortExec: expr=[b1@0 ASC, a1@1 ASC], preserve_partitioning=[true]
AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]
RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10
AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]
RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
- SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ SortExec: expr=[b@0 ASC, a@1 ASC], preserve_partitioning=[true]
AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]
RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10
AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]
@@ -2599,8 +2599,8 @@ fn repartition_transitively_past_sort_with_projection() -> Result<()> {
let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
assert_plan!(plan_distrib,
@r"
- SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
");
// Since this projection is trivial, increasing parallelism is not beneficial
@@ -2674,8 +2674,8 @@ fn repartition_transitively_past_sort_with_projection_and_filter() -> Result<()>
assert_plan!(plan_distrib,
@r"
SortPreservingMergeExec: [a@0 ASC]
- SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
FilterExec: c@2 = 0
RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
@@ -2689,8 +2689,8 @@ fn repartition_transitively_past_sort_with_projection_and_filter() -> Result<()>
assert_plan!(plan_sort,
@r"
SortPreservingMergeExec: [a@0 ASC]
- SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
FilterExec: c@2 = 0
RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
index ecff2edbbe..8e8d222bb0 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs
@@ -2974,3 +2974,241 @@ async fn test_parallelize_sorts_remaps_index_through_reordering_projection() ->
Ok(())
}
+
+#[tokio::test]
+async fn test_push_sort_through_reordered_projection_to_union() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let ordering: LexOrdering = [sort_expr("a", &schema)].into();
+
+ let sorted_source = parquet_exec_with_sort(schema.clone(), vec![ordering.clone()]);
+ let unsorted_source = sort_exec(ordering.clone(), parquet_exec(schema.clone()));
+ let union = union_exec(vec![sorted_source, unsorted_source]);
+
+ let projection = projection_exec(
+ vec![
+ (col("c", &schema)?, "c".to_string()),
+ (col("b", &schema)?, "b".to_string()),
+ (col("a", &schema)?, "a".to_string()),
+ ],
+ union,
+ )?;
+
+ let physical_plan =
+ sort_exec([sort_expr("a", &projection.schema())].into(), projection);
+
+ let test = EnforceSortingTest::new(physical_plan).with_repartition_sorts(true);
+ assert_snapshot!(test.run(), @r"
+ Input Plan:
+ SortExec: expr=[a@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ SortPreservingMergeExec: [a@2 ASC]
+ ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_push_sort_through_alias_reordered_projection() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let source = parquet_exec(schema.clone());
+ let projection = projection_exec(
+ vec![
+ (col("c", &schema)?, "c_alias".to_string()),
+ (col("b", &schema)?, "b_alias".to_string()),
+ (col("a", &schema)?, "a_alias".to_string()),
+ ],
+ source,
+ )?;
+
+ let physical_plan = sort_exec(
+ [sort_expr("a_alias", &projection.schema())].into(),
+ projection,
+ );
+
+ let test = EnforceSortingTest::new(physical_plan);
+ assert_snapshot!(test.run(), @r"
+ Input Plan:
+ SortExec: expr=[a_alias@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_does_not_push_sort_through_computed_projection() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let source = parquet_exec(schema.clone());
+ let computed_expr = Arc::new(BinaryExpr::new(
+ col("a", &schema)?,
+ Operator::Plus,
+ col("b", &schema)?,
+ )) as Arc<dyn PhysicalExpr>;
+ let projection = projection_exec(
+ vec![
+ (computed_expr, "sort_key".to_string()),
+ (col("c", &schema)?, "c".to_string()),
+ ],
+ source,
+ )?;
+
+ let physical_plan = sort_exec(
+ [sort_expr("sort_key", &projection.schema())].into(),
+ projection,
+ );
+
+ let test = EnforceSortingTest::new(physical_plan);
+ assert_snapshot!(test.run(), @"
+ Input Plan:
+ SortExec: expr=[sort_key@0 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[a@0 + b@1 as sort_key, c@2 as c]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ SortExec: expr=[sort_key@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[a@0 + b@1 as sort_key, c@2 as c]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_push_fetch_sort_through_alias_reordered_projection() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let source = parquet_exec(schema.clone());
+ let projection = projection_exec(
+ vec![
+ (col("c", &schema)?, "c_alias".to_string()),
+ (col("b", &schema)?, "b_alias".to_string()),
+ (col("a", &schema)?, "a_alias".to_string()),
+ ],
+ source,
+ )?;
+
+ let physical_plan = sort_exec_with_fetch(
+ [sort_expr("a_alias", &projection.schema())].into(),
+ Some(3),
+ projection,
+ );
+
+ let test = EnforceSortingTest::new(physical_plan);
+ assert_snapshot!(test.run(), @r"
+ Input Plan:
+ SortExec: TopK(fetch=3), expr=[a_alias@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ SortExec: TopK(fetch=3), expr=[a@0 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_push_sort_through_reordered_projection_remaps_multiple_keys_and_options()
+-> Result<()> {
+ let schema = create_test_schema3()?;
+ let source = parquet_exec(schema.clone());
+
+ let projection = projection_exec(
+ vec![
+ (col("c", &schema)?, "c_alias".to_string()),
+ (col("b", &schema)?, "b_alias".to_string()),
+ (col("a", &schema)?, "a_alias".to_string()),
+ ],
+ source,
+ )?;
+
+ let projection_schema = projection.schema();
+ let ordering: LexOrdering = [
+ sort_expr_options(
+ "c_alias",
+ &projection_schema,
+ SortOptions {
+ descending: true,
+ nulls_first: false,
+ },
+ ),
+ sort_expr("a_alias", &projection_schema),
+ ]
+ .into();
+
+ let physical_plan = sort_exec(ordering, projection);
+
+ let test = EnforceSortingTest::new(physical_plan);
+ assert_snapshot!(test.run(), @r"
+ Input Plan:
+ SortExec: expr=[c_alias@0 DESC NULLS LAST, a_alias@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ ProjectionExec: expr=[c@2 as c_alias, b@1 as b_alias, a@0 as a_alias]
+ SortExec: expr=[c@2 DESC NULLS LAST, a@0 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_does_not_push_fetch_sort_through_projection_over_union() -> Result<()> {
+ let schema = create_test_schema3()?;
+ let union = union_exec(vec![
+ parquet_exec(schema.clone()),
+ parquet_exec(schema.clone()),
+ ]);
+
+ let projection = projection_exec(
+ vec![
+ (col("c", &schema)?, "c".to_string()),
+ (col("b", &schema)?, "b".to_string()),
+ (col("a", &schema)?, "a".to_string()),
+ ],
+ union,
+ )?;
+
+ let physical_plan = sort_exec_with_fetch(
+ [sort_expr("a", &projection.schema())].into(),
+ Some(4),
+ projection,
+ );
+
+ let test = EnforceSortingTest::new(physical_plan);
+ assert_snapshot!(test.run(), @r"
+ Input Plan:
+ SortExec: TopK(fetch=4), expr=[a@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+
+ Optimized Plan:
+ SortExec: TopK(fetch=4), expr=[a@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+
+ Ok(())
+}
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index 3fdbc9d312..4e2f2ce601 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -293,8 +293,8 @@ fn test_projection_over_multi_partition_sort_limit() {
assert_ensure_requirements_plan!(limit, @r"
GlobalLimitExec: skip=0, fetch=21
SortPreservingMergeExec: [a@0 DESC]
- SortExec: expr=[a@0 DESC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@0 as a, b@1 as b]
+ ProjectionExec: expr=[a@0 as a, b@1 as b]
+ SortExec: expr=[a@0 DESC], preserve_partitioning=[true]
MockMultiPartitionExec
");
}
@@ -580,8 +580,8 @@ fn test_sort_pushdown_through_projection_adds_spm() {
assert_ensure_requirements_plan!(output_req, @r"
OutputRequirementExec: order_by=[(a@0, desc)], dist_by=SinglePartition
SortPreservingMergeExec: [a@0 DESC], fetch=21
- SortExec: TopK(fetch=21), expr=[a@0 DESC], preserve_partitioning=[true]
- ProjectionExec: expr=[a@0 as a, b@1 as b]
+ ProjectionExec: expr=[a@0 as a, b@1 as b]
+ SortExec: TopK(fetch=21), expr=[a@0 DESC], preserve_partitioning=[true]
MockMultiPartitionExec
");
}
diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs
index a7cec182f7..2293098bb8 100644
--- a/datafusion/core/tests/sql/explain_analyze.rs
+++ b/datafusion/core/tests/sql/explain_analyze.rs
@@ -774,8 +774,8 @@ async fn test_physical_plan_display_indent() {
actual,
@r"
SortPreservingMergeExec: [the_min@2 DESC], fetch=10
- SortExec: TopK(fetch=10), expr=[the_min@2 DESC], preserve_partitioning=[true]
- ProjectionExec: expr=[c1@0 as c1, max(aggregate_test_100.c12)@1 as max(aggregate_test_100.c12), min(aggregate_test_100.c12)@2 as the_min]
+ ProjectionExec: expr=[c1@0 as c1, max(aggregate_test_100.c12)@1 as max(aggregate_test_100.c12), min(aggregate_test_100.c12)@2 as the_min]
+ SortExec: TopK(fetch=10), expr=[min(aggregate_test_100.c12)@2 DESC], preserve_partitioning=[true]
AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[max(aggregate_test_100.c12), min(aggregate_test_100.c12)]
RepartitionExec: partitioning=Hash([c1@0], 9000), input_partitions=9000
AggregateExec: mode=Partial, gby=[c1@0 as c1], aggr=[max(aggregate_test_100.c12), min(aggregate_test_100.c12)]
diff --git a/datafusion/core/tests/sql/runtime_config.rs b/datafusion/core/tests/sql/runtime_config.rs
index 5f1e0629ec..b0e4bccf30 100644
--- a/datafusion/core/tests/sql/runtime_config.rs
+++ b/datafusion/core/tests/sql/runtime_config.rs
@@ -31,6 +31,8 @@ use datafusion_execution::cache::default_cache::DefaultCache;
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_physical_plan::common::collect;
+use crate::helper::plan_metrics::plan_spill_count;
+
#[tokio::test]
async fn test_memory_limit_with_spill() {
let ctx = SessionContext::new();
@@ -57,8 +59,7 @@ async fn test_memory_limit_with_spill() {
let stream = plan.execute(0, task_ctx).unwrap();
let _results = collect(stream).await;
- let metrics = plan.metrics().unwrap();
- let spill_count = metrics.spill_count().unwrap();
+ let spill_count = plan_spill_count(plan.as_ref());
assert!(spill_count > 0, "Expected spills but none occurred");
}
@@ -87,8 +88,7 @@ async fn test_no_spill_with_adequate_memory() {
let stream = plan.execute(0, task_ctx).unwrap();
let _results = collect(stream).await;
- let metrics = plan.metrics().unwrap();
- let spill_count = metrics.spill_count().unwrap();
+ let spill_count = plan_spill_count(plan.as_ref());
assert_eq!(spill_count, 0, "Expected no spills but some occurred");
}
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 159c5a9e50..d69ae34610 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -356,7 +356,20 @@ fn pushdown_requirement_to_children(
return Ok(None);
};
match determine_children_requirement(&parent_required, &child_req, child_plan) {
- RequirementsCompatibility::Satisfy => Ok(Some(vec![Some(child_req)])),
+ RequirementsCompatibility::Satisfy => {
+ // Window input requirements may be empty or constant-only.
+ // Such requirements do not guarantee the parent's output ordering, so
+ // keep the sort above the window unless the window output is known
+ // to satisfy it.
+ if !plan
+ .equivalence_properties()
+ .ordering_satisfy_requirement(parent_required.first().clone())?
+ {
+ return Ok(None);
+ }
+
+ Ok(Some(vec![Some(child_req)]))
+ }
RequirementsCompatibility::Compatible(adjusted) => {
// If parent requirements are more specific than output ordering
// of the window plan, then we can deduce that the parent expects
@@ -364,7 +377,7 @@ fn pushdown_requirement_to_children(
// that's the case, we block the pushdown of sort operation.
if !plan
.equivalence_properties()
- .ordering_satisfy_requirement(parent_required.into_single())?
+ .ordering_satisfy_requirement(parent_required.first().clone())?
{
return Ok(None);
}
@@ -469,15 +482,12 @@ fn pushdown_requirement_to_children(
}
} else if let Some(aggregate_exec) = plan.downcast_ref::<AggregateExec>() {
handle_aggregate_pushdown(aggregate_exec, parent_required)
+ } else if let Some(projection_exec) = plan.downcast_ref::<ProjectionExec>() {
+ handle_projection_pushdown(projection_exec, &parent_required)
} else if maintains_input_order.is_empty()
|| !maintains_input_order.iter().any(|o| *o)
|| plan.is::<RepartitionExec>()
|| plan.is::<FilterExec>()
- // A `ProjectionExec` with a fetch is handled in the fetch branch above
- // (it remaps the requirement through the projection's column mapping).
- // Without a fetch we do not push a sort requirement through a
- // projection (the sort is placed above it).
- || plan.is::<ProjectionExec>()
|| pushdown_would_violate_requirements(&parent_required, plan.as_ref())
{
// If the current plan is a leaf node or can not maintain any of the input ordering, can not pushed down requirements.
@@ -1042,6 +1052,45 @@ enum RequirementsCompatibility {
NonCompatible,
}
+/// Attempts to push parent ordering requirements through a [`ProjectionExec`].
+///
+/// This is safe when every required sort expression refers to a projected output
+/// column that is backed by a simple input column. In that case, the requirement
+/// can be remapped from the projection output schema to the projection input
+/// schema while preserving the original sort options.
+///
+/// For example, a parent requirement on `a@2` over:
+///
+/// ```text
+/// ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
+/// ```
+///
+/// is remapped to a child requirement on `a@0`.
+///
+/// The implementation is intentionally conservative: computed projection
+/// expressions and non-column sort expressions are not pushed down. Returning
+/// `Ok(None)` leaves sorting above the projection, preserving correctness.
+fn handle_projection_pushdown(
+ projection_exec: &ProjectionExec,
+ parent_required: &OrderingRequirements,
+) -> Result<Option<Vec<Option<OrderingRequirements>>>> {
+ // Only push sorting through pure column projections. Source-dependent
+ // expressions must stay close enough to the scan to be rewritten
+ // by the source and cannot be evaluated by [`ProjectionExec`].
+ if projection_exec
+ .expr()
+ .iter()
+ .any(|expr| !expr.expr.is::<Column>())
+ {
+ return Ok(None);
+ }
+
+ Ok(
+ remap_requirement_through_projection(projection_exec, parent_required)
+ .map(|requirements| vec![Some(requirements)]),
+ )
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/datafusion/sqllogictest/test_files/aggregates_topk.slt b/datafusion/sqllogictest/test_files/aggregates_topk.slt
index 81c85c433b..39e3d91aa1 100644
--- a/datafusion/sqllogictest/test_files/aggregates_topk.slt
+++ b/datafusion/sqllogictest/test_files/aggregates_topk.slt
@@ -227,8 +227,8 @@ logical_plan
04)------TableScan: string_topk projection=[category, val]
physical_plan
01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2
-02)--SortExec: TopK(fetch=2), expr=[max_val@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[category@0 as category, max(string_topk.val)@1 as max_val]
+02)--ProjectionExec: expr=[category@0 as category, max(string_topk.val)@1 as max_val]
+03)----SortExec: TopK(fetch=2), expr=[max(string_topk.val)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]
05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk.val)], lim=[2]
@@ -252,8 +252,8 @@ logical_plan
06)----------TableScan: string_topk projection=[category, val]
physical_plan
01)SortPreservingMergeExec: [max_val@1 DESC], fetch=2
-02)--SortExec: TopK(fetch=2), expr=[max_val@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[category@0 as category, max(string_topk_view.val)@1 as max_val]
+02)--ProjectionExec: expr=[category@0 as category, max(string_topk_view.val)@1 as max_val]
+03)----SortExec: TopK(fetch=2), expr=[max(string_topk_view.val)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]
05)--------RepartitionExec: partitioning=Hash([category@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[category@0 as category], aggr=[max(string_topk_view.val)], lim=[2]
@@ -284,8 +284,8 @@ logical_plan
04)------TableScan: traces projection=[trace_id]
physical_plan
01)SortPreservingMergeExec: [max_trace@1 DESC], fetch=2
-02)--SortExec: TopK(fetch=2), expr=[max_trace@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[trace_id@0 as trace_id, max(traces.trace_id)@1 as max_trace]
+02)--ProjectionExec: expr=[trace_id@0 as trace_id, max(traces.trace_id)@1 as max_trace]
+03)----SortExec: TopK(fetch=2), expr=[max(traces.trace_id)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]
05)--------RepartitionExec: partitioning=Hash([trace_id@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[trace_id@0 as trace_id], aggr=[max(traces.trace_id)], lim=[2]
diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.slt
index 7c2a8bcaa1..96c4f38c65 100644
--- a/datafusion/sqllogictest/test_files/clickbench.slt
+++ b/datafusion/sqllogictest/test_files/clickbench.slt
@@ -213,8 +213,8 @@ logical_plan
06)----------TableScan: hits_raw projection=[AdvEngineID], partial_filters=[hits_raw.AdvEngineID != Int16(0)]
physical_plan
01)SortPreservingMergeExec: [count(*)@1 DESC]
-02)--SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*)]
+02)--ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*)]
+03)----SortExec: expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([AdvEngineID@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))]
@@ -239,8 +239,8 @@ logical_plan
06)----------TableScan: hits_raw projection=[RegionID, UserID]
physical_plan
01)SortPreservingMergeExec: [u@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[RegionID@0 as RegionID, count(alias1)@1 as u]
+02)--ProjectionExec: expr=[RegionID@0 as RegionID, count(alias1)@1 as u]
+03)----SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[count(alias1)]
05)--------RepartitionExec: partitioning=Hash([RegionID@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[count(alias1)]
@@ -269,8 +269,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[RegionID, UserID, ResolutionWidth, AdvEngineID]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)]
+02)--ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)]
05)--------RepartitionExec: partitioning=Hash([RegionID@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)]
@@ -298,15 +298,15 @@ logical_plan
07)------------TableScan: hits_raw projection=[UserID, MobilePhoneModel], partial_filters=[hits_raw.MobilePhoneModel != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [u@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u]
+02)--ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u]
+03)----SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)]
05)--------RepartitionExec: partitioning=Hash([MobilePhoneModel@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)]
07)------------AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel, alias1@1 as alias1], aggr=[]
08)--------------RepartitionExec: partitioning=Hash([MobilePhoneModel@0, alias1@1], 4), input_partitions=4
09)----------------AggregateExec: mode=Partial, gby=[MobilePhoneModel@1 as MobilePhoneModel, UserID@0 as alias1], aggr=[]
-10)------------------FilterExec: MobilePhoneModel@1 !=
+10)------------------FilterExec: MobilePhoneModel@1 !=
11)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
12)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[UserID, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@34 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()]
@@ -328,15 +328,15 @@ logical_plan
07)------------TableScan: hits_raw projection=[UserID, MobilePhone, MobilePhoneModel], partial_filters=[hits_raw.MobilePhoneModel != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [u@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[u@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u]
+02)--ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u]
+03)----SortExec: TopK(fetch=10), expr=[count(alias1)@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)]
05)--------RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)]
07)------------AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, alias1@2 as alias1], aggr=[]
08)--------------RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1, alias1@2], 4), input_partitions=4
09)----------------AggregateExec: mode=Partial, gby=[MobilePhone@1 as MobilePhone, MobilePhoneModel@2 as MobilePhoneModel, UserID@0 as alias1], aggr=[]
-10)------------------FilterExec: MobilePhoneModel@2 !=
+10)------------------FilterExec: MobilePhoneModel@2 !=
11)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
12)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[UserID, MobilePhone, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@34 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()]
@@ -357,12 +357,12 @@ logical_plan
06)----------TableScan: hits_raw projection=[SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [c@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c]
+02)--ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([SearchPhrase@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))]
-07)------------FilterExec: SearchPhrase@0 !=
+07)------------FilterExec: SearchPhrase@0 !=
08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
09)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
@@ -384,15 +384,15 @@ logical_plan
07)------------TableScan: hits_raw projection=[UserID, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [u@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u]
+02)--ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u]
+03)----SortExec: TopK(fetch=10), expr=[count(alias1)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)]
05)--------RepartitionExec: partitioning=Hash([SearchPhrase@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)]
07)------------AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase, alias1@1 as alias1], aggr=[]
08)--------------RepartitionExec: partitioning=Hash([SearchPhrase@0, alias1@1], 4), input_partitions=4
09)----------------AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase, UserID@0 as alias1], aggr=[]
-10)------------------FilterExec: SearchPhrase@1 !=
+10)------------------FilterExec: SearchPhrase@1 !=
11)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
12)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[UserID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
@@ -413,12 +413,12 @@ logical_plan
06)----------TableScan: hits_raw projection=[SearchEngineID, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c]
+02)--ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))]
-07)------------FilterExec: SearchPhrase@1 !=
+07)------------FilterExec: SearchPhrase@1 !=
08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
09)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
@@ -438,8 +438,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[UserID]
physical_plan
01)SortPreservingMergeExec: [count(*)@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[count(*)@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*)]
+02)--ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([UserID@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[UserID@0 as UserID], aggr=[count(Int64(1))]
@@ -466,8 +466,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[UserID, SearchPhrase]
physical_plan
01)SortPreservingMergeExec: [count(*)@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[count(*)@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*)]
+02)--ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))]
@@ -521,8 +521,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[EventTime, UserID, SearchPhrase]
physical_plan
01)SortPreservingMergeExec: [count(*)@3 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[count(*)@3 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*)]
+02)--ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@3 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[UserID@1 as UserID, date_part(MINUTE, to_timestamp_seconds(EventTime@0)) as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))]
@@ -597,8 +597,8 @@ logical_plan
06)----------TableScan: hits_raw projection=[URL, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View(""), hits_raw.URL LIKE Utf8View("%google%")]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, min(hits.URL)@1 as min(hits.URL), count(Int64(1))@2 as c]
+02)--ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, min(hits.URL)@1 as min(hits.URL), count(Int64(1))@2 as c]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[min(hits.URL), count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([SearchPhrase@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase], aggr=[min(hits.URL), count(Int64(1))]
@@ -623,8 +623,8 @@ logical_plan
06)----------TableScan: hits_raw projection=[Title, UserID, URL, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View(""), hits_raw.Title LIKE Utf8View("%Google%"), hits_raw.URL NOT LIKE Utf8View("%.google.%")]
physical_plan
01)SortPreservingMergeExec: [c@3 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@3 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, min(hits.URL)@1 as min(hits.URL), min(hits.Title)@2 as min(hits.Title), count(Int64(1))@3 as c, count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)]
+02)--ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, min(hits.URL)@1 as min(hits.URL), min(hits.Title)@2 as min(hits.Title), count(Int64(1))@3 as c, count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@3 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[min(hits.URL), min(hits.Title), count(Int64(1)), count(DISTINCT hits.UserID)]
05)--------RepartitionExec: partitioning=Hash([SearchPhrase@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchPhrase@3 as SearchPhrase], aggr=[min(hits.URL), min(hits.Title), count(Int64(1)), count(DISTINCT hits.UserID)]
@@ -672,10 +672,11 @@ logical_plan
physical_plan
01)ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase]
02)--SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST], fetch=10
-03)----SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST], preserve_partitioning=[true]
-04)------FilterExec: SearchPhrase@1 != , projection=[SearchPhrase@1, EventTime@0]
-05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
+03)----ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime]
+04)------SortExec: TopK(fetch=10), expr=[EventTime@0 ASC NULLS LAST], preserve_partitioning=[true]
+05)--------FilterExec: SearchPhrase@1 !=
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
query T
SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10;
@@ -693,7 +694,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [SearchPhrase@0 ASC NULLS LAST], fetch=10
02)--SortExec: TopK(fetch=10), expr=[SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----FilterExec: SearchPhrase@0 !=
+03)----FilterExec: SearchPhrase@0 !=
04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
@@ -715,10 +716,11 @@ logical_plan
physical_plan
01)ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase]
02)--SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=10
-03)----SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true]
-04)------FilterExec: SearchPhrase@1 != , projection=[SearchPhrase@1, EventTime@0]
-05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
+03)----ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime]
+04)------SortExec: TopK(fetch=10), expr=[EventTime@0 ASC NULLS LAST, SearchPhrase@1 ASC NULLS LAST], preserve_partitioning=[true]
+05)--------FilterExec: SearchPhrase@1 !=
+06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]
query T
SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10;
@@ -738,13 +740,13 @@ logical_plan
07)------------TableScan: hits_raw projection=[CounterID, URL], partial_filters=[hits_raw.URL != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [l@1 DESC], fetch=25
-02)--SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c]
+02)--ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c]
+03)----SortExec: TopK(fetch=25), expr=[avg(length(hits.URL))@1 DESC], preserve_partitioning=[true]
04)------FilterExec: count(Int64(1))@2 > 100000
05)--------AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))]
06)----------RepartitionExec: partitioning=Hash([CounterID@0], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))]
-08)--------------FilterExec: URL@1 !=
+08)--------------FilterExec: URL@1 !=
09)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
10)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[CounterID, URL], file_type=parquet, predicate=URL@13 != , pruning_predicate=URL_null_count@2 != row_count@3 AND (URL_min@0 != OR != URL_max@1), required_guarantees=[URL not in ()]
@@ -766,13 +768,13 @@ logical_plan
07)------------TableScan: hits_raw projection=[Referer], partial_filters=[hits_raw.Referer != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [l@1 DESC], fetch=25
-02)--SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)]
+02)--ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)]
+03)----SortExec: TopK(fetch=25), expr=[avg(length(hits.Referer))@1 DESC], preserve_partitioning=[true]
04)------FilterExec: count(Int64(1))@2 > 100000
05)--------AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)]
06)----------RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[regexp_replace(Referer@0, ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)]
-08)--------------FilterExec: Referer@0 !=
+08)--------------FilterExec: Referer@0 !=
09)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
10)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[Referer], file_type=parquet, predicate=Referer@14 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()]
@@ -815,8 +817,8 @@ logical_plan
07)------------TableScan: hits_raw projection=[ClientIP, IsRefresh, ResolutionWidth, SearchEngineID, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+02)--ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
05)--------RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[SearchEngineID@3 as SearchEngineID, ClientIP@0 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
@@ -842,8 +844,8 @@ logical_plan
07)------------TableScan: hits_raw projection=[WatchID, ClientIP, IsRefresh, ResolutionWidth, SearchPhrase], partial_filters=[hits_raw.SearchPhrase != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+02)--ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
05)--------RepartitionExec: partitioning=Hash([WatchID@0, ClientIP@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
@@ -867,8 +869,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[WatchID, ClientIP, IsRefresh, ResolutionWidth]
physical_plan
01)SortPreservingMergeExec: [c@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+02)--ProjectionExec: expr=[WatchID@0 as WatchID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
05)--------RepartitionExec: partitioning=Hash([WatchID@0, ClientIP@1], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[WatchID@0 as WatchID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)]
@@ -900,8 +902,8 @@ logical_plan
05)--------TableScan: hits_raw projection=[URL]
physical_plan
01)SortPreservingMergeExec: [c@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c]
+02)--ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([URL@0], 4), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))]
@@ -989,8 +991,8 @@ logical_plan
07)------------TableScan: hits_raw projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], partial_filters=[hits_raw.CounterID = Int32(62), hits_raw.EventDate >= UInt16(15887), hits_raw.EventDate <= UInt16(15917), hits_raw.DontCountHits = Int16(0), hits_raw.IsRefresh = Int16(0), hits_raw.URL != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [pageviews@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews]
+02)--ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([URL@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))]
@@ -1016,8 +1018,8 @@ logical_plan
07)------------TableScan: hits_raw projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], partial_filters=[hits_raw.CounterID = Int32(62), hits_raw.EventDate >= UInt16(15887), hits_raw.EventDate <= UInt16(15917), hits_raw.DontCountHits = Int16(0), hits_raw.IsRefresh = Int16(0), hits_raw.Title != Utf8View("")]
physical_plan
01)SortPreservingMergeExec: [pageviews@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews]
+02)--ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([Title@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[Title@0 as Title], aggr=[count(Int64(1))]
@@ -1045,8 +1047,8 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=1000, fetch=10
02)--SortPreservingMergeExec: [pageviews@1 DESC], fetch=1010
-03)----SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews]
+03)----ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews]
+04)------SortExec: TopK(fetch=1010), expr=[count(Int64(1))@1 DESC], preserve_partitioning=[true]
05)--------AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))]
06)----------RepartitionExec: partitioning=Hash([URL@0], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))]
@@ -1074,8 +1076,8 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=1000, fetch=10
02)--SortPreservingMergeExec: [pageviews@5 DESC], fetch=1010
-03)----SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews]
+03)----ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews]
+04)------SortExec: TopK(fetch=1010), expr=[count(Int64(1))@5 DESC], preserve_partitioning=[true]
05)--------AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))]
06)----------RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[TraficSourceID@2 as TraficSourceID, SearchEngineID@3 as SearchEngineID, AdvEngineID@4 as AdvEngineID, CASE WHEN SearchEngineID@3 = 0 AND AdvEngineID@4 = 0 THEN Referer@1 ELSE END as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@0 as URL], aggr=[count(Int64(1))]
@@ -1103,8 +1105,8 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=100, fetch=10
02)--SortPreservingMergeExec: [pageviews@2 DESC], fetch=110
-03)----SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews]
+03)----ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews]
+04)------SortExec: TopK(fetch=110), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
05)--------AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))]
06)----------RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))]
@@ -1133,8 +1135,8 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=10000, fetch=10
02)--SortPreservingMergeExec: [pageviews@2 DESC], fetch=10010
-03)----SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews]
+03)----ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews]
+04)------SortExec: TopK(fetch=10010), expr=[count(Int64(1))@2 DESC], preserve_partitioning=[true]
05)--------AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))]
06)----------RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))]
diff --git a/datafusion/sqllogictest/test_files/copy.slt b/datafusion/sqllogictest/test_files/copy.slt
index 7aa7269b58..77977a6afc 100644
--- a/datafusion/sqllogictest/test_files/copy.slt
+++ b/datafusion/sqllogictest/test_files/copy.slt
@@ -33,7 +33,7 @@ COPY source_table TO 'test_files/scratch/copy/partitioned_table1/' STORED AS par
# validate multiple partitioned parquet file output
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table1/' PARTITIONED BY (col2);
query IT
@@ -44,7 +44,7 @@ select * from validate_partitioned_parquet order by col1, col2;
# validate partition paths were actually generated
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet_bar STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet_bar STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table1/col2=Bar';
query I
@@ -61,7 +61,7 @@ OPTIONS ('format.compression' 'zstd(10)');
# validate multiple partitioned parquet file output
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet2 STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet2 STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table2/' PARTITIONED BY (column2, column3);
query ITT
@@ -72,7 +72,7 @@ select * from validate_partitioned_parquet2 order by column1,column2,column3;
3 c z
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet_a_x STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet_a_x STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table2/column2=a/column3=x';
query I
@@ -89,7 +89,7 @@ OPTIONS ('format.compression' 'zstd(10)');
# validate multiple partitioned parquet file output
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet3 STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet3 STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table3/' PARTITIONED BY (column1, column3);
query TTT
@@ -100,7 +100,7 @@ select column1, column2, column3 from validate_partitioned_parquet3 order by col
3 c z
statement ok
-CREATE EXTERNAL TABLE validate_partitioned_parquet_1_x STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_partitioned_parquet_1_x STORED AS PARQUET
LOCATION 'test_files/scratch/copy/partitioned_table3/column1=1/column3=x';
query T
@@ -143,7 +143,7 @@ select column1, column2, column3, column4, column5, column6, column7, column8, c
statement ok
-create table test ("'test'" varchar, "'test2'" varchar, "'test3'" varchar);
+create table test ("'test'" varchar, "'test2'" varchar, "'test3'" varchar);
# https://github.com/apache/datafusion/issues/9714
## Until the partition by parsing uses ColumnDef, this test is meaningless since it becomes an overfit. Even in
@@ -249,7 +249,7 @@ select * from validate_parquet;
2 Bar
query I
-copy (values (struct(timestamp '2021-01-01 01:00:01', 1)), (struct(timestamp '2022-01-01 01:00:01', 2)),
+copy (values (struct(timestamp '2021-01-01 01:00:01', 1)), (struct(timestamp '2022-01-01 01:00:01', 2)),
(struct(timestamp '2023-01-03 01:00:01', 3)), (struct(timestamp '2024-01-01 01:00:01', 4)))
to 'test_files/scratch/copy/table_nested2/' STORED AS PARQUET;
----
@@ -267,15 +267,15 @@ select * from validate_parquet_nested2;
{c0: 2024-01-01T01:00:01, c1: 4}
query I
-COPY
-(values (struct ('foo', (struct ('foo', make_array(struct('a',1), struct('b',2))))), make_array(timestamp '2023-01-01 01:00:01',timestamp '2023-01-01 01:00:01')),
-(struct('bar', (struct ('foo', make_array(struct('aa',10), struct('bb',20))))), make_array(timestamp '2024-01-01 01:00:01', timestamp '2024-01-01 01:00:01')))
+COPY
+(values (struct ('foo', (struct ('foo', make_array(struct('a',1), struct('b',2))))), make_array(timestamp '2023-01-01 01:00:01',timestamp '2023-01-01 01:00:01')),
+(struct('bar', (struct ('foo', make_array(struct('aa',10), struct('bb',20))))), make_array(timestamp '2024-01-01 01:00:01', timestamp '2024-01-01 01:00:01')))
to 'test_files/scratch/copy/table_nested/' STORED AS PARQUET;
----
2
statement ok
-CREATE EXTERNAL TABLE validate_parquet_nested STORED AS PARQUET
+CREATE EXTERNAL TABLE validate_parquet_nested STORED AS PARQUET
LOCATION 'test_files/scratch/copy/table_nested/';
query ??
@@ -285,14 +285,14 @@ select * from validate_parquet_nested;
{c0: bar, c1: {c0: foo, c1: [{c0: aa, c1: 10}, {c0: bb, c1: 20}]}} [2024-01-01T01:00:01, 2024-01-01T01:00:01]
query I
-copy (values ([struct('foo', 1), struct('bar', 2)]))
+copy (values ([struct('foo', 1), struct('bar', 2)]))
to 'test_files/scratch/copy/array_of_struct/'
STORED AS PARQUET;
----
1
statement ok
-CREATE EXTERNAL TABLE validate_array_of_struct
+CREATE EXTERNAL TABLE validate_array_of_struct
STORED AS PARQUET LOCATION 'test_files/scratch/copy/array_of_struct/';
query ?
@@ -301,7 +301,7 @@ select * from validate_array_of_struct;
[{c0: foo, c1: 1}, {c0: bar, c1: 2}]
query I
-copy (values (struct('foo', [1,2,3], struct('bar', [2,3,4]))))
+copy (values (struct('foo', [1,2,3], struct('bar', [2,3,4]))))
to 'test_files/scratch/copy/struct_with_array/' STORED AS PARQUET;
----
1
@@ -578,8 +578,8 @@ select * from validate_arrow_file;
# Copy from dict encoded values to single arrow file
query I
-COPY (values
-('c', arrow_cast('foo', 'Dictionary(Int32, Utf8)')), ('d', arrow_cast('bar', 'Dictionary(Int32, Utf8)')))
+COPY (values
+('c', arrow_cast('foo', 'Dictionary(Int32, Utf8)')), ('d', arrow_cast('bar', 'Dictionary(Int32, Utf8)')))
to 'test_files/scratch/copy/table_dict.arrow' STORED AS ARROW;
----
2
diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.slt
index ae7ca1abab..d64efe80cc 100644
--- a/datafusion/sqllogictest/test_files/explain_analyze.slt
+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt
@@ -423,8 +423,8 @@ JOIN t2 ON t1.k > t2.k;
----
Plan with Metrics
01)PiecewiseMergeJoin: operator=Gt, join_type=Inner, on=(k > k), metrics=[output_bytes=0.0 B, build_mem_used=144.0 B]
-02)--SortExec: expr=[k@0 ASC], preserve_partitioning=[false], metrics=[output_bytes=16.0 B, spilled_bytes=0.0 B]
-03)----ProjectionExec: expr=[column1@0 as k], metrics=[output_bytes=32.0 B]
+02)--ProjectionExec: expr=[column1@0 as k], metrics=[output_bytes=16.0 B]
+03)----SortExec: expr=[column1@0 ASC], preserve_partitioning=[false], metrics=[output_bytes=16.0 B, spilled_bytes=0.0 B]
04)------DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]
05)--ProjectionExec: expr=[column1@0 as k], metrics=[output_bytes=32.0 B]
06)----DataSourceExec: partitions=1, partition_sizes=[1], metrics=[]
diff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.slt
index d435898f7b..7b0d8a00d5 100644
--- a/datafusion/sqllogictest/test_files/group_by.slt
+++ b/datafusion/sqllogictest/test_files/group_by.slt
@@ -2942,8 +2942,8 @@ logical_plan
08)----------SubqueryAlias: e
09)------------TableScan: sales_global projection=[sn, ts, currency, amount]
physical_plan
-01)SortExec: expr=[sn@2 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[zip_code@1 as zip_code, country@2 as country, sn@0 as sn, ts@3 as ts, currency@4 as currency, last_value(e.amount) ORDER BY [e.sn ASC NULLS LAST]@5 as last_rate]
+01)ProjectionExec: expr=[zip_code@1 as zip_code, country@2 as country, sn@0 as sn, ts@3 as ts, currency@4 as currency, last_value(e.amount) ORDER BY [e.sn ASC NULLS LAST]@5 as last_rate]
+02)--SortExec: expr=[sn@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----AggregateExec: mode=Single, gby=[sn@2 as sn, zip_code@0 as zip_code, country@1 as country, ts@3 as ts, currency@4 as currency], aggr=[last_value(e.amount) ORDER BY [e.sn ASC NULLS LAST]]
04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(currency@2, currency@4)], filter=ts@0 >= ts@1, projection=[zip_code@4, country@5, sn@6, ts@7, currency@8, sn@0, amount@3]
05)--------DataSourceExec: partitions=1, partition_sizes=[1]
@@ -2985,8 +2985,8 @@ logical_plan
04)------TableScan: sales_global projection=[country, ts, amount]
physical_plan
01)SortPreservingMergeExec: [country@0 ASC NULLS LAST]
-02)--SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[country@0 as country, first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@1 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@2 as fv2]
+02)--ProjectionExec: expr=[country@0 as country, first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@1 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@2 as fv2]
+03)----SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[country@0 as country], aggr=[first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST], last_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]]
05)--------RepartitionExec: partitioning=Hash([country@0], 8), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[country@0 as country], aggr=[first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST], last_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]]
@@ -3019,8 +3019,8 @@ logical_plan
04)------TableScan: sales_global projection=[country, ts, amount]
physical_plan
01)SortPreservingMergeExec: [country@0 ASC NULLS LAST]
-02)--SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[country@0 as country, first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@1 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.ts DESC NULLS FIRST]@2 as fv2]
+02)--ProjectionExec: expr=[country@0 as country, first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST]@1 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.ts DESC NULLS FIRST]@2 as fv2]
+03)----SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[country@0 as country], aggr=[first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST], last_value(sales_global.amount) ORDER BY [sales_global.ts DESC NULLS FIRST]]
05)--------RepartitionExec: partitioning=Hash([country@0], 8), input_partitions=1
06)----------AggregateExec: mode=Partial, gby=[country@0 as country], aggr=[first_value(sales_global.amount) ORDER BY [sales_global.ts ASC NULLS LAST], last_value(sales_global.amount) ORDER BY [sales_global.ts DESC NULLS FIRST]]
@@ -3181,8 +3181,8 @@ logical_plan
04)------TableScan: sales_global projection=[country, amount]
physical_plan
01)SortPreservingMergeExec: [country@0 ASC NULLS LAST]
-02)--SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[country@0 as country, array_agg(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]@1 as array_agg1]
+02)--ProjectionExec: expr=[country@0 as country, array_agg(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]@1 as array_agg1]
+03)----SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[country@0 as country], aggr=[array_agg(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]]
05)--------RepartitionExec: partitioning=Hash([country@0], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[country@0 as country], aggr=[array_agg(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]]
@@ -3216,8 +3216,8 @@ logical_plan
04)------TableScan: sales_global projection=[country, amount]
physical_plan
01)SortPreservingMergeExec: [country@0 ASC NULLS LAST]
-02)--SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[country@0 as country, array_agg(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]@1 as amounts, first_value(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]@2 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]@3 as fv2]
+02)--ProjectionExec: expr=[country@0 as country, array_agg(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]@1 as amounts, first_value(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST]@2 as fv1, last_value(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]@3 as fv2]
+03)----SortExec: expr=[country@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[country@0 as country], aggr=[array_agg(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST], first_value(sales_global.amount) ORDER BY [sales_global.amount ASC NULLS LAST], last_value(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]]
05)--------RepartitionExec: partitioning=Hash([country@0], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[country@0 as country], aggr=[array_agg(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST], last_value(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST], last_value(sales_global.amount) ORDER BY [sales_global.amount DESC NULLS FIRST]]
@@ -3484,8 +3484,8 @@ logical_plan
09)------------TableScan: sales_global_with_pk projection=[sn, amount]
physical_plan
01)SortPreservingMergeExec: [sn@0 ASC NULLS LAST]
-02)--SortExec: expr=[sn@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[sn@0 as sn, sum(l.amount)@2 as sum(l.amount), amount@1 as amount]
+02)--ProjectionExec: expr=[sn@0 as sn, sum(l.amount)@2 as sum(l.amount), amount@1 as amount]
+03)----SortExec: expr=[sn@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[sn@0 as sn, amount@1 as amount], aggr=[sum(l.amount)]
05)--------RepartitionExec: partitioning=Hash([sn@0, amount@1], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[sn@1 as sn, amount@2 as amount], aggr=[sum(l.amount)]
@@ -3630,8 +3630,8 @@ logical_plan
08)--------------TableScan: sales_global_with_pk projection=[zip_code, country, sn, ts, currency, amount]
physical_plan
01)SortPreservingMergeExec: [sn@2 ASC NULLS LAST]
-02)--SortExec: expr=[sn@2 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[zip_code@1 as zip_code, country@2 as country, sn@0 as sn, ts@3 as ts, currency@4 as currency, amount@5 as amount, sum_amount@6 as sum_amount]
+02)--ProjectionExec: expr=[zip_code@1 as zip_code, country@2 as country, sn@0 as sn, ts@3 as ts, currency@4 as currency, amount@5 as amount, sum_amount@6 as sum_amount]
+03)----SortExec: expr=[sn@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[sn@0 as sn, zip_code@1 as zip_code, country@2 as country, ts@3 as ts, currency@4 as currency, amount@5 as amount, sum_amount@6 as sum_amount], aggr=[]
05)--------RepartitionExec: partitioning=Hash([sn@0, zip_code@1, country@2, ts@3, currency@4, amount@5, sum_amount@6], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[sn@2 as sn, zip_code@0 as zip_code, country@1 as country, ts@3 as ts, currency@4 as currency, amount@5 as amount, sum_amount@6 as sum_amount], aggr=[]
@@ -4327,8 +4327,8 @@ logical_plan
04)------TableScan: csv_with_timestamps projection=[ts]
physical_plan
01)SortPreservingMergeExec: [months@0 DESC], fetch=5
-02)--SortExec: TopK(fetch=5), expr=[months@0 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as months]
+02)--ProjectionExec: expr=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as months]
+03)----SortExec: TopK(fetch=5), expr=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0 as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[], lim=[5]
05)--------RepartitionExec: partitioning=Hash([date_part(Utf8("MONTH"),csv_with_timestamps.ts)@0], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[date_part(MONTH, ts@0) as date_part(Utf8("MONTH"),csv_with_timestamps.ts)], aggr=[], lim=[5]
@@ -4438,8 +4438,8 @@ logical_plan
05)--------TableScan: aggregate_test_100 projection=[c1, c2, c3, c4]
physical_plan
01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST]
-02)--SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c1@0 as c1, count(alias1)@1 as count(DISTINCT aggregate_test_100.c2), min(alias1)@2 as min(DISTINCT aggregate_test_100.c2), sum(alias2)@3 as sum(aggregate_test_100.c3), max(alias3)@4 as max(aggregate_test_100.c4)]
+02)--ProjectionExec: expr=[c1@0 as c1, count(alias1)@1 as count(DISTINCT aggregate_test_100.c2), min(alias1)@2 as min(DISTINCT aggregate_test_100.c2), sum(alias2)@3 as sum(aggregate_test_100.c3), max(alias3)@4 as max(aggregate_test_100.c4)]
+03)----SortExec: expr=[c1@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[count(alias1), min(alias1), sum(alias2), max(alias3)]
05)--------RepartitionExec: partitioning=Hash([c1@0], 8), input_partitions=8
06)----------AggregateExec: mode=Partial, gby=[c1@0 as c1], aggr=[count(alias1), min(alias1), sum(alias2), max(alias3)]
diff --git a/datafusion/sqllogictest/test_files/insert_to_external.slt b/datafusion/sqllogictest/test_files/insert_to_external.slt
index e78c9dbcc4..f8bd555612 100644
--- a/datafusion/sqllogictest/test_files/insert_to_external.slt
+++ b/datafusion/sqllogictest/test_files/insert_to_external.slt
@@ -128,8 +128,8 @@ logical_plan
03)----Values: (Int64(5), Int64(1)), (Int64(4), Int64(2)), (Int64(7), Int64(7)), (Int64(7), Int64(8)), (Int64(7), Int64(9))...
physical_plan
01)DataSinkExec: sink=CsvSink(file_groups=[])
-02)--SortExec: expr=[a@0 ASC NULLS LAST, b@1 DESC], preserve_partitioning=[false]
-03)----ProjectionExec: expr=[column1@0 as a, column2@1 as b]
+02)--ProjectionExec: expr=[column1@0 as a, column2@1 as b]
+03)----SortExec: expr=[column1@0 ASC NULLS LAST, column2@1 DESC], preserve_partitioning=[false]
04)------DataSourceExec: partitions=1, partition_sizes=[1]
query I
@@ -696,7 +696,7 @@ LOCATION 'test_files/scratch/insert_to_external/external_parquet_table_q7/';
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt
index 1101ad6d2b..3b8f66def3 100644
--- a/datafusion/sqllogictest/test_files/joins.slt
+++ b/datafusion/sqllogictest/test_files/joins.slt
@@ -5402,10 +5402,10 @@ LEFT JOIN issue_19067_right r ON l.join_key = r.join_key
ORDER BY l.id;
----
physical_plan
-01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[id@2 as id, join_key@3 as left_key, join_key@0 as right_key, value@1 as value]
-03)----HashJoinExec: mode=CollectLeft, join_type=Right, on=[(join_key@0, join_key@1)]
-04)------DataSourceExec: partitions=1, partition_sizes=[1]
+01)ProjectionExec: expr=[id@2 as id, join_key@3 as left_key, join_key@0 as right_key, value@1 as value]
+02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(join_key@0, join_key@1)]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+04)----SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
05)------DataSourceExec: partitions=1, partition_sizes=[1]
statement count 0
@@ -5543,7 +5543,7 @@ statement count 0
DROP TABLE t2;
statement ok
-CREATE TABLE t1(a INT, b INT) AS VALUES
+CREATE TABLE t1(a INT, b INT) AS VALUES
(NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5);
statement ok
@@ -5555,7 +5555,7 @@ CREATE TABLE t2(c INT) AS VALUES (1), (2);
query II
SELECT sub.a, sub.b FROM (
SELECT * FROM t1 ORDER BY b LIMIT 1
-) sub
+) sub
JOIN t2 ON sub.a = t2.c;
----
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index b984705908..58a655c02b 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -748,7 +748,7 @@ explain select * from testSubQueryLimit as t1 join (select * from testSubQueryLi
----
logical_plan
01)Limit: skip=0, fetch=10
-02)--Cross Join:
+02)--Cross Join:
03)----SubqueryAlias: t1
04)------Limit: skip=0, fetch=10
05)--------TableScan: testsubquerylimit projection=[a, b], fetch=10
@@ -773,7 +773,7 @@ explain select * from testSubQueryLimit as t1 join (select * from testSubQueryLi
----
logical_plan
01)Limit: skip=0, fetch=2
-02)--Cross Join:
+02)--Cross Join:
03)----SubqueryAlias: t1
04)------Limit: skip=0, fetch=2
05)--------TableScan: testsubquerylimit projection=[a, b], fetch=2
@@ -1035,8 +1035,8 @@ physical_plan
01)GlobalLimitExec: skip=1, fetch=None
02)--SortExec: expr=[sy@2 DESC], preserve_partitioning=[false]
03)----SortPreservingMergeExec: [sx@1 DESC], fetch=4
-04)------SortExec: TopK(fetch=4), expr=[sx@1 DESC], preserve_partitioning=[true]
-05)--------ProjectionExec: expr=[g@0 as g, sum(t22489.x)@1 as sx, sum(t22489.y)@2 as sy]
+04)------ProjectionExec: expr=[g@0 as g, sum(t22489.x)@1 as sx, sum(t22489.y)@2 as sy]
+05)--------SortExec: TopK(fetch=4), expr=[sum(t22489.x)@1 DESC], preserve_partitioning=[true]
06)----------AggregateExec: mode=FinalPartitioned, gby=[g@0 as g], aggr=[sum(t22489.x), sum(t22489.y)]
07)------------RepartitionExec: partitioning=Hash([g@0], 4), input_partitions=1
08)--------------AggregateExec: mode=Partial, gby=[g@0 as g], aggr=[sum(t22489.x), sum(t22489.y)]
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 6907e489e6..79fb676f4b 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -272,8 +272,8 @@ logical_plan
04)------TableScan: aggregate_test_100 projection=[c2, c3]
physical_plan
01)SortPreservingMergeExec: [c2@0 ASC NULLS LAST]
-02)--SortExec: expr=[c2@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c2@0 as c2, sum(aggregate_test_100.c3)@1 as total_sal]
+02)--ProjectionExec: expr=[c2@0 as c2, sum(aggregate_test_100.c3)@1 as total_sal]
+03)----SortExec: expr=[c2@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[sum(aggregate_test_100.c3)]
05)--------RepartitionExec: partitioning=Hash([c2@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[c2@0 as c2], aggr=[sum(aggregate_test_100.c3)]
@@ -291,8 +291,8 @@ logical_plan
04)------TableScan: aggregate_test_100 projection=[c2, c3]
physical_plan
01)SortPreservingMergeExec: [total_sal@1 ASC NULLS LAST, c2@0 ASC NULLS LAST]
-02)--SortExec: expr=[total_sal@1 ASC NULLS LAST, c2@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c2@0 as c2, sum(aggregate_test_100.c3)@1 as total_sal]
+02)--ProjectionExec: expr=[c2@0 as c2, sum(aggregate_test_100.c3)@1 as total_sal]
+03)----SortExec: expr=[sum(aggregate_test_100.c3)@1 ASC NULLS LAST, c2@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[c2@0 as c2], aggr=[sum(aggregate_test_100.c3)]
05)--------RepartitionExec: partitioning=Hash([c2@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[c2@0 as c2], aggr=[sum(aggregate_test_100.c3)]
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index c1cb8ed561..fcadd3be1f 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -412,8 +412,9 @@ logical_plan
02)--Projection: three_cols.col_a, three_cols.col_b, three_cols.col_c, three_cols.col_b AS col_b_dup
03)----TableScan: three_cols projection=[col_a, col_b, col_c]
physical_plan
-01)SortExec: expr=[col_a@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c, col_b@1 as col_b_dup], file_type=parquet, sort_order_for_reorder=[col_a@0 ASC NULLS LAST]
+01)ProjectionExec: expr=[col_a@0 as col_a, col_b@1 as col_b, col_c@2 as col_c, col_b@1 as col_b_dup]
+02)--SortExec: expr=[col_a@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c], file_type=parquet, sort_order_for_reorder=[col_a@0 ASC NULLS LAST]
# Verify correctness
query IIII
@@ -564,8 +565,8 @@ logical_plan
04)------Projection: get_field(simple_struct.s, Utf8("value")) AS __datafusion_extracted_1, simple_struct.id
05)--------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(1)]
physical_plan
-01)SortExec: expr=[simple_struct.s[value]@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as simple_struct.s[value]]
+01)ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as simple_struct.s[value]]
+02)--SortExec: expr=[__datafusion_extracted_1@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----FilterExec: id@1 > 1
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 1, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
@@ -592,8 +593,8 @@ logical_plan
04)------Projection: get_field(simple_struct.s, Utf8("value")) AS __datafusion_extracted_1, simple_struct.id
05)--------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(1)]
physical_plan
-01)SortExec: TopK(fetch=2), expr=[simple_struct.s[value]@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as simple_struct.s[value]]
+01)ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as simple_struct.s[value]]
+02)--SortExec: TopK(fetch=2), expr=[__datafusion_extracted_1@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----FilterExec: id@1 > 1
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 1 AND DynamicFilter [ empty ], dynamic_rg_pruning=eligible, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
@@ -762,8 +763,8 @@ logical_plan
05)--------TableScan: multi_struct projection=[id, s], partial_filters=[multi_struct.id > Int64(2)]
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]
-02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as multi_struct.s[value]]
+02)--ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as multi_struct.s[value]]
+03)----SortExec: expr=[id@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------FilterExec: id@1 > 2
05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3
06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, id], file_type=parquet, predicate=id@0 > 2, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 2, required_guarantees=[]
@@ -1688,8 +1689,9 @@ logical_plan
04)------Projection: get_field(simple_struct.s, Utf8("value")) AS __datafusion_extracted_1, get_field(simple_struct.s, Utf8("label")) AS __datafusion_extracted_2
05)--------TableScan: simple_struct projection=[s]
physical_plan
-01)SortExec: expr=[t.s[value]@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as t.s[value], get_field(s@1, label) as t.s[label]], file_type=parquet
+01)ProjectionExec: expr=[__datafusion_extracted_1@0 as t.s[value], __datafusion_extracted_2@1 as t.s[label]]
+02)--SortExec: expr=[__datafusion_extracted_1@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, get_field(s@1, label) as __datafusion_extracted_2], file_type=parquet
# Verify correctness
query IT
@@ -1816,13 +1818,14 @@ logical_plan
12)--------------TableScan: simple_struct projection=[id, s], partial_filters=[simple_struct.id > Int64(3)]
physical_plan
01)SortPreservingMergeExec: [t.s[value]@0 ASC NULLS LAST]
-02)--SortExec: expr=[t.s[value]@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[__datafusion_extracted_1@0 as t.s[value], __datafusion_extracted_2@1 as t.s[label]]
-04)------UnionExec
+02)--ProjectionExec: expr=[__datafusion_extracted_1@0 as t.s[value], __datafusion_extracted_2@1 as t.s[label]]
+03)----UnionExec
+04)------SortExec: expr=[__datafusion_extracted_1@0 ASC NULLS LAST], preserve_partitioning=[false]
05)--------FilterExec: id@2 <= 3, projection=[__datafusion_extracted_1@0, __datafusion_extracted_2@1]
06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, get_field(s@1, label) as __datafusion_extracted_2, id], file_type=parquet, predicate=id@0 <= 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_min@0 <= 3, required_guarantees=[]
-07)--------FilterExec: id@2 > 3, projection=[__datafusion_extracted_1@0, __datafusion_extracted_2@1]
-08)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, get_field(s@1, label) as __datafusion_extracted_2, id], file_type=parquet, predicate=id@0 > 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 3, required_guarantees=[]
+07)------SortExec: expr=[__datafusion_extracted_1@0 ASC NULLS LAST], preserve_partitioning=[false]
+08)--------FilterExec: id@2 > 3, projection=[__datafusion_extracted_1@0, __datafusion_extracted_2@1]
+09)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, get_field(s@1, label) as __datafusion_extracted_2, id], file_type=parquet, predicate=id@0 > 3, pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 3, required_guarantees=[]
# Verify correctness
query IT
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index 6cb92025ca..e879947e32 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -654,8 +654,9 @@ query TT
EXPLAIN ANALYZE SELECT b, a FROM topk_proj ORDER BY a LIMIT 2;
----
Plan with Metrics
-01)SortExec: TopK(fetch=2), expr=[a@1 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@1 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@1 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.21% (141/1.07 K)]
+01)ProjectionExec: expr=[b@1 as b, a@0 as a], metrics=[output_rows=2, output_batches=1]
+02)--SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible, pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.21% (141/1.07 K)]
# Case 2: prune — `SELECT a` — filter stays as `a < 2` on the scan.
query TT
diff --git a/datafusion/sqllogictest/test_files/pwmj.slt b/datafusion/sqllogictest/test_files/pwmj.slt
index 295eb94318..9789c0e4e5 100644
--- a/datafusion/sqllogictest/test_files/pwmj.slt
+++ b/datafusion/sqllogictest/test_files/pwmj.slt
@@ -39,9 +39,9 @@ query II
SELECT t1.t1_id, t2.t2_id
FROM join_t1 t1
JOIN join_t2 t2
- ON t1.t1_id > t2.t2_id
-WHERE t1.t1_id > 10
- AND t2.t2_int > 1
+ ON t1.t1_id > t2.t2_id
+WHERE t1.t1_id > 10
+ AND t2.t2_int > 1
ORDER BY 1;
----
22 11
@@ -53,9 +53,9 @@ query IITI
SELECT *
FROM join_t1 t1
JOIN join_t2 t2
- ON t1.t1_id > t2.t2_id
-WHERE t1.t1_id > 10
- AND t2.t2_int > 1
+ ON t1.t1_id > t2.t2_id
+WHERE t1.t1_id > 10
+ AND t2.t2_int > 1
ORDER BY 1;
----
22 11 z 3
@@ -67,9 +67,9 @@ EXPLAIN
SELECT t1.t1_id, t2.t2_id
FROM join_t1 t1
JOIN join_t2 t2
- ON t1.t1_id > t2.t2_id
-WHERE t1.t1_id > 10
- AND t2.t2_int > 1
+ ON t1.t1_id > t2.t2_id
+WHERE t1.t1_id > 10
+ AND t2.t2_int > 1
ORDER BY 1;
----
logical_plan
@@ -326,8 +326,8 @@ logical_plan
06)------SubqueryAlias: t2
07)--------TableScan: null_join_t2 projection=[id]
physical_plan
-01)SortExec: expr=[left_id@0 ASC NULLS LAST, right_id@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[id@0 as left_id, id@1 as right_id]
+01)ProjectionExec: expr=[id@0 as left_id, id@1 as right_id]
+02)--SortExec: expr=[id@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false]
03)----NestedLoopJoinExec: join_type=Inner, filter=id@0 < id@0 + id@1
04)------DataSourceExec: partitions=1, partition_sizes=[1]
05)------DataSourceExec: partitions=1, partition_sizes=[1]
@@ -339,8 +339,8 @@ JOIN null_join_t2 t2
ON t1.id < t2.id
ORDER BY 1,2;
----
-1 3
-2 3
+1 3
+2 3
statement ok
set datafusion.optimizer.enable_piecewise_merge_join = false;
diff --git a/datafusion/sqllogictest/test_files/references.slt b/datafusion/sqllogictest/test_files/references.slt
index 146046cffa..3da1b385ee 100644
--- a/datafusion/sqllogictest/test_files/references.slt
+++ b/datafusion/sqllogictest/test_files/references.slt
@@ -105,8 +105,8 @@ logical_plan
02)--Projection: test....., test..... AS c3
03)----TableScan: test projection=[....]
physical_plan
-01)SortExec: expr=[....@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[....@0 as ...., ....@0 as c3]
+01)ProjectionExec: expr=[....@0 as ...., ....@0 as c3]
+02)--SortExec: expr=[....@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----DataSourceExec: partitions=1, partition_sizes=[1]
diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
index 043a62314c..af74280c10 100644
--- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
+++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt
@@ -367,8 +367,8 @@ logical_plan
15)--------------------TableScan: fact_table_ordered projection=[timestamp, value, f_dkey]
physical_plan
01)SortPreservingMergeExec: [env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST]
-02)--SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value]
+02)--ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value]
+03)----SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, time_bin@1 as time_bin], aggr=[avg(a.max_bin_value)]
05)--------RepartitionExec: partitioning=Hash([env@0, time_bin@1], 3), input_partitions=3
06)----------AggregateExec: mode=Partial, gby=[env@1 as env, time_bin@0 as time_bin], aggr=[avg(a.max_bin_value)]
@@ -464,8 +464,8 @@ logical_plan
15)--------------------TableScan: fact_table_ordered projection=[timestamp, value, f_dkey]
physical_plan
01)SortPreservingMergeExec: [env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST]
-02)--SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value]
+02)--ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value]
+03)----SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, time_bin@1 as time_bin], aggr=[avg(a.max_bin_value)]
05)--------RepartitionExec: partitioning=Hash([env@0, time_bin@1], 3), input_partitions=3
06)----------AggregateExec: mode=Partial, gby=[env@1 as env, time_bin@0 as time_bin], aggr=[avg(a.max_bin_value)]
diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt
index 3a9ae30d04..4107921d2f 100644
--- a/datafusion/sqllogictest/test_files/select.slt
+++ b/datafusion/sqllogictest/test_files/select.slt
@@ -1577,9 +1577,9 @@ physical_plan
03)----RepartitionExec: partitioning=Hash([c2@0], 2), input_partitions=2
04)------AggregateExec: mode=Partial, gby=[c2@0 as c2], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1
-06)----------ProjectionExec: expr=[c2@0 as c2]
-07)------------SortExec: TopK(fetch=4), expr=[c1@1 ASC NULLS LAST, c2@0 ASC NULLS LAST], preserve_partitioning=[false]
-08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c2, c1], file_type=csv, has_header=true
+06)----------ProjectionExec: expr=[c2@1 as c2]
+07)------------SortExec: TopK(fetch=4), expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST], preserve_partitioning=[false]
+08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c2], file_type=csv, has_header=true
# FilterExec can track equality of non-column expressions.
# plan below shouldn't have a SortExec because given column 'a' is ordered.
@@ -1968,7 +1968,7 @@ SELECT COUNT(*) FROM t0 AS tt0 WHERE (4==(3/0));
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/subquery_sort.slt b/datafusion/sqllogictest/test_files/subquery_sort.slt
index 6df93a3daa..080fd57c27 100644
--- a/datafusion/sqllogictest/test_files/subquery_sort.slt
+++ b/datafusion/sqllogictest/test_files/subquery_sort.slt
@@ -116,12 +116,11 @@ logical_plan
06)----------WindowAggr: windowExpr=[[rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
07)------------TableScan: sink_table projection=[c1, c3, c9]
physical_plan
-01)ProjectionExec: expr=[c1@0 as c1, r@1 as r]
-02)--SortExec: TopK(fetch=2), expr=[c1@0 ASC NULLS LAST, c3@2 ASC NULLS LAST, c9@3 ASC NULLS LAST], preserve_partitioning=[false]
-03)----ProjectionExec: expr=[c1@0 as c1, rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r, c3@1 as c3, c9@2 as c9]
-04)------BoundedWindowAggExec: wdw=[rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
-05)--------SortExec: expr=[c1@0 DESC], preserve_partitioning=[false]
-06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c3, c9], file_type=csv, has_header=true
+01)ProjectionExec: expr=[c1@0 as c1, rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r]
+02)--SortExec: TopK(fetch=2), expr=[c1@0 ASC NULLS LAST, c3@1 ASC NULLS LAST, c9@2 ASC NULLS LAST], preserve_partitioning=[false]
+03)----BoundedWindowAggExec: wdw=[rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [sink_table.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+04)------SortExec: expr=[c1@0 DESC], preserve_partitioning=[false]
+05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c3, c9], file_type=csv, has_header=true
#Test with utf8view for window function
statement ok
@@ -142,12 +141,11 @@ logical_plan
06)----------WindowAggr: windowExpr=[[rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
07)------------TableScan: sink_table_with_utf8view projection=[c1, c3, c9]
physical_plan
-01)ProjectionExec: expr=[c1@0 as c1, r@1 as r]
-02)--SortExec: TopK(fetch=2), expr=[c1@0 ASC NULLS LAST, c3@2 ASC NULLS LAST, c9@3 ASC NULLS LAST], preserve_partitioning=[false]
-03)----ProjectionExec: expr=[c1@0 as c1, rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r, c3@1 as c3, c9@2 as c9]
-04)------BoundedWindowAggExec: wdw=[rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
-05)--------SortExec: expr=[c1@0 DESC], preserve_partitioning=[false]
-06)----------DataSourceExec: partitions=1, partition_sizes=[1]
+01)ProjectionExec: expr=[c1@0 as c1, rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as r]
+02)--SortExec: TopK(fetch=2), expr=[c1@0 ASC NULLS LAST, c3@1 ASC NULLS LAST, c9@2 ASC NULLS LAST], preserve_partitioning=[false]
+03)----BoundedWindowAggExec: wdw=[rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [sink_table_with_utf8view.c1 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+04)------SortExec: expr=[c1@0 DESC], preserve_partitioning=[false]
+05)--------DataSourceExec: partitions=1, partition_sizes=[1]
statement ok
DROP TABLE sink_table_with_utf8view;
diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.slt
index d669e845ac..e9c272889c 100644
--- a/datafusion/sqllogictest/test_files/topk.slt
+++ b/datafusion/sqllogictest/test_files/topk.slt
@@ -370,8 +370,9 @@ query TT
explain select number, letter, age, number as column4, letter as column5 from partial_sorted order by number desc, column4 desc, letter asc, column5 asc, age desc limit 3;
----
physical_plan
-01)SortExec: TopK(fetch=3), expr=[number@0 DESC, letter@1 ASC NULLS LAST, age@2 DESC], preserve_partitioning=[false], sort_prefix=[number@0 DESC, letter@1 ASC NULLS LAST]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age, number@0 as column4, letter@1 as column5], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
+01)ProjectionExec: expr=[number@0 as number, letter@1 as letter, age@2 as age, number@0 as column4, letter@1 as column5]
+02)--SortExec: TopK(fetch=3), expr=[number@0 DESC, letter@1 ASC NULLS LAST, age@2 DESC], preserve_partitioning=[false], sort_prefix=[number@0 DESC, letter@1 ASC NULLS LAST]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
# Verify that the sort prefix is correctly computed over normalized, order-maintaining projections (number + 1, number, number + 1, age)
query TT
@@ -379,8 +380,8 @@ explain select number + 1 as number_plus, number, number + 1 as other_number_plu
----
physical_plan
01)SortPreservingMergeExec: [number_plus@0 DESC, number@1 DESC, other_number_plus@2 DESC, age@3 ASC NULLS LAST], fetch=3
-02)--SortExec: TopK(fetch=3), expr=[number_plus@0 DESC, number@1 DESC, age@3 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[number_plus@0 DESC, number@1 DESC]
-03)----ProjectionExec: expr=[__common_expr_1@0 as number_plus, number@1 as number, __common_expr_1@0 as other_number_plus, age@2 as age]
+02)--ProjectionExec: expr=[__common_expr_1@0 as number_plus, number@1 as number, __common_expr_1@0 as other_number_plus, age@2 as age]
+03)----SortExec: TopK(fetch=3), expr=[__common_expr_1@0 DESC, number@1 DESC, age@2 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[__common_expr_1@0 DESC, number@1 DESC]
04)------ProjectionExec: expr=[CAST(number@0 AS Int64) + 1 as __common_expr_1, number@0 as number, age@1 as age]
05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, maintains_sort_order=true
06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, age], output_ordering=[number@0 DESC], file_type=parquet, predicate=DynamicFilter [ empty ], dynamic_rg_pruning=eligible
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part
index 92518116d9..b227f94553 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part
@@ -48,8 +48,8 @@ logical_plan
06)----------TableScan: lineitem projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], partial_filters=[lineitem.l_shipdate <= Date32("1998-09-02")]
physical_plan
01)SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST]
-02)--SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order]
+02)--ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order]
+03)----SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(__common_expr_1) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(__common_expr_1 * 1 + lineitem.l_tax) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q10.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q10.slt.part
index f30d2c567c..9b5db48e83 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q10.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q10.slt.part
@@ -70,8 +70,8 @@ logical_plan
17)----------TableScan: nation projection=[n_nationkey, n_name]
physical_plan
01)SortPreservingMergeExec: [revenue@2 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[revenue@2 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment]
+02)--ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment]
+03)----SortExec: TopK(fetch=10), expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
05)--------RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
index cd86b618f0..c1e0a638cc 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q11.slt.part
@@ -75,8 +75,8 @@ logical_plan
physical_plan
01)ScalarSubqueryExec: subqueries=1
02)--SortPreservingMergeExec: [value@1 DESC], fetch=10
-03)----SortExec: TopK(fetch=10), expr=[value@1 DESC], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]
+03)----ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value]
+04)------SortExec: TopK(fetch=10), expr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 DESC], preserve_partitioning=[true]
05)--------FilterExec: CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) > scalar_subquery(<pending>)
06)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)]
07)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q12.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q12.slt.part
index dbc09b476d..a9d579b6a5 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q12.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q12.slt.part
@@ -60,8 +60,8 @@ logical_plan
09)----------TableScan: orders projection=[o_orderkey, o_orderpriority]
physical_plan
01)SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST]
-02)--SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count]
+02)--ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count]
+03)----SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = 1-URGENT OR orders.o_orderpriority = 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != 1-URGENT AND orders.o_orderpriority != 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
05)--------RepartitionExec: partitioning=Hash([l_shipmode@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = 1-URGENT OR orders.o_orderpriority = 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != 1-URGENT AND orders.o_orderpriority != 2-HIGH THEN 1 ELSE 0 END) as sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q13.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q13.slt.part
index e3823eafc7..9f9cbb3b6a 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q13.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q13.slt.part
@@ -54,8 +54,8 @@ logical_plan
12)--------------------TableScan: orders projection=[o_orderkey, o_custkey, o_comment], partial_filters=[orders.o_comment NOT LIKE Utf8View("%special%requests%")]
physical_plan
01)SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist]
+02)--ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist]
+03)----SortExec: TopK(fetch=10), expr=[count(Int64(1))@1 DESC, c_count@0 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([c_count@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part
index ab830714b1..5902204e2f 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part
@@ -66,8 +66,8 @@ logical_plan
14)----------------TableScan: supplier projection=[s_suppkey, s_comment], partial_filters=[supplier.s_comment LIKE Utf8View("%Customer%Complaints%")]
physical_plan
01)SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt]
+02)--ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt]
+03)----SortExec: TopK(fetch=10), expr=[count(alias1)@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)]
05)--------RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
index 812f5d2cba..47e5d6d888 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q21.slt.part
@@ -90,8 +90,8 @@ logical_plan
30)------------------TableScan: lineitem projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
physical_plan
01)SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST]
-02)--SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait]
+02)--ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait]
+03)----SortExec: expr=[count(Int64(1))@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([s_name@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part
index 40fa8939c2..d3f27021f1 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part
@@ -74,8 +74,8 @@ logical_plan
physical_plan
01)ScalarSubqueryExec: subqueries=1
02)--SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST]
-03)----SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]
-04)------ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]
+03)----ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal]
+04)------SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true]
05)--------AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]
06)----------RepartitionExec: partitioning=Hash([cntrycode@0], 4), input_partitions=4
07)------------AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
index a92a752211..724ef72ca3 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q3.slt.part
@@ -59,8 +59,8 @@ logical_plan
15)--------------TableScan: lineitem projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], partial_filters=[lineitem.l_shipdate > Date32("1995-03-15")]
physical_plan
01)SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority]
+02)--ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority]
+03)----SortExec: TopK(fetch=10), expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 DESC, o_orderdate@1 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=SinglePartitioned, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5]
06)----------RepartitionExec: partitioning=Hash([o_orderkey@0], 4), input_partitions=4
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q4.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q4.slt.part
index 1bc1b1fefb..470d7a6527 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q4.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q4.slt.part
@@ -54,8 +54,8 @@ logical_plan
12)----------------TableScan: lineitem projection=[l_orderkey, l_commitdate, l_receiptdate], partial_filters=[lineitem.l_receiptdate > lineitem.l_commitdate]
physical_plan
01)SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST]
-02)--SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count]
+02)--ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count]
+03)----SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))]
05)--------RepartitionExec: partitioning=Hash([o_orderpriority@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
index 036c0e3b8c..0c4bdfda8d 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q5.slt.part
@@ -68,8 +68,8 @@ logical_plan
23)--------------TableScan: region projection=[r_regionkey, r_name], partial_filters=[region.r_name = Utf8View("ASIA")]
physical_plan
01)SortPreservingMergeExec: [revenue@1 DESC]
-02)--SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue]
+02)--ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue]
+03)----SortExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
05)--------RepartitionExec: partitioning=Hash([n_name@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * 1 - lineitem.l_discount) as sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part
index cfadd18cf1..0db80ae202 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part
@@ -85,8 +85,8 @@ logical_plan
25)----------------TableScan: nation projection=[n_nationkey, n_name], partial_filters=[nation.n_name = Utf8View("GERMANY") OR nation.n_name = Utf8View("FRANCE")]
physical_plan
01)SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST]
-02)--SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue]
+02)--ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue]
+03)----SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]
05)--------RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]
diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
index ade7ed7a6c..29869bcddf 100644
--- a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
+++ b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part
@@ -75,8 +75,8 @@ logical_plan
21)------------TableScan: nation projection=[n_nationkey, n_name]
physical_plan
01)SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], fetch=10
-02)--SortExec: TopK(fetch=10), expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit]
+02)--ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit]
+03)----SortExec: TopK(fetch=10), expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]
05)--------RepartitionExec: partitioning=Hash([nation@0, o_year@1], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]
diff --git a/datafusion/sqllogictest/test_files/unnest.slt b/datafusion/sqllogictest/test_files/unnest.slt
index 04a6efd960..5cca3cbfe4 100644
--- a/datafusion/sqllogictest/test_files/unnest.slt
+++ b/datafusion/sqllogictest/test_files/unnest.slt
@@ -278,8 +278,8 @@ NULL NULL 17
NULL NULL 18
query IIII
-select
- unnest(column1), unnest(column2) + 2,
+select
+ unnest(column1), unnest(column2) + 2,
column3 * 10, unnest(array_remove(column1, 4))
from unnest_table;
----
@@ -903,7 +903,7 @@ query TT
explain select * from unnest_table u, unnest(u.column1);
----
logical_plan
-01)Cross Join:
+01)Cross Join:
02)--SubqueryAlias: u
03)----TableScan: unnest_table projection=[column1, column2, column3, column4, column5]
04)--Subquery:
@@ -1060,8 +1060,8 @@ logical_plan
04)------Projection: t.column1 AS __unnest_placeholder(t.column1), t.column2
05)--------TableScan: t projection=[column1, column2]
physical_plan
-01)SortExec: expr=[unnested@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[__unnest_placeholder(t.column1,depth=1)@0 as unnested, column2@1 as column2]
+01)ProjectionExec: expr=[__unnest_placeholder(t.column1,depth=1)@0 as unnested, column2@1 as column2]
+02)--SortExec: expr=[__unnest_placeholder(t.column1,depth=1)@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----UnnestExec
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/unnest/ordered_array.parquet]]}, projection=[column1@0 as __unnest_placeholder(t.column1), column2], output_ordering=[column2@1 ASC NULLS LAST], file_type=parquet
@@ -1107,8 +1107,8 @@ logical_plan
05)--------Projection: struct(t.column1, t.column2, t.column3) AS __unnest_placeholder(struct(t.column1,t.column2,t.column3))
06)----------TableScan: t projection=[column1, column2, column3]
physical_plan
-01)SortExec: expr=[struct(t.column1,t.column2,t.column3).c0@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[__unnest_placeholder(struct(t.column1,t.column2,t.column3)).c0@0 as struct(t.column1,t.column2,t.column3).c0, __unnest_placeholder(struct(t.column1,t.column2,t.column3)).c1@1 as struct(t.column1,t.column2,t.column3).c1, __unnest_placeholder(struct(t.column1,t.column2,t.column3)).c2@2 as struct(t.column1,t.column2,t.column3).c2]
+01)ProjectionExec: expr=[__unnest_placeholder(struct(t.column1,t.column2,t.column3)).c0@0 as struct(t.column1,t.column2,t.column3).c0, __unnest_placeholder(struct(t.column1,t.column2,t.column3)).c1@1 as struct(t.column1,t.column2,t.column3).c1, __unnest_placeholder(struct(t.column1,t.column2,t.column3)).c2@2 as struct(t.column1,t.column2,t.column3).c2]
+02)--SortExec: expr=[__unnest_placeholder(struct(t.column1,t.column2,t.column3)).c0@0 ASC NULLS LAST], preserve_partitioning=[false]
03)----UnnestExec
04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/unnest/ordered_tuples.parquet]]}, projection=[struct(column1@0, column2@1, column3@2) as __unnest_placeholder(struct(t.column1,t.column2,t.column3))], file_type=parquet
@@ -1423,7 +1423,7 @@ DROP TABLE unused_unnest_pruning;
## Regression: pushing a leaf-extracted projection (containing get_field,
## which has MoveTowardsLeafNodes placement) through an `Unnest` used to
## trip `Assertion failed: expr.is_empty(): Unnest` inside
-## `PushDownLeafProjections`. The optimizer must not try to pushdown these
+## `PushDownLeafProjections`. The optimizer must not try to pushdown these
## projections through an `Unnest` and should produce a valid plan.
statement ok
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index b6cc822bbc..e1edca260e 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -272,8 +272,8 @@ logical_plan
16)------------------EmptyRelation: rows=1
physical_plan
01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]
-02)--SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[b@0 as b, max(d.a)@1 as max_a]
+02)--ProjectionExec: expr=[b@0 as b, max(d.a)@1 as max_a]
+03)----SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]
04)------AggregateExec: mode=FinalPartitioned, gby=[b@0 as b], aggr=[max(d.a)]
05)--------RepartitionExec: partitioning=Hash([b@0], 4), input_partitions=4
06)----------AggregateExec: mode=Partial, gby=[b@1 as b], aggr=[max(d.a)], ordering_mode=Sorted
@@ -2261,8 +2261,9 @@ physical_plan
06)----------ProjectionExec: expr=[c2@1 as c2, c8@2 as c8, c9@3 as c9, c1_alias@4 as c1_alias, sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST, t1.c8 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING@5 as sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST, t1.c8 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING, sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 5 FOLLOWING@6 as sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 5 FOLLOWING]
07)------------BoundedWindowAggExec: wdw=[sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 5 FOLLOWING: Field { "sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND 5 FOLLOWING": nullable UInt64 }, frame: ROWS BETWEEN 1 PRECEDING AND 5 FOLLOWING], mode=[Sorted]
08)--------------WindowAggExec: wdw=[sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST, t1.c8 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(t1.c9) PARTITION BY [t1.c1, t1.c2] ORDER BY [t1.c9 ASC NULLS LAST, t1.c8 ASC NULLS LAST] ROWS BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING", data_type: UInt64, nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(1)), end_bound: Following(UInt64(NULL)), is_causal: false }]
-09)----------------SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, c9@3 ASC NULLS LAST, c8@2 ASC NULLS LAST], preserve_partitioning=[false]
-10)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c2, c8, c9, c1@0 as c1_alias], file_type=csv, has_header=true
+09)----------------ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, c8@2 as c8, c9@3 as c9, c1@0 as c1_alias]
+10)------------------SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, c9@3 ASC NULLS LAST, c8@2 ASC NULLS LAST], preserve_partitioning=[false]
+11)--------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c1, c2, c8, c9], file_type=csv, has_header=true
query IIIII
SELECT c9,
@@ -2408,8 +2409,8 @@ logical_plan
03)----WindowAggr: windowExpr=[[row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
04)------TableScan: aggregate_test_100 projection=[c9]
physical_plan
-01)SortExec: TopK(fetch=5), expr=[rn1@1 DESC], preserve_partitioning=[false]
-02)--ProjectionExec: expr=[c9@0 as c9, row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rn1]
+01)ProjectionExec: expr=[c9@0 as c9, row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rn1]
+02)--SortExec: TopK(fetch=5), expr=[row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 DESC], preserve_partitioning=[false]
03)----BoundedWindowAggExec: wdw=[row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [aggregate_test_100.c9 DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
04)------SortExec: expr=[c9@0 DESC], preserve_partitioning=[false]
05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100.csv]]}, projection=[c9], file_type=csv, has_header=true
@@ -5485,7 +5486,7 @@ order by c1, c2, rank;
query TT
explain select c1, c2, rank1, rank2
from (
- select c1, c2, rank() over (partition by c1 order by c2) as rank1,
+ select c1, c2, rank() over (partition by c1 order by c2) as rank1,
rank() over (partition by c2, c1 order by c1) as rank2
from t1
)
@@ -5501,8 +5502,8 @@ logical_plan
06)----------TableScan: t1 projection=[c1, c2]
physical_plan
01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank1@2 ASC NULLS LAST, rank2@3 ASC NULLS LAST]
-02)--SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank1@2 ASC NULLS LAST, rank2@3 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank1, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank2]
+02)--ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank1, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank2]
+03)----SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 ASC NULLS LAST, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 ASC NULLS LAST], preserve_partitioning=[true]
04)------BoundedWindowAggExec: wdw=[rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
05)--------SortExec: expr=[c2@1 ASC NULLS LAST, c1@0 ASC NULLS LAST], preserve_partitioning=[true]
06)----------RepartitionExec: partitioning=Hash([c2@1, c1@0], 2), input_partitions=2
@@ -5516,7 +5517,7 @@ physical_plan
query IIII
select c1, c2, rank1, rank2
from (
- select c1, c2, rank() over (partition by c1 order by c2) as rank1,
+ select c1, c2, rank() over (partition by c1 order by c2) as rank1,
rank() over (partition by c2, c1 order by c1) as rank2
from t1
)
@@ -5533,7 +5534,7 @@ order by c1, c2, rank1, rank2;
query TT
explain select c1, c2, rank1, rank2
from (
- select c1, c2, rank() over (partition by c1 order by c2) as rank1,
+ select c1, c2, rank() over (partition by c1 order by c2) as rank1,
rank() over (partition by c2, c1 order by c1) as rank2
from t1
)
@@ -5549,8 +5550,8 @@ logical_plan
06)----------TableScan: t1 projection=[c1, c2]
physical_plan
01)SortPreservingMergeExec: [c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank1@2 ASC NULLS LAST, rank2@3 ASC NULLS LAST]
-02)--SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank1@2 ASC NULLS LAST, rank2@3 ASC NULLS LAST], preserve_partitioning=[true]
-03)----ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank1, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank2]
+02)--ProjectionExec: expr=[c1@0 as c1, c2@1 as c2, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank1, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank2]
+03)----SortExec: expr=[c1@0 ASC NULLS LAST, c2@1 ASC NULLS LAST, rank() PARTITION BY [t1.c1] ORDER BY [t1.c2 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 ASC NULLS LAST, rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 ASC NULLS LAST], preserve_partitioning=[true]
04)------BoundedWindowAggExec: wdw=[rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [t1.c2, t1.c1] ORDER BY [t1.c1 ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
05)--------SortExec: expr=[c2@1 ASC NULLS LAST, c1@0 ASC NULLS LAST], preserve_partitioning=[true]
06)----------RepartitionExec: partitioning=Hash([c2@1, c1@0], 2), input_partitions=2
@@ -5563,7 +5564,7 @@ physical_plan
query IIII
select c1, c2, rank1, rank2
from (
- select c1, c2, rank() over (partition by c1 order by c2) as rank1,
+ select c1, c2, rank() over (partition by c1 order by c2) as rank1,
rank() over (partition by c2, c1 order by c1) as rank2
from t1
)
@@ -6086,7 +6087,6 @@ physical_plan
05)--------SortExec: TopK(fetch=5), expr=[c1@2 ASC NULLS LAST, c2@3 ASC NULLS LAST], preserve_partitioning=[true]
06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-0.csv, WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-1.csv], [WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-2.csv, WORKSPACE_ROOT/datafusion/core/tests/data/partitioned_csv/partition-3.csv]]}, projection=[c2@1 >= 2 as __common_expr_1, c2@1 >= 2 AND c2@1 < 4 AND c1@0 > 0 as __common_expr_2, c1, c2], file_type=csv, has_header=false
-
# FILTER filters out some rows
query IIIII??
SELECT