Skip to content

Apache DataFusion SQL Query Engine

fix: preserve window inputs when replacing TopK sorts (#26719)

Kristoffer Vestergård Johansson32bd81fb226c
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs
index 43cd556597..8ffaed07a6 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs
@@ -8,6 +8,8 @@
use std::sync::Arc;
+use super::sort_pushdown::can_push_fetch_through;
+use crate::row_bound::{RowBound, limit_node};
use crate::utils::{
is_coalesce_partitions, is_repartition, is_sort, is_sort_preserving_merge,
};
@@ -97,6 +99,7 @@ pub fn plan_with_order_preserving_variants(
sort_input.data = false;
return Ok(sort_input);
}
+ let child_fetch = fetch.filter(|_| can_push_fetch_through(&sort_input.plan));
sort_input.children = sort_input
.children
.into_iter()
@@ -107,7 +110,7 @@ pub fn plan_with_order_preserving_variants(
node,
is_spr_better,
is_spm_better,
- fetch,
+ child_fetch,
)
} else {
Ok(node)
@@ -272,5 +275,12 @@ pub fn replace_with_order_preserving_variants(
for child in alternate_plan.children.iter_mut() {
child.data = false;
}
+ if let Some(fetch) = requirements.plan.fetch() {
+ let plan = limit_node(
+ Arc::clone(&alternate_plan.plan),
+ RowBound::fetch(Some(fetch)),
+ );
+ alternate_plan = PlanContext::new(plan, false, vec![alternate_plan]);
+ }
Ok(Transformed::yes(alternate_plan))
}
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index a821c0e49c..f2a01598b9 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -132,7 +132,7 @@ fn min_fetch(f1: Option<usize>, f2: Option<usize>) -> Option<usize> {
}
/// Returns whether a fetch on `plan` can move to its input.
-fn can_push_fetch_through(plan: &Arc<dyn ExecutionPlan>) -> bool {
+pub(crate) fn can_push_fetch_through(plan: &Arc<dyn ExecutionPlan>) -> bool {
match plan.limit_behavior() {
LimitBehavior::MoveToInput => true,
LimitBehavior::Merge => plan.fetch().is_none(),
diff --git a/datafusion/sqllogictest/test_files/window_limits.slt b/datafusion/sqllogictest/test_files/window_limits.slt
index 04a19ffaa8..c055b604f5 100644
--- a/datafusion/sqllogictest/test_files/window_limits.slt
+++ b/datafusion/sqllogictest/test_files/window_limits.slt
@@ -898,6 +898,47 @@ physical_plan
statement ok
DROP TABLE huge_frame_t;
+# Regression for #26617: count all rows before applying LIMIT.
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+query II
+SELECT count(*) OVER () AS n,
+ row_number() OVER (PARTITION BY length('x') ORDER BY a) AS r
+FROM (SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3)
+ORDER BY r LIMIT 1;
+----
+3 1
+
+query II
+SELECT count(*) OVER () AS n,
+ row_number() OVER (PARTITION BY length('x') ORDER BY a) AS r
+FROM (SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3)
+ORDER BY r LIMIT 1 OFFSET 1;
+----
+3 2
+
+# When replacing the outer TopK is safe, LimitPushdown must absorb the
+# redundant GlobalLimitExec into SortPreservingMergeExec.
+query TT
+EXPLAIN
+SELECT depname, row_number() OVER (PARTITION BY depname ORDER BY empno) AS r
+FROM employees
+ORDER BY depname LIMIT 1;
+----
+logical_plan
+01)Sort: employees.depname ASC NULLS LAST, fetch=1
+02)--Projection: employees.depname, row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS r
+03)----WindowAggr: windowExpr=[[row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+04)------TableScan: employees projection=[depname, empno]
+physical_plan
+01)SortPreservingMergeExec: [depname@0 ASC NULLS LAST], fetch=1
+02)--ProjectionExec: expr=[depname@0 as depname, row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]
+03)----BoundedWindowAggExec: wdw=[row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+04)------SortExec: expr=[depname@0 ASC NULLS LAST, empno@1 ASC NULLS LAST], preserve_partitioning=[true]
+05)--------RepartitionExec: partitioning=Hash([depname@0], 4), input_partitions=1
+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[depname, empno], file_type=csv, has_header=true
+
# Every window in a plan should get the limit pushed past it, not just the first
# one. See https://github.com/apache/datafusion/issues/26659
#
@@ -1482,3 +1523,51 @@ physical_plan
07)------------RepartitionExec: partitioning=Hash([depname@0], 4), input_partitions=1, maintains_sort_order=true
08)--------------SortExec: TopK(fetch=6), expr=[depname@0 ASC NULLS LAST, empno@1 ASC NULLS LAST], preserve_partitioning=[true]
09)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[depname, empno, salary], file_type=csv, has_header=true
+
+# A fetch above an OFFSET must not truncate the OFFSET input.
+statement ok
+SET datafusion.execution.target_partitions = 4;
+
+query I
+COPY (SELECT * FROM (VALUES (1), (3), (5), (7)) AS t(a))
+TO 'test_files/scratch/window_limits/sorted_offset/first.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'false');
+----
+4
+
+query I
+COPY (SELECT * FROM (VALUES (2), (4), (6), (8)) AS t(a))
+TO 'test_files/scratch/window_limits/sorted_offset/second.csv'
+STORED AS CSV OPTIONS ('format.has_header' 'false');
+----
+4
+
+statement ok
+CREATE EXTERNAL TABLE sorted_offset_t (a INT NOT NULL)
+STORED AS CSV
+WITH ORDER (a ASC)
+LOCATION 'test_files/scratch/window_limits/sorted_offset/'
+OPTIONS ('format.has_header' 'false');
+
+query I
+SELECT count(*) FROM (SELECT * FROM (SELECT a FROM sorted_offset_t LIMIT 3 OFFSET 2) ORDER BY a LIMIT 1);
+----
+1
+
+query TT
+EXPLAIN SELECT * FROM (SELECT a FROM sorted_offset_t LIMIT 3 OFFSET 2) ORDER BY a LIMIT 1;
+----
+logical_plan
+01)Sort: sorted_offset_t.a ASC NULLS LAST, fetch=1
+02)--Limit: skip=0, fetch=3
+03)----TableScan: sorted_offset_t projection=[a], fetch=3, skip=2
+physical_plan
+01)GlobalLimitExec: skip=2, fetch=1
+02)--SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=3
+03)----DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/first.csv:0..4], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/second.csv:0..4], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/first.csv:4..8], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/second.csv:4..8]]}, projection=[a], limit=3, output_ordering=[a@0 ASC NULLS LAST], file_type=csv, has_header=false
+
+# Also cover the OFFSET-only shape from #26690.
+query I
+SELECT count(*) FROM (SELECT * FROM (SELECT a FROM sorted_offset_t OFFSET 2) ORDER BY a LIMIT 3);
+----
+3