Apache DataFusion SQL Query Engine
fix: preserve window inputs when replacing TopK sorts (#26719)
Kristoffer Vestergård Johansson32bd81fb226c
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rsindex 43cd556597..8ffaed07a6 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/replace_with_order_preserving_variants.rs@@ -8,6 +8,8 @@use std::sync::Arc;+use super::sort_pushdown::can_push_fetch_through;+use crate::row_bound::{RowBound, limit_node};use crate::utils::{is_coalesce_partitions, is_repartition, is_sort, is_sort_preserving_merge,};@@ -97,6 +99,7 @@ pub fn plan_with_order_preserving_variants(sort_input.data = false;return Ok(sort_input);}+ let child_fetch = fetch.filter(|_| can_push_fetch_through(&sort_input.plan));sort_input.children = sort_input.children.into_iter()@@ -107,7 +110,7 @@ pub fn plan_with_order_preserving_variants(node,is_spr_better,is_spm_better,- fetch,+ child_fetch,)} else {Ok(node)@@ -272,5 +275,12 @@ pub fn replace_with_order_preserving_variants(for child in alternate_plan.children.iter_mut() {child.data = false;}+ if let Some(fetch) = requirements.plan.fetch() {+ let plan = limit_node(+ Arc::clone(&alternate_plan.plan),+ RowBound::fetch(Some(fetch)),+ );+ alternate_plan = PlanContext::new(plan, false, vec![alternate_plan]);+ }Ok(Transformed::yes(alternate_plan))}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex a821c0e49c..f2a01598b9 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -132,7 +132,7 @@ fn min_fetch(f1: Option<usize>, f2: Option<usize>) -> Option<usize> {}/// Returns whether a fetch on `plan` can move to its input.-fn can_push_fetch_through(plan: &Arc<dyn ExecutionPlan>) -> bool {+pub(crate) fn can_push_fetch_through(plan: &Arc<dyn ExecutionPlan>) -> bool {match plan.limit_behavior() {LimitBehavior::MoveToInput => true,LimitBehavior::Merge => plan.fetch().is_none(),diff --git a/datafusion/sqllogictest/test_files/window_limits.slt b/datafusion/sqllogictest/test_files/window_limits.sltindex 04a19ffaa8..c055b604f5 100644--- a/datafusion/sqllogictest/test_files/window_limits.slt+++ b/datafusion/sqllogictest/test_files/window_limits.slt@@ -898,6 +898,47 @@ physical_planstatement okDROP TABLE huge_frame_t;+# Regression for #26617: count all rows before applying LIMIT.+statement ok+SET datafusion.execution.target_partitions = 4;++query II+SELECT count(*) OVER () AS n,+ row_number() OVER (PARTITION BY length('x') ORDER BY a) AS r+FROM (SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3)+ORDER BY r LIMIT 1;+----+3 1++query II+SELECT count(*) OVER () AS n,+ row_number() OVER (PARTITION BY length('x') ORDER BY a) AS r+FROM (SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3)+ORDER BY r LIMIT 1 OFFSET 1;+----+3 2++# When replacing the outer TopK is safe, LimitPushdown must absorb the+# redundant GlobalLimitExec into SortPreservingMergeExec.+query TT+EXPLAIN+SELECT depname, row_number() OVER (PARTITION BY depname ORDER BY empno) AS r+FROM employees+ORDER BY depname LIMIT 1;+----+logical_plan+01)Sort: employees.depname ASC NULLS LAST, fetch=1+02)--Projection: employees.depname, row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS r+03)----WindowAggr: windowExpr=[[row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]+04)------TableScan: employees projection=[depname, empno]+physical_plan+01)SortPreservingMergeExec: [depname@0 ASC NULLS LAST], fetch=1+02)--ProjectionExec: expr=[depname@0 as depname, row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as r]+03)----BoundedWindowAggExec: wdw=[row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [employees.depname] ORDER BY [employees.empno ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+04)------SortExec: expr=[depname@0 ASC NULLS LAST, empno@1 ASC NULLS LAST], preserve_partitioning=[true]+05)--------RepartitionExec: partitioning=Hash([depname@0], 4), input_partitions=1+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[depname, empno], file_type=csv, has_header=true+# Every window in a plan should get the limit pushed past it, not just the first# one. See https://github.com/apache/datafusion/issues/26659#@@ -1482,3 +1523,51 @@ physical_plan07)------------RepartitionExec: partitioning=Hash([depname@0], 4), input_partitions=1, maintains_sort_order=true08)--------------SortExec: TopK(fetch=6), expr=[depname@0 ASC NULLS LAST, empno@1 ASC NULLS LAST], preserve_partitioning=[true]09)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/testing/data/csv/aggregate_test_100_with_dates.csv]]}, projection=[depname, empno, salary], file_type=csv, has_header=true++# A fetch above an OFFSET must not truncate the OFFSET input.+statement ok+SET datafusion.execution.target_partitions = 4;++query I+COPY (SELECT * FROM (VALUES (1), (3), (5), (7)) AS t(a))+TO 'test_files/scratch/window_limits/sorted_offset/first.csv'+STORED AS CSV OPTIONS ('format.has_header' 'false');+----+4++query I+COPY (SELECT * FROM (VALUES (2), (4), (6), (8)) AS t(a))+TO 'test_files/scratch/window_limits/sorted_offset/second.csv'+STORED AS CSV OPTIONS ('format.has_header' 'false');+----+4++statement ok+CREATE EXTERNAL TABLE sorted_offset_t (a INT NOT NULL)+STORED AS CSV+WITH ORDER (a ASC)+LOCATION 'test_files/scratch/window_limits/sorted_offset/'+OPTIONS ('format.has_header' 'false');++query I+SELECT count(*) FROM (SELECT * FROM (SELECT a FROM sorted_offset_t LIMIT 3 OFFSET 2) ORDER BY a LIMIT 1);+----+1++query TT+EXPLAIN SELECT * FROM (SELECT a FROM sorted_offset_t LIMIT 3 OFFSET 2) ORDER BY a LIMIT 1;+----+logical_plan+01)Sort: sorted_offset_t.a ASC NULLS LAST, fetch=1+02)--Limit: skip=0, fetch=3+03)----TableScan: sorted_offset_t projection=[a], fetch=3, skip=2+physical_plan+01)GlobalLimitExec: skip=2, fetch=1+02)--SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=3+03)----DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/first.csv:0..4], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/second.csv:0..4], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/first.csv:4..8], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/window_limits/sorted_offset/second.csv:4..8]]}, projection=[a], limit=3, output_ordering=[a@0 ASC NULLS LAST], file_type=csv, has_header=false++# Also cover the OFFSET-only shape from #26690.+query I+SELECT count(*) FROM (SELECT * FROM (SELECT a FROM sorted_offset_t OFFSET 2) ORDER BY a LIMIT 3);+----+3