Apache DataFusion SQL Query Engine
fix: Avoid pushing down sort under limits (#25408)
Adam Gutglick7e102d74cd3b
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rsindex 8e2c026dcd..8b1b4ff011 100644--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs@@ -1533,18 +1533,13 @@ fn test_collect_left_join_keeps_hash_partitioned_build_side_coalesce() -> Result}// ========================================================================-// Limits with a `skip`+// Sort pushdown through limits// ========================================================================-/// A sort pushed below `GlobalLimitExec` with a non-zero `skip` must ask its-/// input for `skip + fetch` rows; asking for only `fetch` rows used to leave-/// `LIMIT 10 OFFSET 5` with 5 result rows.-///-/// This checks a single pass only: the sort is inserted by `pushdown_sorts`,-/// which runs after `parallelize_sorts`, so a second pass would additionally-/// parallelize it into `SortPreservingMergeExec` + partitioned `SortExec`.+/// Keep sorting above an unordered limit so its input can stop early and only+/// the rows retained after the offset need sorting.#[test]-fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {+fn test_keep_sort_above_unordered_limit_with_skip() -> Result<()> {let source = Arc::new(MockMultiPartitionExec::new(4));let coalesce = Arc::new(CoalescePartitionsExec::new(source));let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, Some(10)));@@ -1553,10 +1548,121 @@ fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {let optimized = optimize_and_sanity_check(sort)?;assert_snapshot!(plan_string(&optimized), @r"- GlobalLimitExec: skip=5, fetch=10- SortExec: TopK(fetch=15), expr=[a@0 DESC], preserve_partitioning=[false]+ SortExec: expr=[a@0 DESC], preserve_partitioning=[false]+ GlobalLimitExec: skip=5, fetch=10+ CoalescePartitionsExec+ MockMultiPartitionExec+ ");+ Ok(())+}++/// Keep sorting above OFFSET so it skips rows in input order.+#[test]+fn test_keep_sort_above_limit_with_skip_only() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(4));+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));+ let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, None));+ let sort: Arc<dyn ExecutionPlan> =+ Arc::new(SortExec::new(sort_expr_on("a", 0, true, true), limit));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ SortExec: expr=[a@0 DESC], preserve_partitioning=[false]+ GlobalLimitExec: skip=5, fetch=NoneCoalescePartitionsExecMockMultiPartitionExec");Ok(())}++/// Source ordering on `[a]` must not let a TopK on `[a, b]` change which+/// rows the limit selects.+#[test]+fn test_keep_sort_above_ordered_limit_with_skip() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(4));+ let mut ordering = sort_expr_on("a", 0, false, false);+ let ordered_input = Arc::new(MockReqExec::new(+ source,+ Distribution::SinglePartition,+ Some(ordering.clone()),+ ));+ let limit = Arc::new(GlobalLimitExec::new(ordered_input, 5, Some(10)));+ ordering.extend(sort_expr_on("b", 1, false, false));+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]+ GlobalLimitExec: skip=5, fetch=10+ MockReqExec+ SortPreservingMergeExec: [a@0 ASC NULLS LAST]+ MockMultiPartitionExec+ ");+ Ok(())+}++/// Refine the sort below the limit, retaining `skip + fetch` rows for OFFSET.+/// Single partition: a coalesce below the sort would be parallelized into a+/// merge first, which the next test covers.+#[test]+fn test_sort_pushed_into_sort_below_limit_with_skip() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(1));+ let mut ordering = sort_expr_on("b", 1, false, false);+ let inner_sort = Arc::new(SortExec::new(ordering.clone(), source));+ let limit = Arc::new(GlobalLimitExec::new(inner_sort, 5, Some(10)));+ ordering.extend(sort_expr_on("a", 0, false, false));+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ GlobalLimitExec: skip=5, fetch=10+ SortExec: TopK(fetch=15), expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]+ MockMultiPartitionExec+ ");+ Ok(())+}++/// The sort below the limit is parallelized into a merge over per-partition+/// sorts before pushdown runs. The limit then sits above a merge, not a sort,+/// so the finer sort stays above it.+#[test]+fn test_keep_sort_above_limit_over_merged_sort() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(4));+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));+ let mut ordering = sort_expr_on("b", 1, false, false);+ let inner_sort = Arc::new(SortExec::new(ordering.clone(), coalesce));+ let limit = Arc::new(GlobalLimitExec::new(inner_sort, 5, Some(10)));+ ordering.extend(sort_expr_on("a", 0, false, false));+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ SortExec: expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]+ GlobalLimitExec: skip=5, fetch=10+ SortPreservingMergeExec: [b@1 ASC NULLS LAST]+ SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[true]+ MockMultiPartitionExec+ ");+ Ok(())+}++/// Keep sorting above a per-partition limit so each partition keeps its+/// first rows.+#[test]+fn test_keep_sort_above_local_limit() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(4));+ let limit = Arc::new(LocalLimitExec::new(source, 10));+ let coalesce = Arc::new(CoalescePartitionsExec::new(limit));+ let mut ordering = sort_expr_on("a", 0, false, false);+ ordering.extend(sort_expr_on("b", 1, false, false));+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, coalesce));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST]+ SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[true]+ LocalLimitExec: fetch=10+ MockMultiPartitionExec+ ");+ Ok(())+}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex 867dcb7525..2aeddd23aa 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -532,6 +532,22 @@ fn pushdown_requirement_to_children().then(|| Ok(vec![Some(parent_required)])).transpose()+ } else if plan.is::<GlobalLimitExec>() || plan.is::<LocalLimitExec>() {+ // Sorting below a limit changes which rows it keeps. Only refine a+ // SortExec directly below it, which changes how ties are broken.+ // Otherwise, sort the retained rows and let the input stop early.+ let refines_sort_below = plan.children()[0].is::<SortExec>()+ && plan.output_ordering().is_some_and(|ordering| {+ plan.equivalence_properties().requirements_compatible(+ parent_required.first().clone(),+ ordering.clone().into(),+ )+ });+ if refines_sort_below {+ Ok(Some(vec![Some(parent_required)]))+ } else {+ Ok(None)+ }} else if plan.fetch().is_some()&& plan.supports_limit_pushdown()&& plan@@ -562,8 +578,10 @@ fn pushdown_requirement_to_children(} else {parent_required.clone()};++ // Pushing a sort below an unordered fetch changes which rows it keeps.let Some(ordering) = plan.properties().output_ordering() else {- return Ok(Some(vec![Some(child_required)]));+ return Ok(None);};if plan.properties().eq_properties.requirements_compatible(parent_required.first().clone(),diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex 1ff6ca4fb0..06d1c85f5f 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -339,9 +339,8 @@ SELECT COUNT(*) FROM (SELECT a FROM t1 LIMIT 3 OFFSET 8);----2-# A sort above LIMIT ... OFFSET is pushed below the limit as a TopK. The TopK-# has to keep `skip + fetch` rows (3 + 4 = 7) so that the limit can still skip-# 3 rows and return 4; keeping only `fetch` rows returned a single row.+# Keep the sort above an unordered LIMIT ... OFFSET so the scan can stop+# after `skip + fetch` rows (3 + 4 = 7), and only the 4 selected rows are sorted.query TTEXPLAIN SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;----@@ -350,9 +349,9 @@ logical_plan02)--Limit: skip=3, fetch=403)----TableScan: t1 projection=[a], fetch=7physical_plan-01)GlobalLimitExec: skip=3, fetch=4-02)--SortExec: TopK(fetch=7), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]-03)----DataSourceExec: partitions=1, partition_sizes=[1]+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--GlobalLimitExec: skip=3, fetch=4+03)----DataSourceExec: partitions=1, partition_sizes=[1], fetch=7query ISELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;@@ -362,6 +361,170 @@ SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;67+# LIMIT 4 OFFSET 3 keeps input rows (1, 7, 2, 10), then sorts them.+statement ok+CREATE TABLE unsorted (a INT) AS VALUES (5), (3), (9), (1), (7), (2), (10), (4), (8), (6);++query I+SELECT * FROM (SELECT a FROM unsorted LIMIT 4 OFFSET 3) ORDER BY a;+----+1+2+7+10++# OFFSET skips the first 3 input rows (5, 3, 9) before sorting.+query TT+EXPLAIN SELECT * FROM (SELECT a FROM unsorted OFFSET 3) ORDER BY a;+----+logical_plan+01)Sort: unsorted.a ASC NULLS LAST+02)--Limit: skip=3, fetch=None+03)----TableScan: unsorted projection=[a]+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--GlobalLimitExec: skip=3, fetch=None+03)----DataSourceExec: partitions=1, partition_sizes=[1]++query I+SELECT * FROM (SELECT a FROM unsorted OFFSET 3) ORDER BY a;+----+1+2+4+6+7+8+10++# Even with input ordered on `a`, a TopK on `(a, b)` below the limit would+# change which rows it keeps.+query I+COPY (SELECT column1 AS a, column2 AS b FROM (VALUES+ (1, 9), (1, 8), (1, 7), (1, 6), (1, 5), (1, 4), (1, 3), (1, 2), (2, 1), (3, 0)))+TO 'test_files/scratch/limit/ordered_prefix.parquet' STORED AS PARQUET;+----+10++statement ok+CREATE EXTERNAL TABLE ordered_prefix (a BIGINT, b BIGINT)+STORED AS PARQUET+WITH ORDER (a ASC)+LOCATION 'test_files/scratch/limit/ordered_prefix.parquet';++query TT+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix LIMIT 4 OFFSET 3) ORDER BY a, b;+----+logical_plan+01)Sort: ordered_prefix.a ASC NULLS LAST, ordered_prefix.b ASC NULLS LAST+02)--Limit: skip=0, fetch=4+03)----TableScan: ordered_prefix projection=[a, b], fetch=4, skip=3+physical_plan+01)SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--GlobalLimitExec: skip=3, fetch=4+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], limit=7, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet++query II+SELECT * FROM (SELECT a, b FROM ordered_prefix LIMIT 4 OFFSET 3) ORDER BY a, b;+----+1 3+1 4+1 5+1 6++# Refine the explicit sort to `(b, a)`. TopK keeps `skip + fetch` (3 + 4 = 7)+# rows so the limit still returns 4.+query TT+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b LIMIT 4 OFFSET 3) ORDER BY b, a;+----+logical_plan+01)Sort: ordered_prefix.b ASC NULLS LAST, ordered_prefix.a ASC NULLS LAST+02)--Limit: skip=3, fetch=4+03)----Sort: ordered_prefix.b ASC NULLS LAST, fetch=7+04)------TableScan: ordered_prefix projection=[a, b]+physical_plan+01)GlobalLimitExec: skip=3, fetch=4+02)--SortExec: TopK(fetch=7), expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible++query II+SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b LIMIT 4 OFFSET 3) ORDER BY b, a;+----+1 3+1 4+1 5+1 6++# OFFSET without LIMIT: the refined sort is pushed into the explicit sort with+# no fetch, and the limit still skips 3 rows.+query TT+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b OFFSET 3) ORDER BY b, a;+----+logical_plan+01)Sort: ordered_prefix.b ASC NULLS LAST, ordered_prefix.a ASC NULLS LAST+02)--Limit: skip=3, fetch=None+03)----Sort: ordered_prefix.b ASC NULLS LAST+04)------TableScan: ordered_prefix projection=[a, b]+physical_plan+01)GlobalLimitExec: skip=3, fetch=None+02)--SortExec: expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], file_type=parquet, sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST]++query II+SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b OFFSET 3) ORDER BY b, a;+----+1 3+1 4+1 5+1 6+1 7+1 8+1 9++# A sort required by a window function, reaching the limit through a projection,+# also stays above LIMIT ... OFFSET so the input can stop after `skip + fetch`+# rows. It used to be pushed below the limit as a TopK over the whole input+# (#25378).+statement ok+CREATE TABLE window_over_limit (id INT, kind VARCHAR) AS VALUES+ (19, 'a'), (18, 'a'), (17, 'a'), (16, 'a'), (15, 'a'),+ (14, 'a'), (13, 'a'), (12, 'a'), (11, 'a'), (10, 'a'),+ (9, 'a'), (8, 'a'), (7, 'a'), (6, 'a'), (5, 'a'),+ (4, 'a'), (3, 'a'), (2, 'a'), (1, 'a'), (0, 'a');++query TT+EXPLAIN SELECT row_number() OVER (ORDER BY id) AS rn, id+FROM (SELECT id FROM window_over_limit WHERE kind = 'a' LIMIT 5 OFFSET 5);+----+logical_plan+01)Projection: row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn, window_over_limit.id+02)--WindowAggr: windowExpr=[[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]+03)----Projection: window_over_limit.id+04)------Limit: skip=5, fetch=5+05)--------Filter: window_over_limit.kind = Utf8View("a")+06)----------TableScan: window_over_limit projection=[id, kind]+physical_plan+01)ProjectionExec: expr=[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rn, id@0 as id]+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+03)----SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+04)------ProjectionExec: expr=[id@0 as id]+05)--------GlobalLimitExec: skip=5, fetch=5+06)----------FilterExec: kind@1 = a, fetch=10+07)------------DataSourceExec: partitions=1, partition_sizes=[1]++query II+SELECT row_number() OVER (ORDER BY id) AS rn, id+FROM (SELECT id FROM window_over_limit WHERE kind = 'a' LIMIT 5 OFFSET 5);+----+1 10+2 11+3 12+4 13+5 14++statement ok+DROP TABLE window_over_limit;+# The aggregate does not need to be computed because the input statistics are exact and# an OFFSET, but no LIMIT, is specified.query TT