Skip to content

Apache DataFusion SQL Query Engine

fix: Avoid pushing down sort under limits (#25408)

Adam Gutglick7e102d74cd3b
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index 8e2c026dcd..8b1b4ff011 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -1533,18 +1533,13 @@ fn test_collect_left_join_keeps_hash_partitioned_build_side_coalesce() -> Result
}
// ========================================================================
-// Limits with a `skip`
+// Sort pushdown through limits
// ========================================================================
-/// A sort pushed below `GlobalLimitExec` with a non-zero `skip` must ask its
-/// input for `skip + fetch` rows; asking for only `fetch` rows used to leave
-/// `LIMIT 10 OFFSET 5` with 5 result rows.
-///
-/// This checks a single pass only: the sort is inserted by `pushdown_sorts`,
-/// which runs after `parallelize_sorts`, so a second pass would additionally
-/// parallelize it into `SortPreservingMergeExec` + partitioned `SortExec`.
+/// Keep sorting above an unordered limit so its input can stop early and only
+/// the rows retained after the offset need sorting.
#[test]
-fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {
+fn test_keep_sort_above_unordered_limit_with_skip() -> Result<()> {
let source = Arc::new(MockMultiPartitionExec::new(4));
let coalesce = Arc::new(CoalescePartitionsExec::new(source));
let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, Some(10)));
@@ -1553,10 +1548,121 @@ fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {
let optimized = optimize_and_sanity_check(sort)?;
assert_snapshot!(plan_string(&optimized), @r"
- GlobalLimitExec: skip=5, fetch=10
- SortExec: TopK(fetch=15), expr=[a@0 DESC], preserve_partitioning=[false]
+ SortExec: expr=[a@0 DESC], preserve_partitioning=[false]
+ GlobalLimitExec: skip=5, fetch=10
+ CoalescePartitionsExec
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
+
+/// Keep sorting above OFFSET so it skips rows in input order.
+#[test]
+fn test_keep_sort_above_limit_with_skip_only() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));
+ let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, None));
+ let sort: Arc<dyn ExecutionPlan> =
+ Arc::new(SortExec::new(sort_expr_on("a", 0, true, true), limit));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ SortExec: expr=[a@0 DESC], preserve_partitioning=[false]
+ GlobalLimitExec: skip=5, fetch=None
CoalescePartitionsExec
MockMultiPartitionExec
");
Ok(())
}
+
+/// Source ordering on `[a]` must not let a TopK on `[a, b]` change which
+/// rows the limit selects.
+#[test]
+fn test_keep_sort_above_ordered_limit_with_skip() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let mut ordering = sort_expr_on("a", 0, false, false);
+ let ordered_input = Arc::new(MockReqExec::new(
+ source,
+ Distribution::SinglePartition,
+ Some(ordering.clone()),
+ ));
+ let limit = Arc::new(GlobalLimitExec::new(ordered_input, 5, Some(10)));
+ ordering.extend(sort_expr_on("b", 1, false, false));
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+ GlobalLimitExec: skip=5, fetch=10
+ MockReqExec
+ SortPreservingMergeExec: [a@0 ASC NULLS LAST]
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
+
+/// Refine the sort below the limit, retaining `skip + fetch` rows for OFFSET.
+/// Single partition: a coalesce below the sort would be parallelized into a
+/// merge first, which the next test covers.
+#[test]
+fn test_sort_pushed_into_sort_below_limit_with_skip() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(1));
+ let mut ordering = sort_expr_on("b", 1, false, false);
+ let inner_sort = Arc::new(SortExec::new(ordering.clone(), source));
+ let limit = Arc::new(GlobalLimitExec::new(inner_sort, 5, Some(10)));
+ ordering.extend(sort_expr_on("a", 0, false, false));
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ GlobalLimitExec: skip=5, fetch=10
+ SortExec: TopK(fetch=15), expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
+
+/// The sort below the limit is parallelized into a merge over per-partition
+/// sorts before pushdown runs. The limit then sits above a merge, not a sort,
+/// so the finer sort stays above it.
+#[test]
+fn test_keep_sort_above_limit_over_merged_sort() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));
+ let mut ordering = sort_expr_on("b", 1, false, false);
+ let inner_sort = Arc::new(SortExec::new(ordering.clone(), coalesce));
+ let limit = Arc::new(GlobalLimitExec::new(inner_sort, 5, Some(10)));
+ ordering.extend(sort_expr_on("a", 0, false, false));
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, limit));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ SortExec: expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]
+ GlobalLimitExec: skip=5, fetch=10
+ SortPreservingMergeExec: [b@1 ASC NULLS LAST]
+ SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[true]
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
+
+/// Keep sorting above a per-partition limit so each partition keeps its
+/// first rows.
+#[test]
+fn test_keep_sort_above_local_limit() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let limit = Arc::new(LocalLimitExec::new(source, 10));
+ let coalesce = Arc::new(CoalescePartitionsExec::new(limit));
+ let mut ordering = sort_expr_on("a", 0, false, false);
+ ordering.extend(sort_expr_on("b", 1, false, false));
+ let sort: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(ordering, coalesce));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST]
+ SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[true]
+ LocalLimitExec: fetch=10
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 867dcb7525..2aeddd23aa 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -532,6 +532,22 @@ fn pushdown_requirement_to_children(
)
.then(|| Ok(vec![Some(parent_required)]))
.transpose()
+ } else if plan.is::<GlobalLimitExec>() || plan.is::<LocalLimitExec>() {
+ // Sorting below a limit changes which rows it keeps. Only refine a
+ // SortExec directly below it, which changes how ties are broken.
+ // Otherwise, sort the retained rows and let the input stop early.
+ let refines_sort_below = plan.children()[0].is::<SortExec>()
+ && plan.output_ordering().is_some_and(|ordering| {
+ plan.equivalence_properties().requirements_compatible(
+ parent_required.first().clone(),
+ ordering.clone().into(),
+ )
+ });
+ if refines_sort_below {
+ Ok(Some(vec![Some(parent_required)]))
+ } else {
+ Ok(None)
+ }
} else if plan.fetch().is_some()
&& plan.supports_limit_pushdown()
&& plan
@@ -562,8 +578,10 @@ fn pushdown_requirement_to_children(
} else {
parent_required.clone()
};
+
+ // Pushing a sort below an unordered fetch changes which rows it keeps.
let Some(ordering) = plan.properties().output_ordering() else {
- return Ok(Some(vec![Some(child_required)]));
+ return Ok(None);
};
if plan.properties().eq_properties.requirements_compatible(
parent_required.first().clone(),
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index 1ff6ca4fb0..06d1c85f5f 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -339,9 +339,8 @@ SELECT COUNT(*) FROM (SELECT a FROM t1 LIMIT 3 OFFSET 8);
----
2
-# A sort above LIMIT ... OFFSET is pushed below the limit as a TopK. The TopK
-# has to keep `skip + fetch` rows (3 + 4 = 7) so that the limit can still skip
-# 3 rows and return 4; keeping only `fetch` rows returned a single row.
+# Keep the sort above an unordered LIMIT ... OFFSET so the scan can stop
+# after `skip + fetch` rows (3 + 4 = 7), and only the 4 selected rows are sorted.
query TT
EXPLAIN SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;
----
@@ -350,9 +349,9 @@ logical_plan
02)--Limit: skip=3, fetch=4
03)----TableScan: t1 projection=[a], fetch=7
physical_plan
-01)GlobalLimitExec: skip=3, fetch=4
-02)--SortExec: TopK(fetch=7), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
-03)----DataSourceExec: partitions=1, partition_sizes=[1]
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--GlobalLimitExec: skip=3, fetch=4
+03)----DataSourceExec: partitions=1, partition_sizes=[1], fetch=7
query I
SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;
@@ -362,6 +361,170 @@ SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;
6
7
+# LIMIT 4 OFFSET 3 keeps input rows (1, 7, 2, 10), then sorts them.
+statement ok
+CREATE TABLE unsorted (a INT) AS VALUES (5), (3), (9), (1), (7), (2), (10), (4), (8), (6);
+
+query I
+SELECT * FROM (SELECT a FROM unsorted LIMIT 4 OFFSET 3) ORDER BY a;
+----
+1
+2
+7
+10
+
+# OFFSET skips the first 3 input rows (5, 3, 9) before sorting.
+query TT
+EXPLAIN SELECT * FROM (SELECT a FROM unsorted OFFSET 3) ORDER BY a;
+----
+logical_plan
+01)Sort: unsorted.a ASC NULLS LAST
+02)--Limit: skip=3, fetch=None
+03)----TableScan: unsorted projection=[a]
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--GlobalLimitExec: skip=3, fetch=None
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+SELECT * FROM (SELECT a FROM unsorted OFFSET 3) ORDER BY a;
+----
+1
+2
+4
+6
+7
+8
+10
+
+# Even with input ordered on `a`, a TopK on `(a, b)` below the limit would
+# change which rows it keeps.
+query I
+COPY (SELECT column1 AS a, column2 AS b FROM (VALUES
+ (1, 9), (1, 8), (1, 7), (1, 6), (1, 5), (1, 4), (1, 3), (1, 2), (2, 1), (3, 0)))
+TO 'test_files/scratch/limit/ordered_prefix.parquet' STORED AS PARQUET;
+----
+10
+
+statement ok
+CREATE EXTERNAL TABLE ordered_prefix (a BIGINT, b BIGINT)
+STORED AS PARQUET
+WITH ORDER (a ASC)
+LOCATION 'test_files/scratch/limit/ordered_prefix.parquet';
+
+query TT
+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix LIMIT 4 OFFSET 3) ORDER BY a, b;
+----
+logical_plan
+01)Sort: ordered_prefix.a ASC NULLS LAST, ordered_prefix.b ASC NULLS LAST
+02)--Limit: skip=0, fetch=4
+03)----TableScan: ordered_prefix projection=[a, b], fetch=4, skip=3
+physical_plan
+01)SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--GlobalLimitExec: skip=3, fetch=4
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], limit=7, output_ordering=[a@0 ASC NULLS LAST], file_type=parquet
+
+query II
+SELECT * FROM (SELECT a, b FROM ordered_prefix LIMIT 4 OFFSET 3) ORDER BY a, b;
+----
+1 3
+1 4
+1 5
+1 6
+
+# Refine the explicit sort to `(b, a)`. TopK keeps `skip + fetch` (3 + 4 = 7)
+# rows so the limit still returns 4.
+query TT
+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b LIMIT 4 OFFSET 3) ORDER BY b, a;
+----
+logical_plan
+01)Sort: ordered_prefix.b ASC NULLS LAST, ordered_prefix.a ASC NULLS LAST
+02)--Limit: skip=3, fetch=4
+03)----Sort: ordered_prefix.b ASC NULLS LAST, fetch=7
+04)------TableScan: ordered_prefix projection=[a, b]
+physical_plan
+01)GlobalLimitExec: skip=3, fetch=4
+02)--SortExec: TopK(fetch=7), expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], dynamic_rg_pruning=eligible
+
+query II
+SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b LIMIT 4 OFFSET 3) ORDER BY b, a;
+----
+1 3
+1 4
+1 5
+1 6
+
+# OFFSET without LIMIT: the refined sort is pushed into the explicit sort with
+# no fetch, and the limit still skips 3 rows.
+query TT
+EXPLAIN SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b OFFSET 3) ORDER BY b, a;
+----
+logical_plan
+01)Sort: ordered_prefix.b ASC NULLS LAST, ordered_prefix.a ASC NULLS LAST
+02)--Limit: skip=3, fetch=None
+03)----Sort: ordered_prefix.b ASC NULLS LAST
+04)------TableScan: ordered_prefix projection=[a, b]
+physical_plan
+01)GlobalLimitExec: skip=3, fetch=None
+02)--SortExec: expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/ordered_prefix.parquet]]}, projection=[a, b], file_type=parquet, sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST]
+
+query II
+SELECT * FROM (SELECT a, b FROM ordered_prefix ORDER BY b OFFSET 3) ORDER BY b, a;
+----
+1 3
+1 4
+1 5
+1 6
+1 7
+1 8
+1 9
+
+# A sort required by a window function, reaching the limit through a projection,
+# also stays above LIMIT ... OFFSET so the input can stop after `skip + fetch`
+# rows. It used to be pushed below the limit as a TopK over the whole input
+# (#25378).
+statement ok
+CREATE TABLE window_over_limit (id INT, kind VARCHAR) AS VALUES
+ (19, 'a'), (18, 'a'), (17, 'a'), (16, 'a'), (15, 'a'),
+ (14, 'a'), (13, 'a'), (12, 'a'), (11, 'a'), (10, 'a'),
+ (9, 'a'), (8, 'a'), (7, 'a'), (6, 'a'), (5, 'a'),
+ (4, 'a'), (3, 'a'), (2, 'a'), (1, 'a'), (0, 'a');
+
+query TT
+EXPLAIN SELECT row_number() OVER (ORDER BY id) AS rn, id
+FROM (SELECT id FROM window_over_limit WHERE kind = 'a' LIMIT 5 OFFSET 5);
+----
+logical_plan
+01)Projection: row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS rn, window_over_limit.id
+02)--WindowAggr: windowExpr=[[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]
+03)----Projection: window_over_limit.id
+04)------Limit: skip=5, fetch=5
+05)--------Filter: window_over_limit.kind = Utf8View("a")
+06)----------TableScan: window_over_limit projection=[id, kind]
+physical_plan
+01)ProjectionExec: expr=[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rn, id@0 as id]
+02)--BoundedWindowAggExec: wdw=[row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() ORDER BY [window_over_limit.id ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]
+03)----SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+04)------ProjectionExec: expr=[id@0 as id]
+05)--------GlobalLimitExec: skip=5, fetch=5
+06)----------FilterExec: kind@1 = a, fetch=10
+07)------------DataSourceExec: partitions=1, partition_sizes=[1]
+
+query II
+SELECT row_number() OVER (ORDER BY id) AS rn, id
+FROM (SELECT id FROM window_over_limit WHERE kind = 'a' LIMIT 5 OFFSET 5);
+----
+1 10
+2 11
+3 12
+4 13
+5 14
+
+statement ok
+DROP TABLE window_over_limit;
+
# The aggregate does not need to be computed because the input statistics are exact and
# an OFFSET, but no LIMIT, is specified.
query TT