Skip to content

Apache DataFusion SQL Query Engine

fix: sort pushed below `GlobalLimitExec` ignores `skip`, returning too few rows (#24958)

Jay Zhan16ace4f3e5f4
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index 06b57a8630..fe73518bf1 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -1531,3 +1531,32 @@ fn test_collect_left_join_keeps_hash_partitioned_build_side_coalesce() -> Result
Ok(())
}
+
+// ========================================================================
+// Limits with a `skip`
+// ========================================================================
+
+/// A sort pushed below `GlobalLimitExec` with a non-zero `skip` must ask its
+/// input for `skip + fetch` rows; asking for only `fetch` rows used to leave
+/// `LIMIT 10 OFFSET 5` with 5 result rows.
+///
+/// This checks a single pass only: the sort is inserted by `pushdown_sorts`,
+/// which runs after `parallelize_sorts`, so a second pass would additionally
+/// parallelize it into `SortPreservingMergeExec` + partitioned `SortExec`.
+#[test]
+fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {
+ let source = Arc::new(MockMultiPartitionExec::new(4));
+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));
+ let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, Some(10)));
+ let sort: Arc<dyn ExecutionPlan> =
+ Arc::new(SortExec::new(sort_expr_on("a", 0, true, true), limit));
+
+ let optimized = optimize_and_sanity_check(sort)?;
+ assert_snapshot!(plan_string(&optimized), @r"
+ GlobalLimitExec: skip=5, fetch=10
+ SortExec: TopK(fetch=15), expr=[a@0 DESC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ MockMultiPartitionExec
+ ");
+ Ok(())
+}
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index 19faf8ea9f..d7f556b90d 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -79,6 +79,38 @@ impl Default for ParentRequirements {
pub type SortPushDown = PlanContext<ParentRequirements>;
+/// Number of input rows `plan` needs from its children in order to produce
+/// the rows its parent will consume.
+///
+/// `parent_fetch` is the fetch a parent imposes on `plan`'s *output*
+/// (`ParentRequirements::fetch`). It bounds `plan.fetch()` directly because
+/// both count output rows, so the effective output fetch is
+/// `min(plan.fetch(), parent_fetch)`.
+///
+/// The input fetch is that output fetch for every operator except
+/// [`GlobalLimitExec`], which discards `skip` rows first and therefore needs
+/// `skip` more input rows. `skip` must be added *after* taking the minimum:
+/// `min(fetch + skip, parent_fetch)` would be too small whenever the parent
+/// fetch is the tighter bound. Using the bare `fetch` would be wrong too, as
+/// it would turn `LIMIT 10 OFFSET 5` into `TopK(10)` below the limit, i.e. 5
+/// result rows.
+///
+/// `skip` and `fetch` are independent `usize`s, so their sum can overflow. Like
+/// [`combine_limit`], we saturate: `usize::MAX` input rows is never a smaller
+/// bound than the real one, so the pushed-down fetch stays correct.
+///
+/// [`combine_limit`]: datafusion_common::utils::combine_limit
+fn input_fetch(
+ plan: &Arc<dyn ExecutionPlan>,
+ parent_fetch: Option<usize>,
+) -> Option<usize> {
+ let fetch = min_fetch(plan.fetch(), parent_fetch)?;
+ let skip = plan
+ .downcast_ref::<GlobalLimitExec>()
+ .map_or(0, |limit| limit.skip());
+ Some(fetch.saturating_add(skip))
+}
+
/// Assigns the ordering requirement of the root node to the its children.
pub fn assign_initial_requirements(sort_push_down: &mut SortPushDown) {
let reqs = sort_push_down.plan.required_input_ordering();
@@ -364,7 +396,7 @@ fn pushdown_sorts_helper(
// For operators that can take a sort pushdown, continue with updated
// requirements. If this node already outputs single partition (e.g. SPM),
// don't push SinglePartition to children.
- let current_fetch = sort_push_down.plan.fetch();
+ let current_fetch = input_fetch(&sort_push_down.plan, parent_fetch);
let dists = sort_push_down
.plan
.input_distribution_requirements()
@@ -379,7 +411,7 @@ fn pushdown_sorts_helper(
sort_push_down.children.iter_mut().zip(adjusted).enumerate()
{
child.data.ordering_requirement = order;
- child.data.fetch = min_fetch(current_fetch, parent_fetch);
+ child.data.fetch = current_fetch;
child.data.distribution_requirement = stronger_distribution(
&effective_dist,
dists
@@ -1204,6 +1236,7 @@ mod tests {
use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::{BinaryExpr, col};
use datafusion_physical_plan::empty::EmptyExec;
+ use datafusion_physical_plan::limit::GlobalLimitExec;
const DESC: SortOptions = SortOptions {
descending: true,
@@ -1214,6 +1247,53 @@ mod tests {
nulls_first: true,
};
+ #[test]
+ fn input_fetch_adds_skip_for_global_limit() {
+ let input = Arc::new(EmptyExec::new(child_schema()));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(input, 5, Some(10)));
+
+ assert_eq!(input_fetch(&limit, None), Some(15));
+ }
+
+ /// A parent fetch bounds the limit's *output*, so it is applied before
+ /// `skip` is added: `min(10, 3) + 5 = 8`, not `min(10 + 5, 3) = 3`.
+ #[test]
+ fn input_fetch_applies_parent_fetch_before_adding_skip() {
+ let input = Arc::new(EmptyExec::new(child_schema()));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(input, 5, Some(10)));
+
+ assert_eq!(input_fetch(&limit, Some(3)), Some(8));
+ // A looser parent fetch changes nothing.
+ assert_eq!(input_fetch(&limit, Some(20)), Some(15));
+ }
+
+ /// `OFFSET` without `LIMIT` has no fetch of its own, but a parent fetch
+ /// still needs `skip` extra input rows.
+ #[test]
+ fn input_fetch_adds_skip_to_parent_fetch_without_own_fetch() {
+ let input = Arc::new(EmptyExec::new(child_schema()));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(input, 5, None));
+
+ assert_eq!(input_fetch(&limit, None), None);
+ assert_eq!(input_fetch(&limit, Some(3)), Some(8));
+ }
+
+ /// `skip` and `fetch` are unrelated `usize`s, so `skip + fetch` can exceed
+ /// `usize::MAX`. Saturating keeps this at an (unreachable) upper bound
+ /// instead of panicking in debug builds or wrapping to a too-small fetch --
+ /// wrapping to 0 would push down a `TopK(fetch=0)` and drop every row.
+ #[test]
+ fn input_fetch_saturates_instead_of_overflowing() {
+ let input = Arc::new(EmptyExec::new(child_schema()));
+ let limit: Arc<dyn ExecutionPlan> =
+ Arc::new(GlobalLimitExec::new(input, usize::MAX, Some(1)));
+
+ assert_eq!(input_fetch(&limit, None), Some(usize::MAX));
+ }
+
/// Child (input) schema fed to the projections under test: `[a, b, c]`.
fn child_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index e17f633bc6..a7f6782b49 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -339,6 +339,29 @@ SELECT COUNT(*) FROM (SELECT a FROM t1 LIMIT 3 OFFSET 8);
----
2
+# A sort above LIMIT ... OFFSET is pushed below the limit as a TopK. The TopK
+# has to keep `skip + fetch` rows (3 + 4 = 7) so that the limit can still skip
+# 3 rows and return 4; keeping only `fetch` rows returned a single row.
+query TT
+EXPLAIN SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;
+----
+logical_plan
+01)Sort: t1.a ASC NULLS LAST
+02)--Limit: skip=3, fetch=4
+03)----TableScan: t1 projection=[a], fetch=7
+physical_plan
+01)GlobalLimitExec: skip=3, fetch=4
+02)--SortExec: TopK(fetch=7), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]
+03)----DataSourceExec: partitions=1, partition_sizes=[1]
+
+query I
+SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;
+----
+4
+5
+6
+7
+
# The aggregate does not need to be computed because the input statistics are exact and
# an OFFSET, but no LIMIT, is specified.
query TT