Apache DataFusion SQL Query Engine
fix: sort pushed below `GlobalLimitExec` ignores `skip`, returning too few rows (#24958)
Jay Zhan16ace4f3e5f4
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rsindex 06b57a8630..fe73518bf1 100644--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs@@ -1531,3 +1531,32 @@ fn test_collect_left_join_keeps_hash_partitioned_build_side_coalesce() -> ResultOk(())}++// ========================================================================+// Limits with a `skip`+// ========================================================================++/// A sort pushed below `GlobalLimitExec` with a non-zero `skip` must ask its+/// input for `skip + fetch` rows; asking for only `fetch` rows used to leave+/// `LIMIT 10 OFFSET 5` with 5 result rows.+///+/// This checks a single pass only: the sort is inserted by `pushdown_sorts`,+/// which runs after `parallelize_sorts`, so a second pass would additionally+/// parallelize it into `SortPreservingMergeExec` + partitioned `SortExec`.+#[test]+fn test_sort_pushed_below_limit_with_skip_keeps_skip_rows() -> Result<()> {+ let source = Arc::new(MockMultiPartitionExec::new(4));+ let coalesce = Arc::new(CoalescePartitionsExec::new(source));+ let limit = Arc::new(GlobalLimitExec::new(coalesce, 5, Some(10)));+ let sort: Arc<dyn ExecutionPlan> =+ Arc::new(SortExec::new(sort_expr_on("a", 0, true, true), limit));++ let optimized = optimize_and_sanity_check(sort)?;+ assert_snapshot!(plan_string(&optimized), @r"+ GlobalLimitExec: skip=5, fetch=10+ SortExec: TopK(fetch=15), expr=[a@0 DESC], preserve_partitioning=[false]+ CoalescePartitionsExec+ MockMultiPartitionExec+ ");+ Ok(())+}diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rsindex 19faf8ea9f..d7f556b90d 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs@@ -79,6 +79,38 @@ impl Default for ParentRequirements {pub type SortPushDown = PlanContext<ParentRequirements>;+/// Number of input rows `plan` needs from its children in order to produce+/// the rows its parent will consume.+///+/// `parent_fetch` is the fetch a parent imposes on `plan`'s *output*+/// (`ParentRequirements::fetch`). It bounds `plan.fetch()` directly because+/// both count output rows, so the effective output fetch is+/// `min(plan.fetch(), parent_fetch)`.+///+/// The input fetch is that output fetch for every operator except+/// [`GlobalLimitExec`], which discards `skip` rows first and therefore needs+/// `skip` more input rows. `skip` must be added *after* taking the minimum:+/// `min(fetch + skip, parent_fetch)` would be too small whenever the parent+/// fetch is the tighter bound. Using the bare `fetch` would be wrong too, as+/// it would turn `LIMIT 10 OFFSET 5` into `TopK(10)` below the limit, i.e. 5+/// result rows.+///+/// `skip` and `fetch` are independent `usize`s, so their sum can overflow. Like+/// [`combine_limit`], we saturate: `usize::MAX` input rows is never a smaller+/// bound than the real one, so the pushed-down fetch stays correct.+///+/// [`combine_limit`]: datafusion_common::utils::combine_limit+fn input_fetch(+ plan: &Arc<dyn ExecutionPlan>,+ parent_fetch: Option<usize>,+) -> Option<usize> {+ let fetch = min_fetch(plan.fetch(), parent_fetch)?;+ let skip = plan+ .downcast_ref::<GlobalLimitExec>()+ .map_or(0, |limit| limit.skip());+ Some(fetch.saturating_add(skip))+}+/// Assigns the ordering requirement of the root node to the its children.pub fn assign_initial_requirements(sort_push_down: &mut SortPushDown) {let reqs = sort_push_down.plan.required_input_ordering();@@ -364,7 +396,7 @@ fn pushdown_sorts_helper(// For operators that can take a sort pushdown, continue with updated// requirements. If this node already outputs single partition (e.g. SPM),// don't push SinglePartition to children.- let current_fetch = sort_push_down.plan.fetch();+ let current_fetch = input_fetch(&sort_push_down.plan, parent_fetch);let dists = sort_push_down.plan.input_distribution_requirements()@@ -379,7 +411,7 @@ fn pushdown_sorts_helper(sort_push_down.children.iter_mut().zip(adjusted).enumerate(){child.data.ordering_requirement = order;- child.data.fetch = min_fetch(current_fetch, parent_fetch);+ child.data.fetch = current_fetch;child.data.distribution_requirement = stronger_distribution(&effective_dist,dists@@ -1204,6 +1236,7 @@ mod tests {use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::{BinaryExpr, col};use datafusion_physical_plan::empty::EmptyExec;+ use datafusion_physical_plan::limit::GlobalLimitExec;const DESC: SortOptions = SortOptions {descending: true,@@ -1214,6 +1247,53 @@ mod tests {nulls_first: true,};+ #[test]+ fn input_fetch_adds_skip_for_global_limit() {+ let input = Arc::new(EmptyExec::new(child_schema()));+ let limit: Arc<dyn ExecutionPlan> =+ Arc::new(GlobalLimitExec::new(input, 5, Some(10)));++ assert_eq!(input_fetch(&limit, None), Some(15));+ }++ /// A parent fetch bounds the limit's *output*, so it is applied before+ /// `skip` is added: `min(10, 3) + 5 = 8`, not `min(10 + 5, 3) = 3`.+ #[test]+ fn input_fetch_applies_parent_fetch_before_adding_skip() {+ let input = Arc::new(EmptyExec::new(child_schema()));+ let limit: Arc<dyn ExecutionPlan> =+ Arc::new(GlobalLimitExec::new(input, 5, Some(10)));++ assert_eq!(input_fetch(&limit, Some(3)), Some(8));+ // A looser parent fetch changes nothing.+ assert_eq!(input_fetch(&limit, Some(20)), Some(15));+ }++ /// `OFFSET` without `LIMIT` has no fetch of its own, but a parent fetch+ /// still needs `skip` extra input rows.+ #[test]+ fn input_fetch_adds_skip_to_parent_fetch_without_own_fetch() {+ let input = Arc::new(EmptyExec::new(child_schema()));+ let limit: Arc<dyn ExecutionPlan> =+ Arc::new(GlobalLimitExec::new(input, 5, None));++ assert_eq!(input_fetch(&limit, None), None);+ assert_eq!(input_fetch(&limit, Some(3)), Some(8));+ }++ /// `skip` and `fetch` are unrelated `usize`s, so `skip + fetch` can exceed+ /// `usize::MAX`. Saturating keeps this at an (unreachable) upper bound+ /// instead of panicking in debug builds or wrapping to a too-small fetch --+ /// wrapping to 0 would push down a `TopK(fetch=0)` and drop every row.+ #[test]+ fn input_fetch_saturates_instead_of_overflowing() {+ let input = Arc::new(EmptyExec::new(child_schema()));+ let limit: Arc<dyn ExecutionPlan> =+ Arc::new(GlobalLimitExec::new(input, usize::MAX, Some(1)));++ assert_eq!(input_fetch(&limit, None), Some(usize::MAX));+ }+/// Child (input) schema fed to the projections under test: `[a, b, c]`.fn child_schema() -> Arc<Schema> {Arc::new(Schema::new(vec![diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex e17f633bc6..a7f6782b49 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -339,6 +339,29 @@ SELECT COUNT(*) FROM (SELECT a FROM t1 LIMIT 3 OFFSET 8);----2+# A sort above LIMIT ... OFFSET is pushed below the limit as a TopK. The TopK+# has to keep `skip + fetch` rows (3 + 4 = 7) so that the limit can still skip+# 3 rows and return 4; keeping only `fetch` rows returned a single row.+query TT+EXPLAIN SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;+----+logical_plan+01)Sort: t1.a ASC NULLS LAST+02)--Limit: skip=3, fetch=4+03)----TableScan: t1 projection=[a], fetch=7+physical_plan+01)GlobalLimitExec: skip=3, fetch=4+02)--SortExec: TopK(fetch=7), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[1]++query I+SELECT * FROM (SELECT a FROM t1 LIMIT 4 OFFSET 3) ORDER BY a;+----+4+5+6+7+# The aggregate does not need to be computed because the input statistics are exact and# an OFFSET, but no LIMIT, is specified.query TT