Skip to content

Apache DataFusion SQL Query Engine

fix: decline sort pushdown through SortPreservingMergeExec with fetch (#26162) (#27060)

Sanghoon3f4a37fbabdc
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index 86adb307db..0d779a7430 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -522,6 +522,32 @@ fn test_fetched_merge_keeps_stricter_topk_above() {
");
}
+/// A merge guarantees only its own keys across partitions, so a finer
+/// requirement must stay above it.
+#[test]
+fn test_finer_sort_stays_above_merge_without_fetch() {
+ let source: Arc<dyn ExecutionPlan> = Arc::new(MockMultiPartitionExec::new(4));
+ let a = sort_expr_on("a", 0, false, false);
+ let a_b = LexOrdering::new(
+ a.iter()
+ .cloned()
+ .chain(sort_expr_on("b", 1, false, false).iter().cloned()),
+ )
+ .unwrap();
+ let merge = Arc::new(SortPreservingMergeExec::new(a, source));
+ let plan: Arc<dyn ExecutionPlan> = Arc::new(SortExec::new(a_b, merge));
+
+ let mut sort_push_down = SortPushDown::new_default(plan);
+ assign_initial_requirements(&mut sort_push_down);
+ let pushed_down = plan_string(&pushdown_sorts(sort_push_down).unwrap().plan);
+
+ assert_snapshot!(pushed_down, @r"
+ SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+ SortPreservingMergeExec: [a@0 ASC NULLS LAST]
+ MockMultiPartitionExec
+ ");
+}
+
// ========================================================================
// Multiple sort columns test
// ========================================================================
diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
index f2a01598b9..985ec3632d 100644
--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs
@@ -5,10 +5,7 @@
use std::fmt::Debug;
use std::sync::Arc;
-use crate::utils::{
- add_sort_above_with_distribution, is_sort, is_sort_preserving_merge, is_union,
- is_window,
-};
+use crate::utils::{add_sort_above_with_distribution, is_sort, is_union, is_window};
use arrow::datatypes::SchemaRef;
use datafusion_common::tree_node::{Transformed, TreeNode};
@@ -524,21 +521,10 @@ fn pushdown_requirement_to_children(
} else {
Ok(None)
}
- } else if plan.is::<SortPreservingMergeExec>() && plan.fetch().is_some() {
- // Push down through a fetched merge when:
- // - requirement is aligned with output ordering
- // - it preserves ordering during execution
- // Refining a fetched merge must preserve the ordering that selected its rows.
- let Some(ordering) = plan.output_ordering() else {
- return Ok(None);
- };
- Ok(plan
- .equivalence_properties()
- .requirements_compatible(
- parent_required.first().clone(),
- ordering.clone().into(),
- )
- .then(|| vec![Some(parent_required)]))
+ } else if plan.is::<SortPreservingMergeExec>() {
+ // A merge orders rows across partitions only by its own keys, and this
+ // function doesn't rewrite them. Keep the finer parent sort above it.
+ Ok(None)
} else if is_union(plan) {
// `UnionExec` does not have real sort requirements for its input, we
// just propagate the sort requirements down:
@@ -593,21 +579,6 @@ fn pushdown_requirement_to_children(
// Keep sorts above RepartitionExec even when a single input partition
// lets it preserve order; pushing the sort below it provides no benefit.
Ok(None)
- } else if is_sort_preserving_merge(plan) {
- let new_ordering = LexOrdering::from(parent_required.first().clone());
- let mut spm_eqs = plan.equivalence_properties().clone();
- let old_ordering = spm_eqs.output_ordering().unwrap();
- // Sort preserving merge will have new ordering, one requirement above is pushed down to its below.
- let change = spm_eqs.reorder(new_ordering)?;
- if !change || spm_eqs.ordering_satisfy(old_ordering)? {
- // Can push-down through SortPreservingMergeExec, because parent requirement is finer
- // than SortPreservingMergeExec output ordering.
- Ok(Some(vec![Some(parent_required)]))
- } else {
- // Do not push-down through SortPreservingMergeExec when
- // ordering requirement invalidates requirement of sort preserving merge exec.
- Ok(None)
- }
} else if let Some(hash_join) = plan.downcast_ref::<HashJoinExec>() {
handle_hash_join(hash_join, parent_required)
} else {
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index a97ce9efab..83b59d0f0b 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -1331,3 +1331,74 @@ SET datafusion.optimizer.max_passes = 3;
statement ok
DROP TABLE t26431;
+
+# Test for #26162: ORDER BY b, a over a subquery with ORDER BY b LIMIT n
+# Ensure SortPreservingMergeExec with fetch does not push down finer sort requirements
+statement ok
+CREATE TABLE t_26162 AS
+ SELECT 4 AS a, 1 AS b UNION ALL SELECT 8, 2 UNION ALL SELECT 12, 3
+ UNION ALL SELECT 3, 1 UNION ALL SELECT 7, 2 UNION ALL SELECT 11, 3
+ UNION ALL SELECT 2, 1 UNION ALL SELECT 6, 2 UNION ALL SELECT 10, 3
+ UNION ALL SELECT 1, 1 UNION ALL SELECT 5, 2 UNION ALL SELECT 9, 3;
+
+query II
+SELECT * FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 8) ORDER BY b, a;
+----
+1 1
+2 1
+3 1
+4 1
+5 2
+6 2
+7 2
+8 2
+
+query TT
+EXPLAIN SELECT * FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 8) ORDER BY b, a;
+----
+logical_plan
+01)Sort: t_26162.b ASC NULLS LAST, t_26162.a ASC NULLS LAST
+02)--Sort: t_26162.b ASC NULLS LAST, fetch=8
+03)----TableScan: t_26162 projection=[a, b]
+physical_plan
+01)SortExec: expr=[b@1 ASC NULLS LAST, a@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--SortPreservingMergeExec: [b@1 ASC NULLS LAST], fetch=8
+03)----SortExec: TopK(fetch=8), expr=[b@1 ASC NULLS LAST], preserve_partitioning=[true]
+04)------DataSourceExec: partitions=12, partition_sizes=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+
+# Requirements from a window, an ordered aggregate and a UNION also reach the merge.
+query III
+SELECT a, b, row_number() OVER (ORDER BY b, a) FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 8) ORDER BY b, a;
+----
+1 1 1
+2 1 2
+3 1 3
+4 1 4
+5 2 5
+6 2 6
+7 2 7
+8 2 8
+
+query ?
+SELECT array_agg(a ORDER BY b, a) FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 8);
+----
+[1, 2, 3, 4, 5, 6, 7, 8]
+
+query II
+SELECT * FROM (
+ SELECT * FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 4)
+ UNION ALL
+ SELECT * FROM (SELECT a, b FROM t_26162 ORDER BY b LIMIT 4)
+) ORDER BY b, a;
+----
+1 1
+1 1
+2 1
+2 1
+3 1
+3 1
+4 1
+4 1
+
+statement ok
+DROP TABLE t_26162;