Apache DataFusion SQL Query Engine
fix: preserve local TopK limits during exact sort pushdown (#26736)
Geoffrey Claude8ad8490c8bda
diff --git a/datafusion/physical-optimizer/src/pushdown_sort.rs b/datafusion/physical-optimizer/src/pushdown_sort.rsindex 556cde584d..3f2a4f1921 100644--- a/datafusion/physical-optimizer/src/pushdown_sort.rs+++ b/datafusion/physical-optimizer/src/pushdown_sort.rs@@ -160,11 +160,15 @@ impl PhysicalOptimizerRule for PushdownSort {//// If the SortExec carried a fetch (LIMIT), we must preserve it.// First try pushing the limit into the source via `with_fetch()`.- // If the source doesn't support `with_fetch`, fall back to- // wrapping with GlobalLimitExec.+ // If the source doesn't support `with_fetch`, preserve whether+ // the limit applies to each partition or to the entire input.if let Some(fetch) = sort_exec.fetch() {let inner = inner.with_fetch(Some(fetch)).unwrap_or_else(|| {- Arc::new(GlobalLimitExec::new(inner, 0, Some(fetch)))+ if sort_exec.preserve_partitioning() {+ Arc::new(LocalLimitExec::new(inner, fetch))+ } else {+ Arc::new(GlobalLimitExec::new(inner, 0, Some(fetch)))+ }});Ok(Transformed::yes(inner))} else {diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rsindex 87b9034035..ffcff02cdd 100644--- a/datafusion/sqllogictest/src/test_context.rs+++ b/datafusion/sqllogictest/src/test_context.rs@@ -59,6 +59,7 @@ use tempfile::TempDir;mod custom_sort_pushdown;mod evaluation_counter;+mod exact_sort_pushdown;mod range_partitioning;/// Context for running tests@@ -168,6 +169,8 @@ impl TestContext {test_ctx.session_ctx(),).expect("register custom projection table");+ exact_sort_pushdown::register_exact_sort_source(test_ctx.session_ctx())+ .expect("register exact sort source");}"parquet_missing_bounds.slt" => {register_parquet_missing_bounds(&mut test_ctx).await;diff --git a/datafusion/sqllogictest/src/test_context/exact_sort_pushdown.rs b/datafusion/sqllogictest/src/test_context/exact_sort_pushdown.rsnew file mode 100644index 0000000000..1587f7ff85--- /dev/null+++ b/datafusion/sqllogictest/src/test_context/exact_sort_pushdown.rs@@ -0,0 +1,215 @@+// SPDX-License-Identifier: Apache-2.0+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements++use std::fmt::{self, Formatter};+use std::sync::Arc;++use arrow::array::record_batch;+use arrow::datatypes::{self as arrow_schema, SchemaRef};+use async_trait::async_trait;+use datafusion::catalog::{Session, TableProvider};+use datafusion::common::Result;+use datafusion::common::tree_node::TreeNodeRecursion;+use datafusion::execution::{SendableRecordBatchStream, TaskContext};+use datafusion::logical_expr::{Expr, TableType};+use datafusion::physical_expr::expressions::Column;+use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr};+use datafusion::physical_plan::test::TestMemoryExec;+use datafusion::physical_plan::{+ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SortOrderPushdownResult,+};+use datafusion::prelude::SessionContext;++pub(super) fn register_exact_sort_source(ctx: &SessionContext) -> Result<()> {+ ctx.register_table("exact_sort_source", Arc::new(exact_sort_source(4)?))?;+ Ok(())+}++fn exact_sort_source(partition_count: i32) -> Result<ExactSortSource> {+ let partitions = (0..partition_count)+ .map(|i| {+ Ok(vec![record_batch!(+ ("group_id", Int32, [0, 0, 1, 1]),+ ("value", Int32, [i, i + 4, i + 8, i + 12])+ )?])+ })+ .collect::<Result<Vec<_>>>()?;+ let input = TestMemoryExec::try_new(&partitions, partitions[0][0].schema(), None)?;+ Ok(ExactSortSource(input))+}++fn ordering() -> LexOrdering {+ [+ PhysicalSortExpr::new_default(Arc::new(Column::new("group_id", 0))).nulls_last(),+ PhysicalSortExpr::new_default(Arc::new(Column::new("value", 1))).nulls_last(),+ ]+ .into()+}++/// Accepts exact sort pushdown without supporting fetch pushdown. Its rows are+/// already sorted within each partition; the hook makes that ordering explicit.+#[derive(Debug, Clone)]+struct ExactSortSource(TestMemoryExec);++#[async_trait]+impl TableProvider for ExactSortSource {+ fn schema(&self) -> SchemaRef {+ self.0.schema()+ }++ fn table_type(&self) -> TableType {+ TableType::Base+ }++ async fn scan(+ &self,+ _: &dyn Session,+ projection: Option<&[usize]>,+ _: &[Expr],+ _: Option<usize>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ assert!(projection.is_none_or(|indices| indices == [0, 1]));+ Ok(Arc::new(self.clone()))+ }+}++impl DisplayAs for ExactSortSource {+ fn fmt_as(&self, _: DisplayFormatType, f: &mut Formatter) -> fmt::Result {+ write!(f, "ExactSortSource")+ }+}++impl ExecutionPlan for ExactSortSource {+ fn name(&self) -> &str {+ "ExactSortSource"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ self.0.properties()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ assert!(children.is_empty());+ Ok(self)+ }++ fn apply_expressions(+ &self,+ _: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }++ fn execute(+ &self,+ partition: usize,+ context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ self.0.execute(partition, context)+ }++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ if order != ordering().as_ref() {+ return Ok(SortOrderPushdownResult::Unsupported);+ }+ Ok(SortOrderPushdownResult::Exact {+ inner: Arc::new(Self(+ self.0.clone().try_with_sort_information(vec![ordering()])?,+ )),+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use datafusion::common::{+ assert_batches_eq, assert_batches_sorted_eq, config::ConfigOptions,+ };+ use datafusion::physical_optimizer::PhysicalOptimizerRule;+ use datafusion::physical_optimizer::pushdown_sort::PushdownSort;+ use datafusion::physical_plan::sorts::sort::SortExec;+ use datafusion::physical_plan::{ExecutionPlanProperties, collect, get_plan_string};++ #[tokio::test]+ async fn exact_pushdown_preserves_global_fetch() -> Result<()> {+ let source = Arc::new(exact_sort_source(1)?);+ let plan = Arc::new(SortExec::new(ordering(), source).with_fetch(Some(3)));+ let mut config = ConfigOptions::new();+ config.optimizer.enable_sort_pushdown = true;+ let optimized = PushdownSort::new().optimize(plan, &config)?;++ // LocalLimitExec returns the same rows on one partition, so check the plan.+ assert_eq!(+ get_plan_string(&optimized),+ ["GlobalLimitExec: skip=0, fetch=3", " ExactSortSource"]+ );++ let batches = collect(optimized, SessionContext::new().task_ctx()).await?;+ assert_batches_eq!(+ [+ "+----------+-------+",+ "| group_id | value |",+ "+----------+-------+",+ "| 0 | 0 |",+ "| 0 | 4 |",+ "| 1 | 8 |",+ "+----------+-------+",+ ],+ &batches+ );+ Ok(())+ }++ #[tokio::test]+ async fn exact_pushdown_without_fetch_preserves_all_rows() -> Result<()> {+ let source = Arc::new(exact_sort_source(4)?);+ let plan =+ Arc::new(SortExec::new(ordering(), source).with_preserve_partitioning(true));+ let mut config = ConfigOptions::new();+ config.optimizer.enable_sort_pushdown = true;+ let optimized = PushdownSort::new().optimize(plan, &config)?;++ assert_eq!(get_plan_string(&optimized), ["ExactSortSource"]);+ assert_eq!(optimized.output_partitioning().partition_count(), 4);++ let batches = collect(optimized, SessionContext::new().task_ctx()).await?;+ assert_batches_sorted_eq!(+ [+ "+----------+-------+",+ "| group_id | value |",+ "+----------+-------+",+ "| 0 | 0 |",+ "| 0 | 1 |",+ "| 0 | 2 |",+ "| 0 | 3 |",+ "| 0 | 4 |",+ "| 0 | 5 |",+ "| 0 | 6 |",+ "| 0 | 7 |",+ "| 1 | 8 |",+ "| 1 | 9 |",+ "| 1 | 10 |",+ "| 1 | 11 |",+ "| 1 | 12 |",+ "| 1 | 13 |",+ "| 1 | 14 |",+ "| 1 | 15 |",+ "+----------+-------+",+ ],+ &batches+ );+ Ok(())+ }+}diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltindex 33b9f45e5b..ed55fd8c8d 100644--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -156,6 +156,54 @@ SET datafusion.execution.parquet.pushdown_filters = true;statement okSET datafusion.optimizer.enable_sort_pushdown = true;+# exact_sort_source has four partitions with interleaved values 0..15.+# Values 0..7 belong to group 0; values 8..15 belong to group 1.+# It supports exact (group_id, value) ordering, but cannot apply a limit itself.+#+# The partitioned window requires redistributing this source's rows by group.+# The outer LIMIT lets the optimizer insert a local TopK before redistribution,+# exercising the standalone sort-pushdown path. Without the window, this query+# uses the already-correct merge/sort path.+# Removing the source-side sort must retain its per-partition limit.+query III+SELECT group_id, value,+ SUM(value) OVER (+ PARTITION BY group_id ORDER BY value ROWS UNBOUNDED PRECEDING+ ) AS running_sum+FROM exact_sort_source+ORDER BY group_id, value+LIMIT 3;+----+0 0 0+0 1 1+0 2 3++# The pushed-down sort must leave a LocalLimitExec below the repartition.+statement ok+SET datafusion.explain.physical_plan_only = true;++query TT+EXPLAIN+SELECT group_id, value,+ SUM(value) OVER (+ PARTITION BY group_id ORDER BY value ROWS UNBOUNDED PRECEDING+ ) AS running_sum+FROM exact_sort_source+ORDER BY group_id, value+LIMIT 3;+----+physical_plan+01)SortPreservingMergeExec: [group_id@0 ASC NULLS LAST, value@1 ASC NULLS LAST], fetch=3+02)--ProjectionExec: expr=[group_id@0 as group_id, value@1 as value, sum(exact_sort_source.value) PARTITION BY [exact_sort_source.group_id] ORDER BY [exact_sort_source.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as running_sum]+03)----BoundedWindowAggExec: wdw=[sum(exact_sort_source.value) PARTITION BY [exact_sort_source.group_id] ORDER BY [exact_sort_source.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(exact_sort_source.value) PARTITION BY [exact_sort_source.group_id] ORDER BY [exact_sort_source.value ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Int64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted]+04)------SortExec: TopK(fetch=3), expr=[group_id@0 ASC NULLS LAST, value@1 ASC NULLS LAST], preserve_partitioning=[true]+05)--------RepartitionExec: partitioning=Hash([group_id@0], 4), input_partitions=4+06)----------LocalLimitExec: fetch=3+07)------------ExactSortSource++statement ok+RESET datafusion.explain.physical_plan_only;+# Test 1: Sort Pushdown for ordered Parquet files# Create a sorted datasetstatement ok