Apache DataFusion SQL Query Engine
Establish the high level API for sort pushdown and the optimizer rule and support reverse files and row groups (#19064)
Qi Zhu3a41cc6078ba
diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rsindex 6539911cb9..c8ed491ef4 100644--- a/datafusion/common/src/config.rs+++ b/datafusion/common/src/config.rs@@ -1079,6 +1079,21 @@ config_namespace! {/// then the output will be coerced to a non-view./// Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`.pub expand_views_at_output: bool, default = false++ /// Enable sort pushdown optimization.+ /// When enabled, attempts to push sort requirements down to data sources+ /// that can natively handle them (e.g., by reversing file/row group read order).+ ///+ /// Returns **inexact ordering**: Sort operator is kept for correctness,+ /// but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N),+ /// providing significant speedup.+ ///+ /// Memory: No additional overhead (only changes read order).+ ///+ /// Future: Will add option to detect perfectly sorted data and eliminate Sort completely.+ ///+ /// Default: true+ pub enable_sort_pushdown: bool, default = true}}diff --git a/datafusion/core/tests/physical_optimizer/mod.rs b/datafusion/core/tests/physical_optimizer/mod.rsindex fe9db1975d..d11322cd26 100644--- a/datafusion/core/tests/physical_optimizer/mod.rs+++ b/datafusion/core/tests/physical_optimizer/mod.rs@@ -32,6 +32,7 @@ mod limit_pushdown;mod limited_distinct_aggregation;mod partition_statistics;mod projection_pushdown;+mod pushdown_sort;mod replace_with_order_preserving_variants;mod sanity_checker;#[expect(clippy::needless_pass_by_value)]diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rsnew file mode 100644index 0000000000..f26ed2905b--- /dev/null+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs@@ -0,0 +1,672 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Tests for sort pushdown optimizer rule (Phase 1)+//!+//! Phase 1 tests verify that:+//! 1. Reverse scan is enabled (reverse_row_groups=true)+//! 2. SortExec is kept (because ordering is inexact)+//! 3. output_ordering remains unchanged+//! 4. Early termination is enabled for TopK queries+//! 5. Prefix matching works correctly++use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_optimizer::pushdown_sort::PushdownSort;+use datafusion_physical_optimizer::PhysicalOptimizerRule;++use crate::physical_optimizer::test_utils::{+ coalesce_batches_exec, coalesce_partitions_exec, parquet_exec,+ parquet_exec_with_sort, repartition_exec, schema, sort_exec, sort_exec_with_fetch,+ sort_expr, OptimizationTest,+};++#[test]+fn test_sort_pushdown_disabled() {+ // When pushdown is disabled, plan should remain unchanged+ let schema = schema();+ let source = parquet_exec(schema.clone());+ let sort_exprs = LexOrdering::new(vec![sort_expr("a", &schema)]).unwrap();+ let plan = sort_exec(sort_exprs, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), false),+ @r###"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "###+ );+}++#[test]+fn test_sort_pushdown_basic_phase1() {+ // Phase 1: Reverse scan enabled, Sort kept, output_ordering unchanged+ let schema = schema();++ // Source has ASC NULLS LAST ordering (default)+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request DESC NULLS LAST ordering (exact reverse)+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_with_limit_phase1() {+ // Phase 1: Sort with fetch enables early termination but keeps Sort+ let schema = schema();++ // Source has ASC ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request DESC ordering with limit+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec_with_fetch(desc_ordering, Some(10), source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_multiple_columns_phase1() {+ // Phase 1: Sort on multiple columns - reverse multi-column ordering+ let schema = schema();++ // Source has [a DESC NULLS LAST, b ASC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone().reverse(), b.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request [a ASC NULLS FIRST, b DESC] ordering (exact reverse)+ let reverse_ordering =+ LexOrdering::new(vec![a.clone().asc().nulls_first(), b.reverse()]).unwrap();+ let plan = sort_exec(reverse_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST, b@1 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++// ============================================================================+// PREFIX MATCHING TESTS+// ============================================================================++#[test]+fn test_prefix_match_single_column() {+ // Test prefix matching: source has [a DESC, b ASC], query needs [a ASC]+ // After reverse: [a ASC, b DESC] which satisfies [a ASC] prefix+ let schema = schema();++ // Source has [a DESC NULLS LAST, b ASC NULLS LAST] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone().reverse(), b]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request only [a ASC NULLS FIRST] - a prefix of the reversed ordering+ let prefix_ordering = LexOrdering::new(vec![a.clone().asc().nulls_first()]).unwrap();+ let plan = sort_exec(prefix_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST, b@1 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_prefix_match_with_limit() {+ // Test prefix matching with LIMIT - important for TopK optimization+ let schema = schema();++ // Source has [a ASC, b DESC, c ASC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let c = sort_expr("c", &schema);+ let source_ordering =+ LexOrdering::new(vec![a.clone(), b.clone().reverse(), c]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request [a DESC NULLS LAST, b ASC NULLS FIRST] with LIMIT 100+ // This is a prefix (2 columns) of the reversed 3-column ordering+ let prefix_ordering =+ LexOrdering::new(vec![a.reverse(), b.clone().asc().nulls_first()]).unwrap();+ let plan = sort_exec_with_fetch(prefix_ordering, Some(100), source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: TopK(fetch=100), expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC, b@1 DESC NULLS LAST, c@2 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: TopK(fetch=100), expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_prefix_match_through_transparent_nodes() {+ // Test prefix matching works through transparent nodes+ let schema = schema();++ // Source has [a DESC NULLS LAST, b ASC, c DESC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let c = sort_expr("c", &schema);+ let source_ordering =+ LexOrdering::new(vec![a.clone().reverse(), b, c.reverse()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);+ let coalesce = coalesce_batches_exec(source, 1024);+ let repartition = repartition_exec(coalesce);++ // Request only [a ASC NULLS FIRST] - prefix of reversed ordering+ let prefix_ordering = LexOrdering::new(vec![a.clone().asc().nulls_first()]).unwrap();+ let plan = sort_exec(prefix_ordering, repartition);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1, maintains_sort_order=true+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST, b@1 ASC, c@2 DESC NULLS LAST], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_no_prefix_match_wrong_direction() {+ // Test that prefix matching does NOT work if the direction is wrong+ let schema = schema();++ // Source has [a DESC, b ASC] ordering+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone().reverse(), b]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request [a DESC] - same direction as source, NOT a reverse prefix+ let same_direction = LexOrdering::new(vec![a.clone().reverse()]).unwrap();+ let plan = sort_exec(same_direction, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r###"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST, b@1 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST, b@1 ASC], file_type=parquet+ "###+ );+}++#[test]+fn test_no_prefix_match_longer_than_source() {+ // Test that prefix matching does NOT work if requested is longer than source+ let schema = schema();++ // Source has [a DESC] ordering (single column)+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone().reverse()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request [a ASC, b DESC] - longer than source, can't be a prefix+ let longer_ordering =+ LexOrdering::new(vec![a.clone().asc().nulls_first(), b.reverse()]).unwrap();+ let plan = sort_exec(longer_ordering, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r###"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST], file_type=parquet+ "###+ );+}++// ============================================================================+// ORIGINAL TESTS+// ============================================================================++#[test]+fn test_sort_through_coalesce_batches() {+ // Sort pushes through CoalesceBatchesExec+ let schema = schema();+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);+ let coalesce = coalesce_batches_exec(source, 1024);++ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, coalesce);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_sort_through_repartition() {+ // Sort should push through RepartitionExec+ let schema = schema();+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);+ let repartition = repartition_exec(source);++ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, repartition);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1, maintains_sort_order=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_nested_sorts() {+ // Nested sort operations - only innermost can be optimized+ let schema = schema();+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let inner_sort = sort_exec(desc_ordering, source);++ let sort_exprs2 = LexOrdering::new(vec![b]).unwrap();+ let plan = sort_exec(sort_exprs2, inner_sort);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[b@1 ASC], preserve_partitioning=[false]+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[b@1 ASC], preserve_partitioning=[false]+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_non_sort_plans_unchanged() {+ // Plans without SortExec should pass through unchanged+ let schema = schema();+ let source = parquet_exec(schema.clone());+ let plan = coalesce_batches_exec(source, 1024);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r###"+ OptimizationTest:+ input:+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ output:+ Ok:+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "###+ );+}++#[test]+fn test_optimizer_properties() {+ // Test optimizer metadata+ let optimizer = PushdownSort::new();++ assert_eq!(optimizer.name(), "PushdownSort");+ assert!(optimizer.schema_check());+}++#[test]+fn test_sort_through_coalesce_partitions() {+ // Sort should push through CoalescePartitionsExec+ let schema = schema();+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);+ let repartition = repartition_exec(source);+ let coalesce_parts = coalesce_partitions_exec(repartition);++ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, coalesce_parts);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1, maintains_sort_order=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_complex_plan_with_multiple_operators() {+ // Test a complex plan with multiple operators between sort and source+ let schema = schema();+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);+ let coalesce_batches = coalesce_batches_exec(source, 1024);+ let repartition = repartition_exec(coalesce_batches);+ let coalesce_parts = coalesce_partitions_exec(repartition);++ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let plan = sort_exec(desc_ordering, coalesce_parts);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1, maintains_sort_order=true+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - CoalescePartitionsExec+ - RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ - CoalesceBatchesExec: target_batch_size=1024+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_multiple_sorts_different_columns() {+ // Test nested sorts on different columns - only innermost can optimize+ let schema = schema();+ let a = sort_expr("a", &schema);+ let c = sort_expr("c", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // First sort by column 'a' DESC (reverse of source)+ let desc_ordering = LexOrdering::new(vec![a.reverse()]).unwrap();+ let sort1 = sort_exec(desc_ordering, source);++ // Then sort by column 'c' (different column, can't optimize)+ let sort_exprs2 = LexOrdering::new(vec![c]).unwrap();+ let plan = sort_exec(sort_exprs2, sort1);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}++#[test]+fn test_no_pushdown_for_unordered_source() {+ // Verify pushdown does NOT happen for sources without ordering+ let schema = schema();+ let source = parquet_exec(schema.clone()); // No output_ordering+ let sort_exprs = LexOrdering::new(vec![sort_expr("a", &schema)]).unwrap();+ let plan = sort_exec(sort_exprs, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r###"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "###+ );+}++#[test]+fn test_no_pushdown_for_non_reverse_sort() {+ // Verify pushdown does NOT happen when sort doesn't reverse source ordering+ let schema = schema();++ // Source sorted by 'a' ASC+ let a = sort_expr("a", &schema);+ let b = sort_expr("b", &schema);+ let source_ordering = LexOrdering::new(vec![a]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Request sort by 'b' (different column)+ let sort_exprs = LexOrdering::new(vec![b]).unwrap();+ let plan = sort_exec(sort_exprs, source);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r###"+ OptimizationTest:+ input:+ - SortExec: expr=[b@1 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[b@1 ASC], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ "###+ );+}++#[test]+fn test_pushdown_through_blocking_node() {+ // Test that pushdown works for inner sort even when outer sort is blocked+ // Structure: Sort -> Aggregate (blocks pushdown) -> Sort -> Scan+ // The outer sort can't push through aggregate, but the inner sort should still optimize+ use datafusion_functions_aggregate::count::count_udaf;+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;+ use datafusion_physical_plan::aggregates::{+ AggregateExec, AggregateMode, PhysicalGroupBy,+ };+ use std::sync::Arc;++ let schema = schema();++ // Bottom: DataSource with [a ASC NULLS LAST] ordering+ let a = sort_expr("a", &schema);+ let source_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let source = parquet_exec_with_sort(schema.clone(), vec![source_ordering]);++ // Inner Sort: [a DESC NULLS FIRST] - exact reverse, CAN push down to source+ let inner_sort_ordering = LexOrdering::new(vec![a.clone().reverse()]).unwrap();+ let inner_sort = sort_exec(inner_sort_ordering, source);++ // Middle: Aggregate (blocks pushdown from outer sort)+ // GROUP BY a, COUNT(b)+ let group_by = PhysicalGroupBy::new_single(vec![(+ Arc::new(datafusion_physical_expr::expressions::Column::new("a", 0)) as _,+ "a".to_string(),+ )]);++ let count_expr = Arc::new(+ AggregateExprBuilder::new(+ count_udaf(),+ vec![+ Arc::new(datafusion_physical_expr::expressions::Column::new("b", 1)) as _,+ ],+ )+ .schema(Arc::clone(&schema))+ .alias("COUNT(b)")+ .build()+ .unwrap(),+ );++ let aggregate = Arc::new(+ AggregateExec::try_new(+ AggregateMode::Final,+ group_by,+ vec![count_expr],+ vec![None],+ inner_sort,+ Arc::clone(&schema),+ )+ .unwrap(),+ );++ // Outer Sort: [a ASC] - this CANNOT push down through aggregate+ let outer_sort_ordering = LexOrdering::new(vec![a.clone()]).unwrap();+ let plan = sort_exec(outer_sort_ordering, aggregate);++ insta::assert_snapshot!(+ OptimizationTest::new(plan, PushdownSort::new(), true),+ @r"+ OptimizationTest:+ input:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - AggregateExec: mode=Final, gby=[a@0 as a], aggr=[COUNT(b)], ordering_mode=Sorted+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ output:+ Ok:+ - SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ - AggregateExec: mode=Final, gby=[a@0 as a], aggr=[COUNT(b)], ordering_mode=Sorted+ - SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ "+ );+}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex 1561ddf440..fa3e860ad3 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -18,7 +18,7 @@//! Test utilities for physical optimizer testsuse std::any::Any;-use std::fmt::Formatter;+use std::fmt::{Display, Formatter};use std::sync::{Arc, LazyLock};use arrow::array::Int32Array;@@ -33,7 +33,9 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;-use datafusion_common::{ColumnStatistics, JoinType, NullEquality, Result, Statistics};+use datafusion_common::{+ internal_err, ColumnStatistics, JoinType, NullEquality, Result, Statistics,+};use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_execution::object_store::ObjectStoreUrl;use datafusion_execution::{SendableRecordBatchStream, TaskContext};@@ -700,3 +702,75 @@ impl TestAggregate {}}}++/// A harness for testing physical optimizers.+#[derive(Debug)]+pub struct OptimizationTest {+ input: Vec<String>,+ output: Result<Vec<String>, String>,+}++impl OptimizationTest {+ pub fn new<O>(+ input_plan: Arc<dyn ExecutionPlan>,+ opt: O,+ enable_sort_pushdown: bool,+ ) -> Self+ where+ O: PhysicalOptimizerRule,+ {+ let input = format_execution_plan(&input_plan);+ let input_schema = input_plan.schema();++ let mut config = ConfigOptions::new();+ config.optimizer.enable_sort_pushdown = enable_sort_pushdown;+ let output_result = opt.optimize(input_plan, &config);+ let output = output_result+ .and_then(|plan| {+ if opt.schema_check() && (plan.schema() != input_schema) {+ internal_err!(+ "Schema mismatch:\n\nBefore:\n{:?}\n\nAfter:\n{:?}",+ input_schema,+ plan.schema()+ )+ } else {+ Ok(plan)+ }+ })+ .map(|plan| format_execution_plan(&plan))+ .map_err(|e| e.to_string());++ Self { input, output }+ }+}++impl Display for OptimizationTest {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ writeln!(f, "OptimizationTest:")?;+ writeln!(f, " input:")?;+ for line in &self.input {+ writeln!(f, " - {line}")?;+ }+ writeln!(f, " output:")?;+ match &self.output {+ Ok(output) => {+ writeln!(f, " Ok:")?;+ for line in output {+ writeln!(f, " - {line}")?;+ }+ }+ Err(err) => {+ writeln!(f, " Err: {err}")?;+ }+ }+ Ok(())+ }+}++pub fn format_execution_plan(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {+ format_lines(&displayable(plan.as_ref()).indent(false).to_string())+}++fn format_lines(s: &str) -> Vec<String> {+ s.trim().split('\n').map(|s| s.to_string()).collect()+}diff --git a/datafusion/datasource-parquet/src/mod.rs b/datafusion/datasource-parquet/src/mod.rsindex 53ee597bd1..eb4cc9e9ad 100644--- a/datafusion/datasource-parquet/src/mod.rs+++ b/datafusion/datasource-parquet/src/mod.rs@@ -30,6 +30,7 @@ mod page_filter;mod reader;mod row_filter;mod row_group_filter;+mod sort;pub mod source;mod writer;diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 7b4db9e772..f1ecc86ce8 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -51,6 +51,7 @@ use datafusion_physical_plan::metrics::{};use datafusion_pruning::{FilePruner, PruningPredicate, build_pruning_predicate};+use crate::sort::reverse_row_selection;#[cfg(feature = "parquet_encryption")]use datafusion_common::config::EncryptionFactoryOptions;#[cfg(feature = "parquet_encryption")]@@ -63,7 +64,7 @@ use parquet::arrow::arrow_reader::{};use parquet::arrow::async_reader::AsyncFileReader;use parquet::arrow::{ParquetRecordBatchStreamBuilder, ProjectionMask};-use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader};+use parquet::file::metadata::{PageIndexPolicy, ParquetMetaDataReader, RowGroupMetaData};/// Implements [`FileOpener`] for a parquet filepub(super) struct ParquetOpener {@@ -115,6 +116,60 @@ pub(super) struct ParquetOpener {/// Maximum size of the predicate cache, in bytes. If none, uses/// the arrow-rs default.pub max_predicate_cache_size: Option<usize>,+ /// Whether to read row groups in reverse order+ pub reverse_row_groups: bool,+}++/// Represents a prepared access plan with optional row selection+struct PreparedAccessPlan {+ /// Row group indexes to read+ row_group_indexes: Vec<usize>,+ /// Optional row selection for filtering within row groups+ row_selection: Option<parquet::arrow::arrow_reader::RowSelection>,+}++impl PreparedAccessPlan {+ /// Create a new prepared access plan from a ParquetAccessPlan+ fn from_access_plan(+ access_plan: ParquetAccessPlan,+ rg_metadata: &[RowGroupMetaData],+ ) -> Result<Self> {+ let row_group_indexes = access_plan.row_group_indexes();+ let row_selection = access_plan.into_overall_row_selection(rg_metadata)?;++ Ok(Self {+ row_group_indexes,+ row_selection,+ })+ }++ /// Reverse the access plan for reverse scanning+ fn reverse(+ mut self,+ file_metadata: &parquet::file::metadata::ParquetMetaData,+ ) -> Result<Self> {+ // Reverse the row group indexes+ self.row_group_indexes = self.row_group_indexes.into_iter().rev().collect();++ // If we have a row selection, reverse it to match the new row group order+ if let Some(row_selection) = self.row_selection {+ self.row_selection =+ Some(reverse_row_selection(&row_selection, file_metadata)?);+ }++ Ok(self)+ }++ /// Apply this access plan to a ParquetRecordBatchStreamBuilder+ fn apply_to_builder(+ self,+ mut builder: ParquetRecordBatchStreamBuilder<Box<dyn AsyncFileReader>>,+ ) -> ParquetRecordBatchStreamBuilder<Box<dyn AsyncFileReader>> {+ if let Some(row_selection) = self.row_selection {+ builder = builder.with_row_selection(row_selection);+ }+ builder.with_row_groups(self.row_group_indexes)+ }}impl FileOpener for ParquetOpener {@@ -212,6 +267,7 @@ impl FileOpener for ParquetOpener {let encryption_context = self.get_encryption_context();let max_predicate_cache_size = self.max_predicate_cache_size;+ let reverse_row_groups = self.reverse_row_groups;Ok(Box::pin(async move {#[cfg(feature = "parquet_encryption")]let file_decryption_properties = encryption_context@@ -479,13 +535,18 @@ impl FileOpener for ParquetOpener {);}- let row_group_indexes = access_plan.row_group_indexes();- if let Some(row_selection) =- access_plan.into_overall_row_selection(rg_metadata)?- {- builder = builder.with_row_selection(row_selection);+ // Prepare the access plan (extract row groups and row selection)+ let mut prepared_plan =+ PreparedAccessPlan::from_access_plan(access_plan, rg_metadata)?;++ // If reverse scanning is enabled, reverse the prepared plan+ if reverse_row_groups {+ prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;}+ // Apply the prepared plan to the builder+ builder = prepared_plan.apply_to_builder(builder);+if let Some(limit) = limit {builder = builder.with_limit(limit)}@@ -500,7 +561,6 @@ impl FileOpener for ParquetOpener {let stream = builder.with_projection(mask).with_batch_size(batch_size)- .with_row_groups(row_group_indexes).with_metrics(arrow_reader_metrics.clone()).build()?;@@ -904,6 +964,7 @@ mod test {use std::sync::Arc;use super::{ConstantColumns, constant_columns_from_stats};+ use crate::{DefaultParquetFileReaderFactory, opener::ParquetOpener};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use bytes::{BufMut, BytesMut};use datafusion_common::{@@ -925,8 +986,7 @@ mod test {use futures::{Stream, StreamExt};use object_store::{ObjectStore, memory::InMemory, path::Path};use parquet::arrow::ArrowWriter;-- use crate::{DefaultParquetFileReaderFactory, opener::ParquetOpener};+ use parquet::file::properties::WriterProperties;fn constant_int_stats() -> (Statistics, SchemaRef) {let schema = Arc::new(Schema::new(vec![@@ -1028,16 +1088,54 @@ mod test {(num_batches, num_rows)}+ /// Helper to collect all int32 values from the first column of batches+ async fn collect_int32_values(+ mut stream: std::pin::Pin<+ Box<+ dyn Stream<Item = Result<arrow::array::RecordBatch, DataFusionError>>+ + Send,+ >,+ >,+ ) -> Vec<i32> {+ use arrow::array::Array;+ let mut values = vec![];+ while let Some(Ok(batch)) = stream.next().await {+ let array = batch+ .column(0)+ .as_any()+ .downcast_ref::<arrow::array::Int32Array>()+ .unwrap();+ for i in 0..array.len() {+ if !array.is_null(i) {+ values.push(array.value(i));+ }+ }+ }+ values+ }+async fn write_parquet(store: Arc<dyn ObjectStore>,filename: &str,batch: arrow::record_batch::RecordBatch,+ ) -> usize {+ write_parquet_batches(store, filename, vec![batch], None).await+ }++ /// Write multiple batches to a parquet file with optional writer properties+ async fn write_parquet_batches(+ store: Arc<dyn ObjectStore>,+ filename: &str,+ batches: Vec<arrow::record_batch::RecordBatch>,+ props: Option<WriterProperties>,) -> usize {let mut out = BytesMut::new().writer();{- let mut writer =- ArrowWriter::try_new(&mut out, batch.schema(), None).unwrap();- writer.write(&batch).unwrap();+ let schema = batches[0].schema();+ let mut writer = ArrowWriter::try_new(&mut out, schema, props).unwrap();+ for batch in batches {+ writer.write(&batch).unwrap();+ }writer.finish().unwrap();}let data = out.into_inner().freeze();@@ -1108,6 +1206,7 @@ mod test {#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,+ reverse_row_groups: false,}};@@ -1179,6 +1278,7 @@ mod test {#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,+ reverse_row_groups: false,}};@@ -1266,6 +1366,7 @@ mod test {#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,+ reverse_row_groups: false,}};@@ -1356,6 +1457,7 @@ mod test {#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,+ reverse_row_groups: false,}};@@ -1454,6 +1556,7 @@ mod test {#[cfg(feature = "parquet_encryption")]encryption_factory: None,max_predicate_cache_size: None,+ reverse_row_groups: false,}};@@ -1495,4 +1598,252 @@ mod test {assert_eq!(num_batches, 0);assert_eq!(num_rows, 0);}++ #[tokio::test]+ async fn test_reverse_scan_row_groups() {+ use parquet::file::properties::WriterProperties;++ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ // Create multiple batches to ensure multiple row groups+ let batch1 =+ record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();+ let batch2 =+ record_batch!(("a", Int32, vec![Some(4), Some(5), Some(6)])).unwrap();+ let batch3 =+ record_batch!(("a", Int32, vec![Some(7), Some(8), Some(9)])).unwrap();++ // Write parquet file with multiple row groups+ // Force small row groups by setting max_row_group_size+ let props = WriterProperties::builder()+ .set_max_row_group_size(3) // Force each batch into its own row group+ .build();++ let data_len = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch1.clone(), batch2, batch3],+ Some(props),+ )+ .await;++ let schema = batch1.schema();+ let file = PartitionedFile::new(+ "test.parquet".to_string(),+ u64::try_from(data_len).unwrap(),+ );++ let make_opener = |reverse_scan: bool| ParquetOpener {+ partition_index: 0,+ projection: ProjectionExprs::from_indices(&[0], &schema),+ batch_size: 1024,+ limit: None,+ predicate: None,+ table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),+ metadata_size_hint: None,+ metrics: ExecutionPlanMetricsSet::new(),+ parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(+ Arc::clone(&store),+ )),+ pushdown_filters: false,+ reorder_filters: false,+ force_filter_selections: false,+ enable_page_index: false,+ enable_bloom_filter: false,+ enable_row_group_stats_pruning: false,+ coerce_int96: None,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties: None,+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: None,+ max_predicate_cache_size: None,+ reverse_row_groups: reverse_scan,+ };++ // Test normal scan (forward)+ let opener = make_opener(false);+ let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let forward_values = collect_int32_values(stream).await;++ // Test reverse scan+ let opener = make_opener(true);+ let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let reverse_values = collect_int32_values(stream).await;++ // The forward scan should return data in the order written+ assert_eq!(forward_values, vec![1, 2, 3, 4, 5, 6, 7, 8, 9]);++ // With reverse scan, row groups are reversed, so we expect:+ // Row group 3 (7,8,9), then row group 2 (4,5,6), then row group 1 (1,2,3)+ assert_eq!(reverse_values, vec![7, 8, 9, 4, 5, 6, 1, 2, 3]);+ }++ #[tokio::test]+ async fn test_reverse_scan_single_row_group() {+ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ // Create a single batch (single row group)+ let batch = record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3)])).unwrap();+ let data_size =+ write_parquet(Arc::clone(&store), "test.parquet", batch.clone()).await;++ let schema = batch.schema();+ let file = PartitionedFile::new(+ "test.parquet".to_string(),+ u64::try_from(data_size).unwrap(),+ );++ let make_opener = |reverse_scan: bool| ParquetOpener {+ partition_index: 0,+ projection: ProjectionExprs::from_indices(&[0], &schema),+ batch_size: 1024,+ limit: None,+ predicate: None,+ table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),+ metadata_size_hint: None,+ metrics: ExecutionPlanMetricsSet::new(),+ parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(+ Arc::clone(&store),+ )),+ pushdown_filters: false,+ reorder_filters: false,+ force_filter_selections: false,+ enable_page_index: false,+ enable_bloom_filter: false,+ enable_row_group_stats_pruning: false,+ coerce_int96: None,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties: None,+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: None,+ max_predicate_cache_size: None,+ reverse_row_groups: reverse_scan,+ };++ // With a single row group, forward and reverse should be the same+ // (only the row group order is reversed, not the rows within)+ let opener_forward = make_opener(false);+ let stream_forward = opener_forward.open(file.clone()).unwrap().await.unwrap();+ let (batches_forward, _) = count_batches_and_rows(stream_forward).await;++ let opener_reverse = make_opener(true);+ let stream_reverse = opener_reverse.open(file).unwrap().await.unwrap();+ let (batches_reverse, _) = count_batches_and_rows(stream_reverse).await;++ // Both should have the same number of batches since there's only one row group+ assert_eq!(batches_forward, batches_reverse);+ }++ #[tokio::test]+ async fn test_reverse_scan_with_row_selection() {+ use parquet::file::properties::WriterProperties;++ let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;++ // Create 3 batches with DIFFERENT selection patterns+ let batch1 =+ record_batch!(("a", Int32, vec![Some(1), Some(2), Some(3), Some(4)]))+ .unwrap(); // 4 rows+ let batch2 =+ record_batch!(("a", Int32, vec![Some(5), Some(6), Some(7), Some(8)]))+ .unwrap(); // 4 rows+ let batch3 =+ record_batch!(("a", Int32, vec![Some(9), Some(10), Some(11), Some(12)]))+ .unwrap(); // 4 rows++ let props = WriterProperties::builder()+ .set_max_row_group_size(4)+ .build();++ let data_len = write_parquet_batches(+ Arc::clone(&store),+ "test.parquet",+ vec![batch1.clone(), batch2, batch3],+ Some(props),+ )+ .await;++ let schema = batch1.schema();++ use crate::ParquetAccessPlan;+ use parquet::arrow::arrow_reader::{RowSelection, RowSelector};++ let mut access_plan = ParquetAccessPlan::new_all(3);+ // Row group 0: skip first 2, select last 2 (should get: 3, 4)+ access_plan.scan_selection(+ 0,+ RowSelection::from(vec![RowSelector::skip(2), RowSelector::select(2)]),+ );+ // Row group 1: select all (should get: 5, 6, 7, 8)+ // Row group 2: select first 2, skip last 2 (should get: 9, 10)+ access_plan.scan_selection(+ 2,+ RowSelection::from(vec![RowSelector::select(2), RowSelector::skip(2)]),+ );++ let file = PartitionedFile::new(+ "test.parquet".to_string(),+ u64::try_from(data_len).unwrap(),+ )+ .with_extensions(Arc::new(access_plan));++ let make_opener = |reverse_scan: bool| ParquetOpener {+ partition_index: 0,+ projection: ProjectionExprs::from_indices(&[0], &schema),+ batch_size: 1024,+ limit: None,+ predicate: None,+ table_schema: TableSchema::from_file_schema(Arc::clone(&schema)),+ metadata_size_hint: None,+ metrics: ExecutionPlanMetricsSet::new(),+ parquet_file_reader_factory: Arc::new(DefaultParquetFileReaderFactory::new(+ Arc::clone(&store),+ )),+ pushdown_filters: false,+ reorder_filters: false,+ force_filter_selections: false,+ enable_page_index: false,+ enable_bloom_filter: false,+ enable_row_group_stats_pruning: false,+ coerce_int96: None,+ #[cfg(feature = "parquet_encryption")]+ file_decryption_properties: None,+ expr_adapter_factory: Arc::new(DefaultPhysicalExprAdapterFactory),+ #[cfg(feature = "parquet_encryption")]+ encryption_factory: None,+ max_predicate_cache_size: None,+ reverse_row_groups: reverse_scan,+ };++ // Forward scan: RG0(3,4), RG1(5,6,7,8), RG2(9,10)+ let opener = make_opener(false);+ let stream = opener.open(file.clone()).unwrap().await.unwrap();+ let forward_values = collect_int32_values(stream).await;++ // Forward scan should produce: RG0(3,4), RG1(5,6,7,8), RG2(9,10)+ assert_eq!(+ forward_values,+ vec![3, 4, 5, 6, 7, 8, 9, 10],+ "Forward scan should select correct rows based on RowSelection"+ );++ // Reverse scan+ // CORRECT behavior: reverse row groups AND their corresponding selections+ // - RG2 is read first, WITH RG2's selection (select 2, skip 2) -> 9, 10+ // - RG1 is read second, WITH RG1's selection (select all) -> 5, 6, 7, 8+ // - RG0 is read third, WITH RG0's selection (skip 2, select 2) -> 3, 4+ let opener = make_opener(true);+ let stream = opener.open(file).unwrap().await.unwrap();+ let reverse_values = collect_int32_values(stream).await;++ // Correct expected result: row groups reversed but each keeps its own selection+ // RG2 with its selection (9,10), RG1 with its selection (5,6,7,8), RG0 with its selection (3,4)+ assert_eq!(+ reverse_values,+ vec![9, 10, 5, 6, 7, 8, 3, 4],+ "Reverse scan should reverse row group order while maintaining correct RowSelection for each group"+ );+ }}diff --git a/datafusion/datasource-parquet/src/sort.rs b/datafusion/datasource-parquet/src/sort.rsnew file mode 100644index 0000000000..4255d4d696--- /dev/null+++ b/datafusion/datasource-parquet/src/sort.rs@@ -0,0 +1,407 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Sort-related utilities for Parquet scanning++use datafusion_common::Result;+use parquet::arrow::arrow_reader::{RowSelection, RowSelector};+use parquet::file::metadata::ParquetMetaData;+use std::collections::HashMap;++/// Reverse a row selection to match reversed row group order.+///+/// When scanning row groups in reverse order, we need to adjust the row selection+/// to account for the new ordering. This function:+/// 1. Maps each selection to its corresponding row group+/// 2. Reverses the order of row groups+/// 3. Reconstructs the row selection for the new order+///+/// # Arguments+/// * `row_selection` - Original row selection+/// * `parquet_metadata` - Metadata containing row group information+///+/// # Returns+/// A new `RowSelection` adjusted for reversed row group order+pub fn reverse_row_selection(+ row_selection: &RowSelection,+ parquet_metadata: &ParquetMetaData,+) -> Result<RowSelection> {+ let rg_metadata = parquet_metadata.row_groups();++ // Build a mapping of row group index to its row range in the file+ let mut rg_row_ranges: Vec<(usize, usize, usize)> =+ Vec::with_capacity(rg_metadata.len());+ let mut current_row = 0;+ for (rg_idx, rg) in rg_metadata.iter().enumerate() {+ let num_rows = rg.num_rows() as usize;+ rg_row_ranges.push((rg_idx, current_row, current_row + num_rows));+ current_row += num_rows;+ }++ // Map selections to row groups+ let mut rg_selections: HashMap<usize, Vec<RowSelector>> = HashMap::new();++ let mut current_file_row = 0;+ for selector in row_selection.iter() {+ let selector_end = current_file_row + selector.row_count;++ // Find which row groups this selector spans+ for (rg_idx, rg_start, rg_end) in rg_row_ranges.iter() {+ if current_file_row < *rg_end && selector_end > *rg_start {+ // This selector overlaps with this row group+ let overlap_start = current_file_row.max(*rg_start);+ let overlap_end = selector_end.min(*rg_end);+ let overlap_count = overlap_end - overlap_start;++ if overlap_count > 0 {+ let entry = rg_selections.entry(*rg_idx).or_default();+ if selector.skip {+ entry.push(RowSelector::skip(overlap_count));+ } else {+ entry.push(RowSelector::select(overlap_count));+ }+ }+ }+ }++ current_file_row = selector_end;+ }++ // Build new selection for reversed row group order+ let mut reversed_selectors = Vec::new();+ for rg_idx in (0..rg_metadata.len()).rev() {+ if let Some(selectors) = rg_selections.get(&rg_idx) {+ reversed_selectors.extend(selectors.iter().cloned());+ } else {+ // No specific selection for this row group means select all+ if let Some((_, start, end)) =+ rg_row_ranges.iter().find(|(idx, _, _)| *idx == rg_idx)+ {+ reversed_selectors.push(RowSelector::select(end - start));+ }+ }+ }++ Ok(RowSelection::from(reversed_selectors))+}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::datatypes::{DataType, Field, Schema};+ use bytes::Bytes;+ use parquet::arrow::ArrowWriter;+ use parquet::file::reader::FileReader;+ use parquet::file::serialized_reader::SerializedFileReader;+ use std::sync::Arc;++ /// Helper function to create a ParquetMetaData with specified row group sizes+ /// by actually writing a parquet file in memory+ fn create_test_metadata(row_group_sizes: Vec<i64>) -> ParquetMetaData {+ // Create a simple schema+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));++ // Create in-memory parquet file with the specified row groups+ let mut buffer = Vec::new();+ {+ let props = parquet::file::properties::WriterProperties::builder()+ .set_max_row_group_size(row_group_sizes[0] as usize)+ .build();++ let mut writer =+ ArrowWriter::try_new(&mut buffer, schema.clone(), Some(props)).unwrap();++ for &size in &row_group_sizes {+ // Create a batch with the specified number of rows+ let array = arrow::array::Int32Array::from(vec![1; size as usize]);+ let batch = arrow::record_batch::RecordBatch::try_new(+ schema.clone(),+ vec![Arc::new(array)],+ )+ .unwrap();+ writer.write(&batch).unwrap();+ }+ writer.close().unwrap();+ }++ // Read back the metadata+ let bytes = Bytes::from(buffer);+ let reader = SerializedFileReader::new(bytes).unwrap();+ reader.metadata().clone()+ }++ #[test]+ fn test_reverse_simple_selection() {+ // 3 row groups with 100 rows each+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Select first 50 rows from first row group+ let selection =+ RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(250)]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // Verify total selected rows remain the same+ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ }++ #[test]+ fn test_reverse_multi_row_group_selection() {+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Select rows spanning multiple row groups+ let selection = RowSelection::from(vec![+ RowSelector::skip(50),+ RowSelector::select(100), // Spans RG0 and RG1+ RowSelector::skip(150),+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // Verify total selected rows remain the same+ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ }++ #[test]+ fn test_reverse_full_selection() {+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Select all rows+ let selection = RowSelection::from(vec![RowSelector::select(300)]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // Should still select all rows, just in reversed row group order+ let total_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(total_selected, 300);+ }++ #[test]+ fn test_reverse_empty_selection() {+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Skip all rows+ let selection = RowSelection::from(vec![RowSelector::skip(300)]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // Should still skip all rows+ let total_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(total_selected, 0);+ }++ #[test]+ fn test_reverse_with_different_row_group_sizes() {+ let metadata = create_test_metadata(vec![50, 150, 100]);++ let selection = RowSelection::from(vec![+ RowSelector::skip(25),+ RowSelector::select(200), // Spans all row groups+ RowSelector::skip(75),+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ }++ #[test]+ fn test_reverse_single_row_group() {+ let metadata = create_test_metadata(vec![100]);++ let selection =+ RowSelection::from(vec![RowSelector::select(50), RowSelector::skip(50)]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // With single row group, selection should remain the same+ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ }++ #[test]+ fn test_reverse_complex_pattern() {+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Complex pattern: select some, skip some, select some more+ let selection = RowSelection::from(vec![+ RowSelector::select(30),+ RowSelector::skip(40),+ RowSelector::select(80),+ RowSelector::skip(50),+ RowSelector::select(100),+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ assert_eq!(original_selected, 210); // 30 + 80 + 100+ }++ #[test]+ fn test_reverse_with_skipped_row_group() {+ // This test covers the "no specific selection" code path (lines 90-95)+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Select only from first and third row groups, skip middle one entirely+ let selection = RowSelection::from(vec![+ RowSelector::select(50), // First 50 of RG0+ RowSelector::skip(150), // Rest of RG0 + all of RG1 + half of RG2+ RowSelector::select(50), // Last 50 of RG2+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ // Verify total selected rows remain the same+ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ assert_eq!(original_selected, 100); // 50 + 50+ }++ #[test]+ fn test_reverse_middle_row_group_only() {+ // Another test to ensure skipped row groups are handled correctly+ let metadata = create_test_metadata(vec![100, 100, 100]);++ // Select only middle row group+ let selection = RowSelection::from(vec![+ RowSelector::skip(100), // Skip RG0+ RowSelector::select(100), // Select all of RG1+ RowSelector::skip(100), // Skip RG2+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ assert_eq!(original_selected, 100);+ }++ #[test]+ fn test_reverse_alternating_row_groups() {+ // Test with more complex skipping pattern+ let metadata = create_test_metadata(vec![100, 100, 100, 100]);++ // Select first and third row groups, skip second and fourth+ let selection = RowSelection::from(vec![+ RowSelector::select(100), // RG0+ RowSelector::skip(100), // RG1+ RowSelector::select(100), // RG2+ RowSelector::skip(100), // RG3+ ]);++ let reversed = reverse_row_selection(&selection, &metadata).unwrap();++ let original_selected: usize = selection+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();+ let reversed_selected: usize = reversed+ .iter()+ .filter(|s| !s.skip)+ .map(|s| s.row_count)+ .sum();++ assert_eq!(original_selected, reversed_selected);+ assert_eq!(original_selected, 200);+ }+}diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 5caaa1c474..4956f83eff 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -44,6 +44,7 @@ use datafusion_physical_expr_adapter::DefaultPhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_plan::DisplayFormatType;+use datafusion_physical_plan::SortOrderPushdownResult;use datafusion_physical_plan::filter_pushdown::PushedDown;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPropagation, PushedDownPredicate,@@ -53,6 +54,7 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;#[cfg(feature = "parquet_encryption")]use datafusion_execution::parquet_encryption::EncryptionFactory;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;use object_store::ObjectStore;#[cfg(feature = "parquet_encryption")]@@ -287,6 +289,14 @@ pub struct ParquetSource {pub(crate) projection: ProjectionExprs,#[cfg(feature = "parquet_encryption")]pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,+ /// The ordering of data within the files+ /// This is set by FileScanConfig when it knows the file ordering+ file_ordering: Option<LexOrdering>,+ /// If true, read files in reverse order and reverse row groups within files.+ /// But it's not guaranteed that rows within row groups are in reverse order,+ /// so we still need to sort them after reading, so the reverse scan is inexact.+ /// Used to optimize ORDER BY ... DESC on sorted data.+ reverse_row_groups: bool,}impl ParquetSource {@@ -311,6 +321,8 @@ impl ParquetSource {metadata_size_hint: None,#[cfg(feature = "parquet_encryption")]encryption_factory: None,+ file_ordering: None,+ reverse_row_groups: false,}}@@ -386,6 +398,12 @@ impl ParquetSource {self}+ /// If set, indicates the ordering of data within the files being read.+ pub fn with_file_ordering(mut self, ordering: Option<LexOrdering>) -> Self {+ self.file_ordering = ordering;+ self+ }+/// Return the value described in [`Self::with_pushdown_filters`]pub(crate) fn pushdown_filters(&self) -> bool {self.table_parquet_options.global.pushdown_filters@@ -465,6 +483,15 @@ impl ParquetSource {)),}}++ pub(crate) fn with_reverse_row_groups(mut self, reverse_row_groups: bool) -> Self {+ self.reverse_row_groups = reverse_row_groups;+ self+ }+ #[cfg(test)]+ pub(crate) fn reverse_row_groups(&self) -> bool {+ self.reverse_row_groups+ }}/// Parses datafusion.common.config.ParquetOptions.coerce_int96 String to a arrow_schema.datatype.TimeUnit@@ -550,6 +577,7 @@ impl FileSource for ParquetSource {#[cfg(feature = "parquet_encryption")]encryption_factory: self.get_encryption_factory_with_config(),max_predicate_cache_size: self.max_predicate_cache_size(),+ reverse_row_groups: self.reverse_row_groups,});Ok(opener)}@@ -603,6 +631,11 @@ impl FileSource for ParquetSource {write!(f, "{predicate_string}")?;+ // Add reverse_scan info if enabled+ if self.reverse_row_groups {+ write!(f, ", reverse_row_groups=true")?;+ }+// Try to build a the pruning predicates.// These are only generated here because it's useful to have *some*// idea of what pushdown is happening when viewing plans.@@ -710,6 +743,72 @@ impl FileSource for ParquetSource {).with_updated_node(source))}++ /// Try to optimize the scan to produce data in the requested sort order.+ ///+ /// This method receives:+ /// 1. The query's required ordering (`order` parameter)+ /// 2. The file's natural ordering (via `self.file_ordering`, set by FileScanConfig)+ ///+ /// With both pieces of information, ParquetSource can decide what optimizations to apply.+ ///+ /// # Phase 1 Behavior (Current)+ /// Returns `Inexact` when reversing the row group scan order would help satisfy the+ /// requested ordering. We still need a Sort operator at a higher level because:+ /// - We only reverse row group read order, not rows within row groups+ /// - This provides approximate ordering that benefits limit pushdown+ ///+ /// # Phase 2 (Future)+ /// Could return `Exact` when we can guarantee perfect ordering through techniques like:+ /// - File reordering based on statistics+ /// - Detecting already-sorted data+ /// This would allow removing the Sort operator entirely.+ ///+ /// # Returns+ /// - `Inexact`: Created an optimized source (e.g., reversed scan) that approximates the order+ /// - `Unsupported`: Cannot optimize for this ordering+ fn try_reverse_output(+ &self,+ order: &[PhysicalSortExpr],+ ) -> datafusion_common::Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {+ // Check if we have file ordering information+ let file_ordering = match &self.file_ordering {+ Some(ordering) => ordering,+ None => return Ok(SortOrderPushdownResult::Unsupported),+ };++ // Create a LexOrdering from the requested order to use the is_reverse method+ let Some(requested_ordering) = LexOrdering::new(order.to_vec()) else {+ // Empty ordering requested, cannot optimize+ return Ok(SortOrderPushdownResult::Unsupported);+ };++ // Check if reversing the file ordering would satisfy the requested ordering+ if file_ordering.is_reverse(&requested_ordering) {+ // Phase 1: Enable reverse row group scanning+ let new_source = self.clone().with_reverse_row_groups(true);++ // Return Inexact because we're only reversing row group order,+ // not guaranteeing perfect row-level ordering+ return Ok(SortOrderPushdownResult::Inexact {+ inner: Arc::new(new_source) as Arc<dyn FileSource>,+ });+ }++ // TODO Phase 2: Add support for other optimizations:+ // - File reordering based on min/max statistics+ // - Detection of exact ordering (return Exact to remove Sort operator)+ // - Partial sort pushdown for prefix matches++ Ok(SortOrderPushdownResult::Unsupported)+ }++ fn with_file_ordering_info(+ &self,+ ordering: Option<LexOrdering>,+ ) -> datafusion_common::Result<Arc<dyn FileSource>> {+ Ok(Arc::new(self.clone().with_file_ordering(ordering)))+ }}#[cfg(test)]@@ -728,4 +827,87 @@ mod tests {// same value. but filter() call Arc::clone internallyassert_eq!(parquet_source.predicate(), parquet_source.filter().as_ref());}++ #[test]+ fn test_reverse_scan_default_value() {+ use arrow::datatypes::Schema;++ let schema = Arc::new(Schema::empty());+ let source = ParquetSource::new(schema);++ assert!(!source.reverse_row_groups());+ }++ #[test]+ fn test_reverse_scan_with_setter() {+ use arrow::datatypes::Schema;++ let schema = Arc::new(Schema::empty());++ let source = ParquetSource::new(schema.clone()).with_reverse_row_groups(true);+ assert!(source.reverse_row_groups());++ let source = source.with_reverse_row_groups(false);+ assert!(!source.reverse_row_groups());+ }++ #[test]+ fn test_reverse_scan_clone_preserves_value() {+ use arrow::datatypes::Schema;++ let schema = Arc::new(Schema::empty());++ let source = ParquetSource::new(schema).with_reverse_row_groups(true);+ let cloned = source.clone();++ assert!(cloned.reverse_row_groups());+ assert_eq!(source.reverse_row_groups(), cloned.reverse_row_groups());+ }++ #[test]+ fn test_reverse_scan_with_other_options() {+ use arrow::datatypes::Schema;+ use datafusion_common::config::TableParquetOptions;++ let schema = Arc::new(Schema::empty());+ let options = TableParquetOptions::default();++ let source = ParquetSource::new(schema)+ .with_table_parquet_options(options)+ .with_metadata_size_hint(8192)+ .with_reverse_row_groups(true);++ assert!(source.reverse_row_groups());+ assert_eq!(source.metadata_size_hint, Some(8192));+ }++ #[test]+ fn test_reverse_scan_builder_pattern() {+ use arrow::datatypes::Schema;++ let schema = Arc::new(Schema::empty());++ let source = ParquetSource::new(schema)+ .with_reverse_row_groups(true)+ .with_reverse_row_groups(false)+ .with_reverse_row_groups(true);++ assert!(source.reverse_row_groups());+ }++ #[test]+ fn test_reverse_scan_independent_of_predicate() {+ use arrow::datatypes::Schema;+ use datafusion_physical_expr::expressions::lit;++ let schema = Arc::new(Schema::empty());+ let predicate = lit(true);++ let source = ParquetSource::new(schema)+ .with_predicate(predicate)+ .with_reverse_row_groups(true);++ assert!(source.reverse_row_groups());+ assert!(source.filter().is_some());+ }}diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex e25a2e889e..2c69987f91 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -31,9 +31,11 @@ use datafusion_common::{Result, not_impl_err};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use datafusion_physical_plan::DisplayFormatType;+use datafusion_physical_plan::SortOrderPushdownResult;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPropagation, PushedDown};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use object_store::ObjectStore;/// Helper function to convert any type implementing FileSource to Arc<dyn FileSource>@@ -129,6 +131,21 @@ pub trait FileSource: Send + Sync {))}+ /// Try to create a new FileSource that can produce data in the specified sort order.+ ///+ /// # Returns+ /// * `Exact` - Created a source that guarantees perfect ordering+ /// * `Inexact` - Created a source optimized for ordering (e.g., reordered files) but not perfectly sorted+ /// * `Unsupported` - Cannot optimize for this ordering+ ///+ /// Default implementation returns `Unsupported`.+ fn try_reverse_output(+ &self,+ _order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>> {+ Ok(SortOrderPushdownResult::Unsupported)+ }+/// Try to push down a projection into a this FileSource.////// `FileSource` implementations that support projection pushdown should@@ -183,4 +200,22 @@ pub trait FileSource: Send + Sync {fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {None}++ /// Set the file ordering information+ ///+ /// This allows the file source to know how the files are sorted,+ /// enabling it to make informed decisions about sort pushdown.+ ///+ /// # Default Implementation+ ///+ /// Returns `not_impl_err!`. FileSource implementations that support+ /// sort optimization should override this method.+ fn with_file_ordering_info(+ &self,+ _ordering: Option<LexOrdering>,+ ) -> Result<Arc<dyn FileSource>> {+ // Default: clone self without modification+ // ParquetSource will override this+ not_impl_err!("with_file_ordering_info not implemented for this FileSource")+ }}diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex ad89406014..16a010cf27 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -34,25 +34,26 @@ use datafusion_execution::{SendableRecordBatchStream, TaskContext, object_store::ObjectStoreUrl,};use datafusion_expr::Operator;++use datafusion_physical_expr::equivalence::project_orderings;use datafusion_physical_expr::expressions::{BinaryExpr, Column};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::utils::reassign_expr_columns;use datafusion_physical_expr::{EquivalenceProperties, Partitioning, split_conjunction};use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::PhysicalExpr;-use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_plan::SortOrderPushdownResult;+use datafusion_physical_plan::coop::cooperative;+use datafusion_physical_plan::execution_plan::SchedulingType;use datafusion_physical_plan::{DisplayAs, DisplayFormatType,display::{ProjectSchemaDisplay, display_orderings},filter_pushdown::FilterPushdownPropagation,metrics::ExecutionPlanMetricsSet,};-use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};--use datafusion_physical_expr::equivalence::project_orderings;-use datafusion_physical_plan::coop::cooperative;-use datafusion_physical_plan::execution_plan::SchedulingType;use log::{debug, warn};+use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};/// The base configurations for a [`DataSourceExec`], the a physical plan for/// any given file format.@@ -845,6 +846,45 @@ impl DataSource for FileScanConfig {}}}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {+ let file_ordering = self.output_ordering.first().cloned();++ if file_ordering.is_none() {+ return Ok(SortOrderPushdownResult::Unsupported);+ }++ // Use the trait method instead of downcasting+ // Try to provide file ordering info to the source+ // If not supported (e.g., CsvSource), fall back to original source+ let file_source_with_ordering = self+ .file_source+ .with_file_ordering_info(file_ordering)+ .unwrap_or_else(|_| Arc::clone(&self.file_source));++ // Try to reverse the datasource with ordering info,+ // and currently only ParquetSource supports it with inexact reverse with row groups.+ let pushdown_result = file_source_with_ordering.try_reverse_output(order)?;++ match pushdown_result {+ SortOrderPushdownResult::Exact { inner } => {+ Ok(SortOrderPushdownResult::Exact {+ inner: self.rebuild_with_source(inner, true)?,+ })+ }+ SortOrderPushdownResult::Inexact { inner } => {+ Ok(SortOrderPushdownResult::Inexact {+ inner: self.rebuild_with_source(inner, false)?,+ })+ }+ SortOrderPushdownResult::Unsupported => {+ Ok(SortOrderPushdownResult::Unsupported)+ }+ }+ }}impl FileScanConfig {@@ -1107,6 +1147,36 @@ impl FileScanConfig {pub fn file_source(&self) -> &Arc<dyn FileSource> {&self.file_source}++ /// Helper: Rebuild FileScanConfig with new file source+ fn rebuild_with_source(+ &self,+ new_file_source: Arc<dyn FileSource>,+ is_exact: bool,+ ) -> Result<Arc<dyn DataSource>> {+ let mut new_config = self.clone();++ // Reverse file groups (FileScanConfig's responsibility)+ new_config.file_groups = new_config+ .file_groups+ .into_iter()+ .map(|group| {+ let mut files = group.into_inner();+ files.reverse();+ files.into()+ })+ .collect();++ new_config.file_source = new_file_source;++ // Phase 1: Clear output_ordering for Inexact+ // (we're only reversing row groups, not guaranteeing perfect ordering)+ if !is_exact {+ new_config.output_ordering = vec![];+ }++ Ok(Arc::new(new_config))+ }}impl Debug for FileScanConfig {diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex 0945ffc94c..a3892dfac9 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -40,7 +40,8 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::{Constraints, Result, Statistics};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};-use datafusion_physical_expr_common::sort_expr::LexOrdering;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_plan::SortOrderPushdownResult;use datafusion_physical_plan::filter_pushdown::{ChildPushdownResult, FilterPushdownPhase, FilterPushdownPropagation, PushedDown,};@@ -190,6 +191,25 @@ pub trait DataSource: Send + Sync + Debug {vec![PushedDown::No; filters.len()],))}++ /// Try to create a new DataSource that produces data in the specified sort order.+ ///+ /// # Arguments+ /// * `order` - The desired output ordering+ ///+ /// # Returns+ /// * `Ok(SortOrderPushdownResult::Exact { .. })` - Created a source that guarantees exact ordering+ /// * `Ok(SortOrderPushdownResult::Inexact { .. })` - Created a source optimized for the ordering+ /// * `Ok(SortOrderPushdownResult::Unsupported)` - Cannot optimize for this ordering+ /// * `Err(e)` - Error occurred+ ///+ /// Default implementation returns `Unsupported`.+ fn try_pushdown_sort(+ &self,+ _order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {+ Ok(SortOrderPushdownResult::Unsupported)+ }}/// [`ExecutionPlan`] that reads one or more files@@ -360,6 +380,19 @@ impl ExecutionPlan for DataSourceExec {}),}}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ // Delegate to the data source and wrap result with DataSourceExec+ self.data_source+ .try_pushdown_sort(order)?+ .try_map(|new_data_source| {+ let new_exec = self.clone().with_data_source(new_data_source);+ Ok(Arc::new(new_exec) as Arc<dyn ExecutionPlan>)+ })+ }}impl DataSourceExec {diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rsindex 5dcdab1552..30ba7de76a 100644--- a/datafusion/execution/src/config.rs+++ b/datafusion/execution/src/config.rs@@ -424,6 +424,13 @@ impl SessionConfig {self.options.optimizer.enable_round_robin_repartition}+ /// Enables or disables sort pushdown optimization, and currently only+ /// applies to Parquet data source.+ pub fn with_enable_sort_pushdown(mut self, enabled: bool) -> Self {+ self.options_mut().optimizer.enable_sort_pushdown = enabled;+ self+ }+/// Set the size of [`sort_spill_reservation_bytes`] to control/// memory pre-reservation///diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rsindex 1a49db3d58..e8558c7643 100644--- a/datafusion/physical-expr-common/src/sort_expr.rs+++ b/datafusion/physical-expr-common/src/sort_expr.rs@@ -426,6 +426,62 @@ impl LexOrdering {self.exprs.truncate(len);true}++ /// Check if reversing this ordering would satisfy another ordering requirement.+ ///+ /// This supports **prefix matching**: if this ordering is `[A DESC, B ASC]`+ /// and `other` is `[A ASC]`, reversing this gives `[A ASC, B DESC]`, which+ /// satisfies `other` since `[A ASC]` is a prefix.+ ///+ /// # Arguments+ /// * `other` - The ordering requirement to check against+ ///+ /// # Returns+ /// `true` if reversing this ordering would satisfy `other`+ ///+ /// # Example+ /// ```text+ /// self: [number DESC, letter ASC]+ /// other: [number ASC]+ /// After reversing self: [number ASC, letter DESC] ✓ Prefix match!+ /// ```+ pub fn is_reverse(&self, other: &LexOrdering) -> bool {+ let self_exprs = self.as_ref();+ let other_exprs = other.as_ref();++ if other_exprs.len() > self_exprs.len() {+ return false;+ }++ other_exprs.iter().zip(self_exprs.iter()).all(|(req, cur)| {+ req.expr.eq(&cur.expr) && is_reversed_sort_options(&req.options, &cur.options)+ })+ }+}++/// Check if two SortOptions represent reversed orderings.+///+/// Returns `true` if both `descending` and `nulls_first` are opposite.+///+/// # Example+/// ```+/// use arrow::compute::SortOptions;+/// # use datafusion_physical_expr_common::sort_expr::is_reversed_sort_options;+///+/// let asc_nulls_last = SortOptions {+/// descending: false,+/// nulls_first: false,+/// };+/// let desc_nulls_first = SortOptions {+/// descending: true,+/// nulls_first: true,+/// };+///+/// assert!(is_reversed_sort_options(&asc_nulls_last, &desc_nulls_first));+/// assert!(is_reversed_sort_options(&desc_nulls_first, &asc_nulls_last));+/// ```+pub fn is_reversed_sort_options(lhs: &SortOptions, rhs: &SortOptions) -> bool {+ lhs.descending != rhs.descending && lhs.nulls_first != rhs.nulls_first}impl PartialEq for LexOrdering {@@ -732,3 +788,50 @@ impl DerefMut for OrderingRequirements {}}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_is_reversed_sort_options() {+ // Test basic reversal: ASC NULLS LAST ↔ DESC NULLS FIRST+ let asc_nulls_last = SortOptions {+ descending: false,+ nulls_first: false,+ };+ let desc_nulls_first = SortOptions {+ descending: true,+ nulls_first: true,+ };+ assert!(is_reversed_sort_options(&asc_nulls_last, &desc_nulls_first));+ assert!(is_reversed_sort_options(&desc_nulls_first, &asc_nulls_last));++ // Test another reversal: ASC NULLS FIRST ↔ DESC NULLS LAST+ let asc_nulls_first = SortOptions {+ descending: false,+ nulls_first: true,+ };+ let desc_nulls_last = SortOptions {+ descending: true,+ nulls_first: false,+ };+ assert!(is_reversed_sort_options(&asc_nulls_first, &desc_nulls_last));+ assert!(is_reversed_sort_options(&desc_nulls_last, &asc_nulls_first));++ // Test non-reversal: same options+ assert!(!is_reversed_sort_options(&asc_nulls_last, &asc_nulls_last));+ assert!(!is_reversed_sort_options(+ &desc_nulls_first,+ &desc_nulls_first+ ));++ // Test non-reversal: only descending differs+ assert!(!is_reversed_sort_options(&asc_nulls_last, &desc_nulls_last));+ assert!(!is_reversed_sort_options(&desc_nulls_last, &asc_nulls_last));++ // Test non-reversal: only nulls_first differs+ assert!(!is_reversed_sort_options(&asc_nulls_last, &asc_nulls_first));+ assert!(!is_reversed_sort_options(&asc_nulls_first, &asc_nulls_last));+ }+}diff --git a/datafusion/physical-optimizer/src/lib.rs b/datafusion/physical-optimizer/src/lib.rsindex cf8cf33cea..1b45f02ebd 100644--- a/datafusion/physical-optimizer/src/lib.rs+++ b/datafusion/physical-optimizer/src/lib.rs@@ -42,6 +42,7 @@ pub mod optimizer;pub mod output_requirements;pub mod projection_pushdown;pub use datafusion_pruning as pruning;+pub mod pushdown_sort;pub mod sanity_checker;pub mod topk_aggregation;pub mod update_aggr_exprs;diff --git a/datafusion/physical-optimizer/src/optimizer.rs b/datafusion/physical-optimizer/src/optimizer.rsindex f8e2e9950a..aa1975d98d 100644--- a/datafusion/physical-optimizer/src/optimizer.rs+++ b/datafusion/physical-optimizer/src/optimizer.rs@@ -37,6 +37,7 @@ use crate::topk_aggregation::TopKAggregation;use crate::update_aggr_exprs::OptimizeAggregateOrder;use crate::limit_pushdown_past_window::LimitPushPastWindows;+use crate::pushdown_sort::PushdownSort;use datafusion_common::Result;use datafusion_common::config::ConfigOptions;use datafusion_physical_plan::ExecutionPlan;@@ -145,6 +146,8 @@ impl PhysicalOptimizer {// are not present, the load of executors such as join or union will be// reduced by narrowing their input tables.Arc::new(ProjectionPushdown::new()),+ // PushdownSort: Detect sorts that can be pushed down to data sources.+ Arc::new(PushdownSort::new()),Arc::new(EnsureCooperative::new()),// This FilterPushdown handles dynamic filters that may have references to the source ExecutionPlan.// Therefore it should be run at the end of the optimization process since any changes to the plan may break the dynamic filter's references.diff --git a/datafusion/physical-optimizer/src/pushdown_sort.rs b/datafusion/physical-optimizer/src/pushdown_sort.rsnew file mode 100644index 0000000000..1fa15492d2--- /dev/null+++ b/datafusion/physical-optimizer/src/pushdown_sort.rs@@ -0,0 +1,129 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Sort Pushdown Optimization+//!+//! This optimizer attempts to push sort requirements down through the execution plan+//! tree to data sources that can natively handle them (e.g., by scanning files in+//! reverse order).+//!+//! ## How it works+//!+//! 1. Detects `SortExec` nodes in the plan+//! 2. Calls `try_pushdown_sort()` on the input to recursively push the sort requirement+//! 3. Each node type defines its own pushdown behavior:+//! - **Transparent nodes** (CoalesceBatchesExec, RepartitionExec, etc.) delegate to+//! their children and wrap the result+//! - **Data sources** (DataSourceExec) check if they can optimize for the ordering+//! - **Blocking nodes** return `Unsupported` to stop pushdown+//! 4. Based on the result:+//! - `Exact`: Remove the Sort operator (data source guarantees perfect ordering)+//! - `Inexact`: Keep Sort but use optimized input (enables early termination for TopK)+//! - `Unsupported`: No change+//!+//! ## Current capabilities (Phase 1)+//!+//! - Reverse scan optimization: when required sort is the reverse of the data source's+//! natural ordering, enable reverse scanning (reading row groups in reverse order)+//! - Supports prefix matching: if data has ordering [A DESC, B ASC] and query needs+//! [A ASC], reversing gives [A ASC, B DESC] which satisfies the requirement+//!+//! TODO Issue: <https://github.com/apache/datafusion/issues/19329>+//! ## Future enhancements (Phase 2),+//!+//! - File reordering based on statistics+//! - Return `Exact` when files are known to be perfectly sorted+//! - Complete Sort elimination when ordering is guaranteed++use crate::PhysicalOptimizerRule;+use datafusion_common::Result;+use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::SortOrderPushdownResult;+use datafusion_physical_plan::sorts::sort::SortExec;+use std::sync::Arc;++/// A PhysicalOptimizerRule that attempts to push down sort requirements to data sources.+///+/// See module-level documentation for details.+#[derive(Debug, Clone, Default)]+pub struct PushdownSort;++impl PushdownSort {+ pub fn new() -> Self {+ Self {}+ }+}++impl PhysicalOptimizerRule for PushdownSort {+ fn optimize(+ &self,+ plan: Arc<dyn ExecutionPlan>,+ config: &ConfigOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Check if sort pushdown optimization is enabled+ if !config.optimizer.enable_sort_pushdown {+ return Ok(plan);+ }++ // Use transform_down to find and optimize all SortExec nodes (including nested ones)+ plan.transform_down(|plan: Arc<dyn ExecutionPlan>| {+ // Check if this is a SortExec+ let Some(sort_exec) = plan.as_any().downcast_ref::<SortExec>() else {+ return Ok(Transformed::no(plan));+ };++ let sort_input = Arc::clone(sort_exec.input());+ let required_ordering = sort_exec.expr();++ // Try to push the sort requirement down through the plan tree+ // Each node type defines its own pushdown behavior via try_pushdown_sort()+ match sort_input.try_pushdown_sort(required_ordering)? {+ SortOrderPushdownResult::Exact { inner } => {+ // Data source guarantees perfect ordering - remove the Sort operator+ Ok(Transformed::yes(inner))+ }+ SortOrderPushdownResult::Inexact { inner } => {+ // Data source is optimized for the ordering but not perfectly sorted+ // Keep the Sort operator but use the optimized input+ // Benefits: TopK queries can terminate early, better cache locality+ Ok(Transformed::yes(Arc::new(+ SortExec::new(required_ordering.clone(), inner)+ .with_fetch(sort_exec.fetch())+ .with_preserve_partitioning(+ sort_exec.preserve_partitioning(),+ ),+ )))+ }+ SortOrderPushdownResult::Unsupported => {+ // Cannot optimize for this ordering - no change+ Ok(Transformed::no(plan))+ }+ }+ })+ .data()+ }++ fn name(&self) -> &str {+ "PushdownSort"+ }++ fn schema_check(&self) -> bool {+ true+ }+}diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex eb3c3b5bef..494b5d60fb 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -40,7 +40,9 @@ use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,};+use crate::sort_pushdown::SortOrderPushdownResult;use datafusion_common::config::ConfigOptions;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use futures::ready;use futures::stream::{Stream, StreamExt};@@ -241,6 +243,20 @@ impl ExecutionPlan for CoalesceBatchesExec {) -> Result<FilterPushdownPropagation<Arc<dyn ExecutionPlan>>> {Ok(FilterPushdownPropagation::if_all(child_pushdown_result))}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ // CoalesceBatchesExec is transparent for sort ordering - it preserves order+ // Delegate to the child and wrap with a new CoalesceBatchesExec+ self.input.try_pushdown_sort(order)?.try_map(|new_input| {+ Ok(Arc::new(+ CoalesceBatchesExec::new(new_input, self.target_batch_size)+ .with_fetch(self.fetch),+ ) as Arc<dyn ExecutionPlan>)+ })+ }}/// Stream for [`CoalesceBatchesExec`]. See [`CoalesceBatchesExec`] for more details.diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 7f207d7f1e..d83f90eb3d 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -30,7 +30,9 @@ use super::{use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};use crate::projection::{ProjectionExec, make_with_child};+use crate::sort_pushdown::SortOrderPushdownResult;use crate::{DisplayFormatType, ExecutionPlan, Partitioning};+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_common::config::ConfigOptions;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};@@ -284,6 +286,42 @@ impl ExecutionPlan for CoalescePartitionsExec {) -> Result<FilterDescription> {FilterDescription::from_children(parent_filters, &self.children())}++ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ // CoalescePartitionsExec merges multiple partitions into one, which loses+ // global ordering. However, we can still push the sort requirement down+ // to optimize individual partitions - the Sort operator above will handle+ // the global ordering.+ //+ // Note: The result will always be at most Inexact (never Exact) when there+ // are multiple partitions, because merging destroys global ordering.+ let result = self.input.try_pushdown_sort(order)?;++ // If we have multiple partitions, we can't return Exact even if the+ // underlying source claims Exact - merging destroys global ordering+ let has_multiple_partitions =+ self.input.output_partitioning().partition_count() > 1;++ result+ .try_map(|new_input| {+ Ok(+ Arc::new(+ CoalescePartitionsExec::new(new_input).with_fetch(self.fetch),+ ) as Arc<dyn ExecutionPlan>,+ )+ })+ .map(|r| {+ if has_multiple_partitions {+ // Downgrade Exact to Inexact when merging multiple partitions+ r.into_inexact()+ } else {+ r+ }+ })+ }}#[cfg(test)]diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex b7967bb7bb..06da0b8933 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -22,6 +22,7 @@ use crate::filter_pushdown::{};pub use crate::metrics::Metric;pub use crate::ordering::InputOrderMode;+use crate::sort_pushdown::SortOrderPushdownResult;pub use crate::stream::EmptyRecordBatchStream;pub use datafusion_common::hash_utils;@@ -54,7 +55,9 @@ use datafusion_common::{use datafusion_common_runtime::JoinSet;use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;-use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};+use datafusion_physical_expr_common::sort_expr::{+ LexOrdering, OrderingRequirements, PhysicalSortExpr,+};use futures::stream::{StreamExt, TryStreamExt};@@ -682,6 +685,29 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {) -> Option<Arc<dyn ExecutionPlan>> {None}++ /// Try to push down sort ordering requirements to this node.+ ///+ /// This method is called during sort pushdown optimization to determine if this+ /// node can optimize for a requested sort ordering. Implementations should:+ ///+ /// - Return [`SortOrderPushdownResult::Exact`] if the node can guarantee the exact+ /// ordering (allowing the Sort operator to be removed)+ /// - Return [`SortOrderPushdownResult::Inexact`] if the node can optimize for the+ /// ordering but cannot guarantee perfect sorting (Sort operator is kept)+ /// - Return [`SortOrderPushdownResult::Unsupported`] if the node cannot optimize+ /// for the ordering+ ///+ /// For transparent nodes (that preserve ordering), implement this to delegate to+ /// children and wrap the result with a new instance of this node.+ ///+ /// Default implementation returns `Unsupported`.+ fn try_pushdown_sort(+ &self,+ _order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ Ok(SortOrderPushdownResult::Unsupported)+ }}/// [`ExecutionPlan`] Invariant Leveldiff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 849b34e703..ec8e154cae 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -50,6 +50,7 @@ pub use crate::execution_plan::{};pub use crate::metrics::Metric;pub use crate::ordering::InputOrderMode;+pub use crate::sort_pushdown::SortOrderPushdownResult;pub use crate::stream::EmptyRecordBatchStream;pub use crate::topk::TopK;pub use crate::visitor::{ExecutionPlanVisitor, accept, visit_execution_plan};@@ -83,6 +84,7 @@ pub mod placeholder_row;pub mod projection;pub mod recursive_query;pub mod repartition;+pub mod sort_pushdown;pub mod sorts;pub mod spill;pub mod stream;diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 9d437dbcf6..5c9472182b 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -61,6 +61,8 @@ use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,};+use crate::sort_pushdown::SortOrderPushdownResult;+use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::stream::Stream;use futures::{FutureExt, StreamExt, TryStreamExt, ready};@@ -1094,6 +1096,27 @@ impl ExecutionPlan for RepartitionExec {Ok(FilterPushdownPropagation::if_all(child_pushdown_result))}+ fn try_pushdown_sort(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn ExecutionPlan>>> {+ // RepartitionExec only maintains input order if preserve_order is set+ // or if there's only one partition+ if !self.maintains_input_order()[0] {+ return Ok(SortOrderPushdownResult::Unsupported);+ }++ // Delegate to the child and wrap with a new RepartitionExec+ self.input.try_pushdown_sort(order)?.try_map(|new_input| {+ let mut new_repartition =+ RepartitionExec::try_new(new_input, self.partitioning().clone())?;+ if self.preserve_order {+ new_repartition = new_repartition.with_preserve_order();+ }+ Ok(Arc::new(new_repartition) as Arc<dyn ExecutionPlan>)+ })+ }+fn repartitioned(&self,target_partitions: usize,diff --git a/datafusion/physical-plan/src/sort_pushdown.rs b/datafusion/physical-plan/src/sort_pushdown.rsnew file mode 100644index 0000000000..8432fd5dab--- /dev/null+++ b/datafusion/physical-plan/src/sort_pushdown.rs@@ -0,0 +1,120 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Sort pushdown types for physical execution plans.+//!+//! This module provides types used for pushing sort ordering requirements+//! down through the execution plan tree to data sources.++/// Result of attempting to push down sort ordering to a node.+///+/// Used by [`ExecutionPlan::try_pushdown_sort`] to communicate+/// whether and how sort ordering was successfully pushed down.+///+/// [`ExecutionPlan::try_pushdown_sort`]: crate::ExecutionPlan::try_pushdown_sort+#[derive(Debug, Clone)]+pub enum SortOrderPushdownResult<T> {+ /// The source can guarantee exact ordering (data is perfectly sorted).+ ///+ /// When this is returned, the optimizer can safely remove the Sort operator+ /// entirely since the data source guarantees the requested ordering.+ Exact {+ /// The optimized node that provides exact ordering+ inner: T,+ },+ /// The source has optimized for the ordering but cannot guarantee perfect sorting.+ ///+ /// This indicates the data source has been optimized (e.g., reordered files/row groups+ /// based on statistics, enabled reverse scanning) but the data may not be perfectly+ /// sorted. The optimizer should keep the Sort operator but benefits from the+ /// optimization (e.g., faster TopK queries due to early termination).+ Inexact {+ /// The optimized node that provides approximate ordering+ inner: T,+ },+ /// The source cannot optimize for this ordering.+ ///+ /// The data source does not support the requested sort ordering and no+ /// optimization was applied.+ Unsupported,+}++impl<T> SortOrderPushdownResult<T> {+ /// Extract the inner value if present+ pub fn into_inner(self) -> Option<T> {+ match self {+ Self::Exact { inner } | Self::Inexact { inner } => Some(inner),+ Self::Unsupported => None,+ }+ }++ /// Map the inner value to a different type while preserving the variant.+ pub fn map<U, F: FnOnce(T) -> U>(self, f: F) -> SortOrderPushdownResult<U> {+ match self {+ Self::Exact { inner } => SortOrderPushdownResult::Exact { inner: f(inner) },+ Self::Inexact { inner } => {+ SortOrderPushdownResult::Inexact { inner: f(inner) }+ }+ Self::Unsupported => SortOrderPushdownResult::Unsupported,+ }+ }++ /// Try to map the inner value, returning an error if the function fails.+ pub fn try_map<U, E, F: FnOnce(T) -> Result<U, E>>(+ self,+ f: F,+ ) -> Result<SortOrderPushdownResult<U>, E> {+ match self {+ Self::Exact { inner } => {+ Ok(SortOrderPushdownResult::Exact { inner: f(inner)? })+ }+ Self::Inexact { inner } => {+ Ok(SortOrderPushdownResult::Inexact { inner: f(inner)? })+ }+ Self::Unsupported => Ok(SortOrderPushdownResult::Unsupported),+ }+ }++ /// Convert this result to `Inexact`, downgrading `Exact` if present.+ ///+ /// This is useful when an operation (like merging multiple partitions)+ /// cannot guarantee exact ordering even if the input provides it.+ ///+ /// # Examples+ ///+ /// ```+ /// # use datafusion_physical_plan::SortOrderPushdownResult;+ /// let exact = SortOrderPushdownResult::Exact { inner: 42 };+ /// let inexact = exact.into_inexact();+ /// assert!(matches!(inexact, SortOrderPushdownResult::Inexact { inner: 42 }));+ ///+ /// let already_inexact = SortOrderPushdownResult::Inexact { inner: 42 };+ /// let still_inexact = already_inexact.into_inexact();+ /// assert!(matches!(still_inexact, SortOrderPushdownResult::Inexact { inner: 42 }));+ ///+ /// let unsupported = SortOrderPushdownResult::<i32>::Unsupported;+ /// let still_unsupported = unsupported.into_inexact();+ /// assert!(matches!(still_unsupported, SortOrderPushdownResult::Unsupported));+ /// ```+ pub fn into_inexact(self) -> Self {+ match self {+ Self::Exact { inner } => Self::Inexact { inner },+ Self::Inexact { inner } => Self::Inexact { inner },+ Self::Unsupported => Self::Unsupported,+ }+ }+}diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rsindex 7c08aaad98..0bf87203ac 100644--- a/datafusion/proto-common/src/generated/pbjson.rs+++ b/datafusion/proto-common/src/generated/pbjson.rs@@ -1111,7 +1111,7 @@ impl serde::Serialize for ColumnStats {struct_ser.serialize_field("distinctCount", v)?;}if let Some(v) = self.byte_size.as_ref() {- struct_ser.serialize_field("ByteSize", v)?;+ struct_ser.serialize_field("byteSize", v)?;}struct_ser.end()}@@ -1134,7 +1134,7 @@ impl<'de> serde::Deserialize<'de> for ColumnStats {"distinct_count","distinctCount","byte_size",- "ByteSize",+ "byteSize",];#[allow(clippy::enum_variant_names)]@@ -1144,7 +1144,6 @@ impl<'de> serde::Deserialize<'de> for ColumnStats {SumValue,NullCount,DistinctCount,-ByteSize,}impl<'de> serde::Deserialize<'de> for GeneratedField {@@ -1172,7 +1171,7 @@ impl<'de> serde::Deserialize<'de> for ColumnStats {"sumValue" | "sum_value" => Ok(GeneratedField::SumValue),"nullCount" | "null_count" => Ok(GeneratedField::NullCount),"distinctCount" | "distinct_count" => Ok(GeneratedField::DistinctCount),- "ByteSize" | "byte_size" => Ok(GeneratedField::ByteSize),+ "byteSize" | "byte_size" => Ok(GeneratedField::ByteSize),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -1232,7 +1231,7 @@ impl<'de> serde::Deserialize<'de> for ColumnStats {}GeneratedField::ByteSize => {if byte_size__.is_some() {- return Err(serde::de::Error::duplicate_field("ByteSize"));+ return Err(serde::de::Error::duplicate_field("byteSize"));}byte_size__ = map_.next_value()?;}diff --git a/datafusion/sqllogictest/test_files/create_external_table.slt b/datafusion/sqllogictest/test_files/create_external_table.sltindex 1e6183f48b..0b15a7f8ec 100644--- a/datafusion/sqllogictest/test_files/create_external_table.slt+++ b/datafusion/sqllogictest/test_files/create_external_table.slt@@ -264,7 +264,7 @@ logical_plan02)--TableScan: t projection=[id]physical_plan01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], file_type=parquet, reverse_row_groups=truestatement okDROP TABLE t;diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex 3e403171e0..3d08cdf751 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -426,3 +426,329 @@ SET datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true;statement okSET datafusion.optimizer.enable_dynamic_filter_pushdown = true;++# Test 6: Sort Pushdown for ordered Parquet files++# Create a sorted dataset+statement ok+CREATE TABLE sorted_data(id INT, value INT, name VARCHAR) AS VALUES+(1, 100, 'a'),+(2, 200, 'b'),+(3, 300, 'c'),+(4, 400, 'd'),+(5, 500, 'e'),+(6, 600, 'f'),+(7, 700, 'g'),+(8, 800, 'h'),+(9, 900, 'i'),+(10, 1000, 'j');++# Copy to parquet with sorting+query I+COPY (SELECT * FROM sorted_data ORDER BY id ASC)+TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet';+----+10++statement ok+CREATE EXTERNAL TABLE sorted_parquet(id INT, value INT, name VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet'+WITH ORDER (id ASC);++# Test 6.1: Sort pushdown with DESC (opposite of ASC)+# Should show reverse_row_groups=true+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Test 6.2: Verify results are correct+query IIT+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+10 1000 j+9 900 i+8 800 h++# Test 6.3: Should NOT apply for ASC (same direction)+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id ASC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id ASC NULLS LAST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test 6.4: Disable sort pushdown+statement ok+SET datafusion.optimizer.enable_sort_pushdown = false;++query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]++# Re-enable+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 6.5: With OFFSET+query TT+EXPLAIN SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;+----+logical_plan+01)Limit: skip=2, fetch=3+02)--Sort: sorted_parquet.id DESC NULLS FIRST, fetch=5+03)----TableScan: sorted_parquet projection=[id, value, name]+physical_plan+01)GlobalLimitExec: skip=2, fetch=3+02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query IIT+SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;+----+8 800 h+7 700 g+6 600 f++# Test 6.6: Reverse scan with row selection (page index pruning)+# This tests that when reverse_row_groups=true, the RowSelection is also properly reversed++# Create a dataset with multiple row groups and enable page index+statement ok+CREATE TABLE multi_rg_data(id INT, category VARCHAR, value INT) AS VALUES+(1, 'alpha', 10),+(2, 'alpha', 20),+(3, 'beta', 30),+(4, 'beta', 40),+(5, 'gamma', 50),+(6, 'gamma', 60),+(7, 'delta', 70),+(8, 'delta', 80);++# Write with small row groups (2 rows each = 4 row groups)+statement ok+SET datafusion.execution.parquet.max_row_group_size = 2;++query I+COPY (SELECT * FROM multi_rg_data ORDER BY id ASC)+TO 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet';+----+8++# Reset row group size+statement ok+SET datafusion.execution.parquet.max_row_group_size = 1048576;++statement ok+CREATE EXTERNAL TABLE multi_rg_sorted(id INT, category VARCHAR, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet'+WITH ORDER (id ASC);++# Enable page index for better pruning+statement ok+SET datafusion.execution.parquet.enable_page_index = true;++statement ok+SET datafusion.execution.parquet.pushdown_filters = true;++# Test with reverse scan and filter that prunes some row groups+# This will create a RowSelection with partial row group scans+query TT+EXPLAIN SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+logical_plan+01)Sort: multi_rg_sorted.id DESC NULLS FIRST, fetch=5+02)--Filter: multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")+03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]+physical_plan+01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]++# Verify the results are correct despite reverse scanning with row selection+# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+6 gamma 60+5 gamma 50+2 alpha 20+1 alpha 10++# Test with more complex selection pattern+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('beta', 'delta')+ORDER BY id DESC;+----+8 delta 80+7 delta 70+4 beta 40+3 beta 30++# Test forward scan for comparison (should give same logical results in ASC order)+query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id ASC;+----+1 alpha 10+2 alpha 20+5 gamma 50+6 gamma 60++# Disable reverse scan and verify it still works+statement ok+SET datafusion.optimizer.enable_sort_pushdown = false;++query ITI+SELECT * FROM multi_rg_sorted+WHERE category IN ('alpha', 'gamma')+ORDER BY id DESC LIMIT 5;+----+6 gamma 60+5 gamma 50+2 alpha 20+1 alpha 10++# Re-enable+statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Test 6.7: Sort pushdown with more than one partition+# Create multiple parquet files to trigger it++# Split data into multiple files+statement ok+CREATE TABLE sorted_data_part1(id INT, value INT, name VARCHAR) AS VALUES+(1, 100, 'a'),+(2, 200, 'b'),+(3, 300, 'c');++statement ok+CREATE TABLE sorted_data_part2(id INT, value INT, name VARCHAR) AS VALUES+(4, 400, 'd'),+(5, 500, 'e'),+(6, 600, 'f');++statement ok+CREATE TABLE sorted_data_part3(id INT, value INT, name VARCHAR) AS VALUES+(7, 700, 'g'),+(8, 800, 'h'),+(9, 900, 'i'),+(10, 1000, 'j');++# Create directory for multi-file parquet+query I+COPY (SELECT * FROM sorted_data_part1 ORDER BY id ASC)+TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet';+----+3++query I+COPY (SELECT * FROM sorted_data_part2 ORDER BY id ASC)+TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet';+----+3++query I+COPY (SELECT * FROM sorted_data_part3 ORDER BY id ASC)+TO 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet';+----+4++# Create external table pointing to directory with multiple files+statement ok+CREATE EXTERNAL TABLE sorted_parquet_multi(id INT, value INT, name VARCHAR)+STORED AS PARQUET+LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/'+WITH ORDER (id ASC);++# Enable multiple partitions+statement ok+SET datafusion.execution.target_partitions = 4;++# Now we should see RepartitionExec because we have 3 input partitions (3 files)+query TT+EXPLAIN SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: sorted_parquet_multi.id DESC NULLS FIRST, fetch=3+02)--TableScan: sorted_parquet_multi projection=[id, value, name]+physical_plan+01)SortPreservingMergeExec: [id@0 DESC], fetch=3+02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++# Verify correctness with repartitioning and multiple files+query IIT+SELECT * FROM sorted_parquet_multi ORDER BY id DESC LIMIT 3;+----+10 1000 j+9 900 i+8 800 h++# Test ASC order (should not trigger reverse scan)+query IIT+SELECT * FROM sorted_parquet_multi ORDER BY id ASC LIMIT 3;+----+1 100 a+2 200 b+3 300 c++# Cleanup+statement ok+DROP TABLE sorted_data_part1;++statement ok+DROP TABLE sorted_data_part2;++statement ok+DROP TABLE sorted_data_part3;++statement ok+DROP TABLE sorted_parquet_multi;++# Reset to default+statement ok+SET datafusion.execution.target_partitions = 4;++# Cleanup+statement ok+DROP TABLE multi_rg_data;++statement ok+DROP TABLE multi_rg_sorted;++statement ok+SET datafusion.execution.parquet.enable_page_index = false;++statement ok+SET datafusion.execution.parquet.pushdown_filters = false;++# Cleanup+statement ok+DROP TABLE sorted_data;++statement ok+DROP TABLE sorted_parquet;++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.sltindex 9f8e264ee8..9087aee56d 100644--- a/datafusion/sqllogictest/test_files/explain.slt+++ b/datafusion/sqllogictest/test_files/explain.slt@@ -240,6 +240,7 @@ physical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVEphysical_plan after LimitPushdown SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVE+physical_plan after PushdownSort SAME TEXT AS ABOVEphysical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVE@@ -320,6 +321,7 @@ physical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVEphysical_plan after LimitPushdown DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]: ScanBytes=Exact(32)),(Col[1]: ScanBytes=Inexact(24)),(Col[2]: ScanBytes=Exact(32)),(Col[3]: ScanBytes=Exact(32)),(Col[4]: ScanBytes=Exact(32)),(Col[5]: ScanBytes=Exact(64)),(Col[6]: ScanBytes=Exact(32)),(Col[7]: ScanBytes=Exact(64)),(Col[8]: ScanBytes=Inexact(88)),(Col[9]: ScanBytes=Inexact(49)),(Col[10]: ScanBytes=Exact(64))]]physical_plan after ProjectionPushdown SAME TEXT AS ABOVE+physical_plan after PushdownSort SAME TEXT AS ABOVEphysical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVE@@ -364,6 +366,7 @@ physical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVEphysical_plan after LimitPushdown DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquetphysical_plan after ProjectionPushdown SAME TEXT AS ABOVE+physical_plan after PushdownSort SAME TEXT AS ABOVEphysical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVE@@ -599,6 +602,7 @@ physical_plan after LimitAggregation SAME TEXT AS ABOVEphysical_plan after LimitPushPastWindows SAME TEXT AS ABOVEphysical_plan after LimitPushdown SAME TEXT AS ABOVEphysical_plan after ProjectionPushdown SAME TEXT AS ABOVE+physical_plan after PushdownSort SAME TEXT AS ABOVEphysical_plan after EnsureCooperative SAME TEXT AS ABOVEphysical_plan after FilterPushdown(Post) SAME TEXT AS ABOVEphysical_plan after SanityCheckPlan SAME TEXT AS ABOVEdiff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.sltindex b5429d68c9..8c11667d97 100644--- a/datafusion/sqllogictest/test_files/information_schema.slt+++ b/datafusion/sqllogictest/test_files/information_schema.slt@@ -297,6 +297,7 @@ datafusion.optimizer.enable_dynamic_filter_pushdown truedatafusion.optimizer.enable_join_dynamic_filter_pushdown truedatafusion.optimizer.enable_piecewise_merge_join falsedatafusion.optimizer.enable_round_robin_repartition true+datafusion.optimizer.enable_sort_pushdown truedatafusion.optimizer.enable_topk_aggregation truedatafusion.optimizer.enable_topk_dynamic_filter_pushdown truedatafusion.optimizer.enable_window_limits true@@ -430,6 +431,7 @@ datafusion.optimizer.enable_dynamic_filter_pushdown true When set to true attempdatafusion.optimizer.enable_join_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase.datafusion.optimizer.enable_piecewise_merge_join false When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter.datafusion.optimizer.enable_round_robin_repartition true When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores+datafusion.optimizer.enable_sort_pushdown true Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: truedatafusion.optimizer.enable_topk_aggregation true When set to true, the optimizer will attempt to perform limit operations during aggregations, if possibledatafusion.optimizer.enable_topk_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase.datafusion.optimizer.enable_window_limits true When set to true, the optimizer will attempt to push limit operations past window functions, if possiblediff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.sltindex 7364fccd8e..aba468d21f 100644--- a/datafusion/sqllogictest/test_files/topk.slt+++ b/datafusion/sqllogictest/test_files/topk.slt@@ -340,7 +340,7 @@ explain select number, letter, age from partial_sorted order by number asc limit----physical_plan01)SortExec: TopK(fetch=3), expr=[number@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=truequery TTexplain select number, letter, age from partial_sorted order by letter asc, number desc limit 3;diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.mdindex 156df1d9d7..750543d0b5 100644--- a/docs/source/user-guide/configs.md+++ b/docs/source/user-guide/configs.md@@ -161,6 +161,7 @@ The following configuration settings are available:| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). || datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave || datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. |+| datafusion.optimizer.enable_sort_pushdown | true | Enable sort pushdown optimization. When enabled, attempts to push sort requirements down to data sources that can natively handle them (e.g., by reversing file/row group read order). Returns **inexact ordering**: Sort operator is kept for correctness, but optimized input enables early termination for TopK queries (ORDER BY ... LIMIT N), providing significant speedup. Memory: No additional overhead (only changes read order). Future: Will add option to detect perfectly sorted data and eliminate Sort completely. Default: true || datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans || datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans || datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans |