Skip to content

Apache DataFusion SQL Query Engine

feat: globally reorder files and row groups by statistics for TopK queries (#21956)

Qi Zhu102da39c7fce
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs
index 1418dcad11..505ccb3a1b 100644
--- a/datafusion/core/tests/dataframe/mod.rs
+++ b/datafusion/core/tests/dataframe/mod.rs
@@ -3268,7 +3268,7 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti
UnionExec
DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
- DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
");
Ok(())
}
@@ -3286,7 +3286,7 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti
UnionExec
DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
- DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
");
Ok(())
diff --git a/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs b/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs
index d14afaf1b3..316cb177fa 100644
--- a/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs
+++ b/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs
@@ -317,10 +317,23 @@ async fn test_fuzz_topk_filter_pushdown() {
.map(|col| orders.get(**col).unwrap())
.multi_cartesian_product()
{
+ // Add remaining columns as tiebreakers (ASC NULLS LAST)
+ // to ensure deterministic results when RG reorder changes
+ // the read order of rows with equal sort key values.
+ let tiebreakers: Vec<String> = ["id", "name", "department"]
+ .iter()
+ .filter(|c| {
+ !order_columns
+ .iter()
+ .take(num_order_by_columns)
+ .any(|oc| **oc == **c)
+ })
+ .map(|c| format!("{c} ASC NULLS LAST"))
+ .collect();
+ let all_orderings =
+ orderings.into_iter().chain(tiebreakers.iter()).join(", ");
let query = format!(
- "SELECT * FROM test_table ORDER BY {} LIMIT {}",
- orderings.into_iter().join(", "),
- limit
+ "SELECT * FROM test_table ORDER BY {all_orderings} LIMIT {limit}",
);
queries.push(query);
}
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
index e2700c3174..6a5833f43d 100644
--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs
@@ -85,7 +85,7 @@ fn test_sort_pushdown_basic_phase1() {
output:
Ok:
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -114,7 +114,7 @@ fn test_sort_with_limit_phase1() {
output:
Ok:
- SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -145,7 +145,7 @@ fn test_sort_multiple_columns_phase1() {
output:
Ok:
- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC, b@1 DESC NULLS LAST]
"
);
}
@@ -180,7 +180,7 @@ fn test_prefix_match_single_column() {
output:
Ok:
- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]
"
);
}
@@ -214,7 +214,7 @@ fn test_prefix_match_with_limit() {
output:
Ok:
- SortExec: TopK(fetch=100), expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST, b@1 ASC], reverse_row_groups=true
"
);
}
@@ -249,7 +249,7 @@ fn test_prefix_match_through_transparent_nodes() {
Ok:
- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]
"
);
}
@@ -285,8 +285,13 @@ fn test_exact_prefix_match_same_direction() {
}
#[test]
-fn test_no_prefix_match_longer_than_source() {
- // Test that prefix matching does NOT work if requested is longer than source
+fn test_inexact_pushdown_when_prefix_longer_than_source() {
+ // Source has [a DESC] ordering, request is [a ASC, b DESC] — longer
+ // than the source ordering so the prefix can't be matched. The
+ // primary sort column 'a' is in the file schema, so sort pushdown
+ // returns `Inexact` with `sort_order_for_reorder` set, drops the
+ // source's `output_ordering` (the runtime reorder invalidates it),
+ // and leaves the outer `SortExec` to enforce the full ordering.
let schema = schema();
// Source has [a DESC] ordering (single column)
@@ -310,7 +315,7 @@ fn test_no_prefix_match_longer_than_source() {
output:
Ok:
- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST], file_type=parquet
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC, b@1 DESC NULLS LAST]
"
);
}
@@ -343,7 +348,7 @@ fn test_sort_through_repartition() {
Ok:
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -375,7 +380,7 @@ fn test_nested_sorts() {
Ok:
- SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -435,7 +440,7 @@ fn test_sort_through_coalesce_partitions() {
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- CoalescePartitionsExec
- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -467,7 +472,7 @@ fn test_complex_plan_with_multiple_operators() {
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- CoalescePartitionsExec
- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -501,14 +506,19 @@ fn test_multiple_sorts_different_columns() {
Ok:
- SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
#[test]
-fn test_no_pushdown_for_unordered_source() {
- // Verify pushdown does NOT happen for sources without ordering
+fn test_inexact_pushdown_for_unordered_source() {
+ // Source has no declared `output_ordering`, request is `[a ASC]`.
+ // The reversed-equivalence check can't fire (nothing to reverse),
+ // but 'a' is in the file schema — sort pushdown returns `Inexact`
+ // with `sort_order_for_reorder` set so the opener can sort row
+ // groups by `min(a)` at scan time. The surrounding `SortExec`
+ // stays in place to enforce the full ordering.
let schema = schema();
let source = parquet_exec(schema.clone()); // No output_ordering
let sort_exprs = LexOrdering::new(vec![sort_expr("a", &schema)]).unwrap();
@@ -524,14 +534,20 @@ fn test_no_pushdown_for_unordered_source() {
output:
Ok:
- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]
"
);
}
#[test]
-fn test_no_pushdown_for_non_reverse_sort() {
- // Verify pushdown does NOT happen when sort doesn't reverse source ordering
+fn test_inexact_pushdown_when_request_doesnt_match_source_ordering() {
+ // The requested sort column ('b') doesn't match the source's natural
+ // ordering ('a' ASC). Neither natural nor reversed satisfies the
+ // request, but 'b' is in the file schema — so sort pushdown returns
+ // `Inexact` with `sort_order_for_reorder` set, drops the source's
+ // claimed `output_ordering` (the runtime row-group reorder
+ // invalidates it), and keeps the surrounding `SortExec` for
+ // correctness.
let schema = schema();
// Source sorted by 'a' ASC
@@ -554,7 +570,7 @@ fn test_no_pushdown_for_non_reverse_sort() {
output:
Ok:
- SortExec: expr=[b@1 ASC], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[b@1 ASC]
"
);
}
@@ -630,7 +646,7 @@ fn test_pushdown_through_blocking_node() {
- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
- AggregateExec: mode=Final, gby=[a@0 as a], aggr=[COUNT(b)], ordering_mode=Sorted
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -668,7 +684,7 @@ fn test_sort_pushdown_through_simple_projection() {
Ok:
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a, b@1 as b]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -703,7 +719,7 @@ fn test_sort_pushdown_through_projection_with_alias() {
Ok:
- SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as id, b@1 as value]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -792,7 +808,7 @@ fn test_sort_pushdown_projection_reordered_columns() {
Ok:
- SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -826,7 +842,7 @@ fn test_sort_pushdown_projection_with_limit() {
Ok:
- SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a, b@1 as b]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -860,7 +876,7 @@ fn test_sort_pushdown_through_projection() {
Ok:
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a, b@1 as b]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
@@ -895,7 +911,7 @@ fn test_sort_pushdown_projection_subset_of_columns() {
Ok:
- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]
- ProjectionExec: expr=[a@0 as a]
- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true
+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true
"
);
}
diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rs
index 295665055b..edbea39948 100644
--- a/datafusion/datasource-parquet/src/access_plan.rs
+++ b/datafusion/datasource-parquet/src/access_plan.rs
@@ -16,7 +16,12 @@
// under the License.
use crate::sort::reverse_row_selection;
+use arrow::datatypes::Schema;
use datafusion_common::{Result, assert_eq_or_internal_err};
+use datafusion_physical_expr::expressions::Column;
+use datafusion_physical_expr_common::sort_expr::LexOrdering;
+use log::debug;
+use parquet::arrow::arrow_reader::statistics::StatisticsConverter;
use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};
@@ -485,6 +490,127 @@ impl PreparedAccessPlan {
})
}
+ /// Reorder row groups by their min statistics for the given sort order.
+ ///
+ /// This helps TopK queries find optimal values first. Row groups are
+ /// always sorted by min values in ASC order — direction (DESC) is
+ /// handled separately by `reverse()` which is applied after reorder.
+ ///
+ /// Gracefully skips reordering when:
+ /// - There is a row_selection (too complex to remap)
+ /// - 0 or 1 row groups (nothing to reorder)
+ /// - Sort expression is not a simple column reference
+ /// - Statistics are unavailable
+ pub(crate) fn reorder_by_statistics(
+ mut self,
+ sort_order: &LexOrdering,
+ file_metadata: &ParquetMetaData,
+ arrow_schema: &Schema,
+ ) -> Result<Self> {
+ // Skip if row_selection present (too complex to remap)
+ if self.row_selection.is_some() {
+ debug!("Skipping RG reorder: row_selection present");
+ return Ok(self);
+ }
+
+ // Nothing to reorder
+ if self.row_group_indexes.len() <= 1 {
+ return Ok(self);
+ }
+
+ let first_sort_expr = sort_order.first();
+
+ // Extract column name from sort expression
+ let column: &Column = match first_sort_expr.expr.downcast_ref::<Column>() {
+ Some(col) => col,
+ None => {
+ debug!("Skipping RG reorder: sort expr is not a simple column");
+ return Ok(self);
+ }
+ };
+
+ // Expected graceful skip: the sort column lives outside the
+ // file schema (e.g. a partition column whose ordering came
+ // through `reversed_satisfies` rather than `column_in_file_schema`).
+ // Parquet has no per-RG stats for it. Bail out quietly — no
+ // `debug_assert!` because this is a normal pushdown shape.
+ if arrow_schema.field_with_name(column.name()).is_err() {
+ debug!(
+ "Skipping RG reorder: column `{}` not in file schema",
+ column.name()
+ );
+ return Ok(self);
+ }
+
+ // From here, any `StatisticsConverter` / stats read / sort
+ // failure is unexpected — the column exists in the file
+ // schema, so building the converter and pulling typed mins
+ // should succeed on any well-formed parquet file. Trip a
+ // `debug_assert!` so CI catches regressions, but stay graceful
+ // in release so a single odd file can't take down a scan.
+ let converter = match StatisticsConverter::try_new(
+ column.name(),
+ arrow_schema,
+ file_metadata.file_metadata().schema_descr(),
+ ) {
+ Ok(c) => c,
+ Err(e) => {
+ debug_assert!(
+ false,
+ "RG reorder: cannot create stats converter for `{}`: {e}",
+ column.name(),
+ );
+ return Ok(self);
+ }
+ };
+
+ // Always sort ASC by min values — direction is handled by reverse
+ let rg_metadata: Vec<&RowGroupMetaData> = self
+ .row_group_indexes
+ .iter()
+ .map(|&idx| file_metadata.row_group(idx))
+ .collect();
+
+ let stat_mins = match converter.row_group_mins(rg_metadata.iter().copied()) {
+ Ok(vals) => vals,
+ Err(e) => {
+ debug_assert!(
+ false,
+ "RG reorder: cannot get min values for `{}`: {e}",
+ column.name(),
+ );
+ return Ok(self);
+ }
+ };
+
+ let sort_options = arrow::compute::SortOptions {
+ descending: false,
+ nulls_first: first_sort_expr.options.nulls_first,
+ };
+ let sorted_indices =
+ match arrow::compute::sort_to_indices(&stat_mins, Some(sort_options), None) {
+ Ok(indices) => indices,
+ Err(e) => {
+ debug_assert!(
+ false,
+ "RG reorder: arrow sort_to_indices failed for `{}`: {e}",
+ column.name(),
+ );
+ return Ok(self);
+ }
+ };
+
+ // Apply the reordering
+ let original_indexes = self.row_group_indexes.clone();
+ self.row_group_indexes = sorted_indices
+ .values()
+ .iter()
+ .map(|&i| original_indexes[i as usize])
+ .collect();
+
+ Ok(self)
+ }
+
/// Reverse the access plan for reverse scanning
pub(crate) fn reverse(mut self, file_metadata: &ParquetMetaData) -> Result<Self> {
// Get the row group indexes before reversing
@@ -722,4 +848,182 @@ mod test {
.unwrap();
Arc::new(SchemaDescriptor::new(Arc::new(schema)))
}
+
+ // ----------------------------------------------------------------
+ // `reorder_by_statistics` tests
+ // ----------------------------------------------------------------
+
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_expr::Operator;
+ use datafusion_physical_expr::expressions::{BinaryExpr, lit};
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+ use parquet::file::metadata::FileMetaData;
+ use parquet::file::statistics::Statistics as ParquetStatistics;
+
+ /// Single-column int32 schema named "a".
+ fn int_schema_descr() -> SchemaDescPtr {
+ use parquet::basic::Type as PhysicalType;
+ use parquet::schema::types::Type as SchemaType;
+ let field = SchemaType::primitive_type_builder("a", PhysicalType::INT32)
+ .build()
+ .unwrap();
+ let schema = SchemaType::group_type_builder("schema")
+ .with_fields(vec![Arc::new(field)])
+ .build()
+ .unwrap();
+ Arc::new(SchemaDescriptor::new(Arc::new(schema)))
+ }
+
+ /// Build a `ParquetMetaData` with one row group per element of
+ /// `mins`. Each row group declares int32 statistics with
+ /// `min == max == mins[i]` so the reorder key is unambiguous.
+ fn parquet_metadata_with_int_mins(mins: &[i32]) -> ParquetMetaData {
+ let schema_descr = int_schema_descr();
+ let row_groups: Vec<RowGroupMetaData> = mins
+ .iter()
+ .map(|&m| {
+ let stats =
+ ParquetStatistics::int32(Some(m), Some(m), None, Some(0), false);
+ let column = ColumnChunkMetaData::builder(schema_descr.column(0))
+ .set_statistics(stats)
+ .set_num_values(100)
+ .build()
+ .unwrap();
+ RowGroupMetaData::builder(schema_descr.clone())
+ .set_num_rows(100)
+ .set_column_metadata(vec![column])
+ .build()
+ .unwrap()
+ })
+ .collect();
+ let file_metadata =
+ FileMetaData::new(0, 0, None, None, schema_descr.clone(), None);
+ ParquetMetaData::new(file_metadata, row_groups)
+ }
+
+ fn arrow_schema_a_int() -> Schema {
+ Schema::new(vec![Field::new("a", DataType::Int32, true)])
+ }
+
+ fn lex_ordering_a_asc() -> LexOrdering {
+ LexOrdering::new(vec![PhysicalSortExpr {
+ expr: Arc::new(Column::new("a", 0)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }])
+ .unwrap()
+ }
+
+ /// Happy path: three row groups with mins 50/10/100. After
+ /// `reorder_by_statistics` the indexes are ordered ASC by `min`,
+ /// i.e. RG 1 (min=10) first, then RG 0 (min=50), then RG 2
+ /// (min=100).
+ #[test]
+ fn reorder_by_statistics_sorts_row_groups_asc_by_min() {
+ let metadata = parquet_metadata_with_int_mins(&[50, 10, 100]);
+ let plan = PreparedAccessPlan::new(vec![0, 1, 2], None).unwrap();
+
+ let result = plan
+ .reorder_by_statistics(
+ &lex_ordering_a_asc(),
+ &metadata,
+ &arrow_schema_a_int(),
+ )
+ .unwrap();
+
+ assert_eq!(result.row_group_indexes, vec![1, 0, 2]);
+ }
+
+ /// A `row_selection` is "too complex to remap" through reorder,
+ /// so the function short-circuits and returns the input untouched.
+ #[test]
+ fn reorder_by_statistics_skips_when_row_selection_present() {
+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);
+ let selection = RowSelection::from(vec![RowSelector::select(100)]);
+ let plan = PreparedAccessPlan::new(vec![0, 1], Some(selection)).unwrap();
+
+ let result = plan
+ .reorder_by_statistics(
+ &lex_ordering_a_asc(),
+ &metadata,
+ &arrow_schema_a_int(),
+ )
+ .unwrap();
+
+ assert_eq!(result.row_group_indexes, vec![0, 1]);
+ }
+
+ /// One row group means nothing to reorder.
+ #[test]
+ fn reorder_by_statistics_skips_when_at_most_one_row_group() {
+ let metadata = parquet_metadata_with_int_mins(&[50]);
+ let plan = PreparedAccessPlan::new(vec![0], None).unwrap();
+
+ let result = plan
+ .reorder_by_statistics(
+ &lex_ordering_a_asc(),
+ &metadata,
+ &arrow_schema_a_int(),
+ )
+ .unwrap();
+
+ assert_eq!(result.row_group_indexes, vec![0]);
+ }
+
+ /// Non-`Column` sort expressions (e.g. `a + 1`,
+ /// `date_trunc(...)`) can't drive a stats lookup, so reorder is
+ /// skipped. The opener falls back to whatever order it received.
+ #[test]
+ fn reorder_by_statistics_skips_for_non_column_sort_expr() {
+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);
+ let plan = PreparedAccessPlan::new(vec![0, 1], None).unwrap();
+ let arrow_schema = arrow_schema_a_int();
+ let order = LexOrdering::new(vec![PhysicalSortExpr {
+ expr: Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Plus,
+ lit(1i32),
+ )),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }])
+ .unwrap();
+
+ let result = plan
+ .reorder_by_statistics(&order, &metadata, &arrow_schema)
+ .unwrap();
+
+ assert_eq!(result.row_group_indexes, vec![0, 1]);
+ }
+
+ /// When the sort column lives outside the file's arrow schema
+ /// (e.g. a partition column that reached this method through
+ /// `try_pushdown_sort`'s reversed-equivalence branch), reorder is
+ /// an expected graceful skip — no `debug_assert!` should fire.
+ #[test]
+ fn reorder_by_statistics_skips_when_column_not_in_arrow_schema() {
+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);
+ let plan = PreparedAccessPlan::new(vec![0, 1], None).unwrap();
+ // Arrow schema only has "a"; the sort references "b".
+ let arrow_schema = arrow_schema_a_int();
+ let order = LexOrdering::new(vec![PhysicalSortExpr {
+ expr: Arc::new(Column::new("b", 0)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }])
+ .unwrap();
+
+ let result = plan
+ .reorder_by_statistics(&order, &metadata, &arrow_schema)
+ .unwrap();
+
+ assert_eq!(result.row_group_indexes, vec![0, 1]);
+ }
}
diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs
index 7f9c51de8a..e41bc82eb7 100644
--- a/datafusion/datasource-parquet/src/opener.rs
+++ b/datafusion/datasource-parquet/src/opener.rs
@@ -51,6 +51,7 @@ use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
use datafusion_physical_expr_common::physical_expr::{
PhysicalExpr, is_dynamic_physical_expr,
};
+use datafusion_physical_expr_common::sort_expr::LexOrdering;
use datafusion_physical_plan::metrics::{
BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,
MetricCategory, PruningMetrics,
@@ -76,9 +77,7 @@ use parquet::arrow::parquet_column;
use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};
use parquet::basic::Type;
use parquet::bloom_filter::Sbbf;
-use parquet::file::metadata::{
- PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData,
-};
+use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader};
/// Stateless Parquet morselizer implementation.
///
@@ -139,6 +138,8 @@ pub(super) struct ParquetMorselizer {
pub max_predicate_cache_size: Option<usize>,
/// Whether to read row groups in reverse order
pub reverse_row_groups: bool,
+ /// Optional sort order used to reorder row groups by their min/max statistics.
+ pub sort_order_for_reorder: Option<LexOrdering>,
}
impl fmt::Debug for ParquetMorselizer {
@@ -289,6 +290,7 @@ struct PreparedParquetOpen {
predicate_creation_errors: Count,
max_predicate_cache_size: Option<usize>,
reverse_row_groups: bool,
+ sort_order_for_reorder: Option<LexOrdering>,
preserve_order: bool,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
@@ -658,6 +660,7 @@ impl ParquetMorselizer {
predicate_creation_errors,
max_predicate_cache_size: self.max_predicate_cache_size,
reverse_row_groups: self.reverse_row_groups,
+ sort_order_for_reorder: self.sort_order_for_reorder.clone(),
preserve_order: self.preserve_order,
#[cfg(feature = "parquet_encryption")]
file_decryption_properties: None,
@@ -1112,6 +1115,41 @@ impl RowGroupsPrunedParquetOpen {
);
}
+ // Prepare access plans, then apply row-group ordering tweaks per
+ // run. Two composable steps:
+ //
+ // 1. `reorder_by_statistics`: sort row groups by `min(col)` ASC.
+ // Fixes out-of-order row groups (e.g. append-heavy workloads).
+ // Skipped gracefully when statistics aren't available or the
+ // sort expression isn't a plain column.
+ //
+ // 2. `reverse`: flip the iteration order for DESC requests, applied
+ // AFTER any reorder so the reversed order is correct whether or
+ // not reorder changed anything. Also handles `row_selection`
+ // remapping.
+ //
+ // For sorted data: reorder is a no-op, reverse gives perfect DESC.
+ // For unsorted data: reorder fixes the order, reverse flips for DESC.
+ //
+ // Both inputs come from the sort-pushdown channel —
+ // `ParquetSource::try_pushdown_sort` sets `sort_order_for_reorder`
+ // and/or `reverse_row_groups`.
+ let prepare_access_plan =
+ |plan: ParquetAccessPlan| -> Result<PreparedAccessPlan> {
+ let mut prepared_plan = plan.prepare(rg_metadata)?;
+ if let Some(sort_order) = prepared.sort_order_for_reorder.as_ref() {
+ prepared_plan = prepared_plan.reorder_by_statistics(
+ sort_order,
+ file_metadata.as_ref(),
+ &prepared.physical_file_schema,
+ )?;
+ }
+ if prepared.reverse_row_groups {
+ prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;
+ }
+ Ok(prepared_plan)
+ };
+
let arrow_reader_metrics = ArrowReaderMetrics::enabled();
let read_plan = build_projection_read_plan(
prepared.projection.expr_iter(),
@@ -1146,12 +1184,7 @@ impl RowGroupsPrunedParquetOpen {
// Build a decoder per run.
let mut decoders = VecDeque::with_capacity(runs.len());
for run in runs {
- let prepared_access_plan = prepare_access_plan(
- run.access_plan,
- rg_metadata,
- file_metadata.as_ref(),
- prepared.reverse_row_groups,
- )?;
+ let prepared_access_plan = prepare_access_plan(run.access_plan)?;
let mut builder =
decoder_config.build(prepared_access_plan, reader_metadata.clone());
if run.needs_filter {
@@ -1291,19 +1324,6 @@ impl<'a> RowFilterGenerator<'a> {
}
}
-fn prepare_access_plan(
- plan: ParquetAccessPlan,
- rg_metadata: &[RowGroupMetaData],
- file_metadata: &ParquetMetaData,
- reverse_row_groups: bool,
-) -> Result<PreparedAccessPlan> {
- let mut prepared_access_plan = plan.prepare(rg_metadata)?;
- if reverse_row_groups {
- prepared_access_plan = prepared_access_plan.reverse(file_metadata)?;
- }
- Ok(prepared_access_plan)
-}
-
/// State shared while building [`ParquetPushDecoder`]s for one file scan.
///
/// A scan can be split into multiple decoder runs when row groups have
@@ -1971,6 +1991,7 @@ mod test {
encryption_factory: None,
max_predicate_cache_size: self.max_predicate_cache_size,
reverse_row_groups: self.reverse_row_groups,
+ sort_order_for_reorder: None,
}
}
}
diff --git a/datafusion/datasource-parquet/src/sort.rs b/datafusion/datasource-parquet/src/sort.rs
index db22363aa3..c1cf4e8b78 100644
--- a/datafusion/datasource-parquet/src/sort.rs
+++ b/datafusion/datasource-parquet/src/sort.rs
@@ -17,7 +17,11 @@
//! Sort-related utilities for Parquet scanning
-use datafusion_common::Result;
+use arrow::datatypes::Schema;
+use datafusion_common::{Result, ScalarValue};
+use datafusion_datasource::PartitionedFile;
+use datafusion_physical_expr::expressions::Column;
+use datafusion_physical_expr_common::sort_expr::LexOrdering;
use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
use parquet::file::metadata::ParquetMetaData;
use std::collections::HashMap;
@@ -118,6 +122,89 @@ pub fn reverse_row_selection(
Ok(RowSelection::from(reversed_selectors))
}
+/// Reorder a file list so the most "promising" files are read first,
+/// matching `PreparedAccessPlan::reorder_by_statistics` at the
+/// row-group level: key off the file's `min(col)`, and let the sort
+/// direction follow the request (ASC by `min` for ASC requests, DESC
+/// by `min` for DESC requests).
+///
+/// Keeping both layers consistent matters because they share the same
+/// convergence story for TopK's dynamic filter: file `i`'s `min` is a
+/// lower bound on every row group inside it, so the order chosen here
+/// is a natural prefix of the order `reorder_by_statistics` will
+/// produce within each file.
+///
+/// No-op when:
+/// * `sort_order` is `None` (sort pushdown didn't fire);
+/// * the leading sort expression is not a plain `Column`; or
+/// * the column is not in `table_schema`.
+///
+/// Files missing statistics sort to the end so present-stats files
+/// run first.
+pub(crate) fn reorder_files_by_min_statistics(
+ mut files: Vec<PartitionedFile>,
+ sort_order: Option<&LexOrdering>,
+ reverse_row_groups: bool,
+ table_schema: &Schema,
+) -> Vec<PartitionedFile> {
+ let Some((col_name, descending)) =
+ extract_topk_sort_info(sort_order, reverse_row_groups)
+ else {
+ return files;
+ };
+
+ let Ok(col_idx) = table_schema.index_of(&col_name) else {
+ return files;
+ };
+
+ files.sort_by(|a, b| {
+ let key_a = file_min_value(a, col_idx);
+ let key_b = file_min_value(b, col_idx);
+ match (key_a, key_b) {
+ (Some(va), Some(vb)) => {
+ let cmp = va.partial_cmp(&vb).unwrap_or(std::cmp::Ordering::Equal);
+ if descending { cmp.reverse() } else { cmp }
+ }
+ (Some(_), None) => std::cmp::Ordering::Less,
+ (None, Some(_)) => std::cmp::Ordering::Greater,
+ (None, None) => std::cmp::Ordering::Equal,
+ }
+ });
+
+ log::debug!(
+ "Reordered {} files by min({}) {} for TopK optimization",
+ files.len(),
+ col_name,
+ if descending { "DESC" } else { "ASC" }
+ );
+
+ files
+}
+
+/// Extract the `(column name, descending)` tuple used by file-level
+/// reordering. Returns `None` when the sort order isn't set or the
+/// leading sort expression isn't a plain `Column`.
+fn extract_topk_sort_info(
+ sort_order: Option<&LexOrdering>,
+ reverse_row_groups: bool,
+) -> Option<(String, bool)> {
+ let sort_order = sort_order?;
+ let first = sort_order.first();
+ let col = first.expr.downcast_ref::<Column>()?;
+ Some((col.name().to_string(), reverse_row_groups))
+}
+
+/// File's per-column `min` for the reorder key.
+fn file_min_value(file: &PartitionedFile, col_idx: usize) -> Option<ScalarValue> {
+ let stats = file.statistics.as_ref()?;
+ stats
+ .column_statistics
+ .get(col_idx)?
+ .min_value
+ .get_value()
+ .cloned()
+}
+
#[cfg(test)]
mod tests {
use crate::ParquetAccessPlan;
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 91e3d22749..cf1fcea005 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -55,7 +55,7 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
#[cfg(feature = "parquet_encryption")]
use datafusion_execution::parquet_encryption::EncryptionFactory;
-use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use itertools::Itertools;
use object_store::ObjectStore;
#[cfg(feature = "parquet_encryption")]
@@ -289,11 +289,13 @@ pub struct ParquetSource {
pub(crate) projection: ProjectionExprs,
#[cfg(feature = "parquet_encryption")]
pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,
- /// If true, read files in reverse order and reverse row groups within files.
- /// But it's not guaranteed that rows within row groups are in reverse order,
- /// so we still need to sort them after reading, so the reverse scan is inexact.
- /// Used to optimize ORDER BY ... DESC on sorted data.
+ /// If true, the opener flips row-group iteration order. Within-
+ /// row-group order is on-disk order, so the scan is `Inexact` and
+ /// a `SortExec` is kept in the plan.
reverse_row_groups: bool,
+ /// Sort order driving `PreparedAccessPlan::reorder_by_statistics`
+ /// in the opener.
+ sort_order_for_reorder: Option<LexOrdering>,
}
impl ParquetSource {
@@ -319,6 +321,7 @@ impl ParquetSource {
#[cfg(feature = "parquet_encryption")]
encryption_factory: None,
reverse_row_groups: false,
+ sort_order_for_reorder: None,
}
}
@@ -474,6 +477,7 @@ impl ParquetSource {
}
}
+ #[cfg(test)]
pub(crate) fn with_reverse_row_groups(mut self, reverse_row_groups: bool) -> Self {
self.reverse_row_groups = reverse_row_groups;
self
@@ -581,9 +585,22 @@ impl FileSource for ParquetSource {
encryption_factory: self.get_encryption_factory_with_config(),
max_predicate_cache_size: self.max_predicate_cache_size(),
reverse_row_groups: self.reverse_row_groups,
+ sort_order_for_reorder: self.sort_order_for_reorder.clone(),
}))
}
+ fn reorder_files(
+ &self,
+ files: Vec<datafusion_datasource::PartitionedFile>,
+ ) -> Vec<datafusion_datasource::PartitionedFile> {
+ crate::sort::reorder_files_by_min_statistics(
+ files,
+ self.sort_order_for_reorder.as_ref(),
+ self.reverse_row_groups,
+ self.table_schema.table_schema(),
+ )
+ }
+
fn table_schema(&self) -> &TableSchema {
&self.table_schema
}
@@ -629,7 +646,11 @@ impl FileSource for ParquetSource {
write!(f, "{predicate_string}")?;
- // Add reverse_scan info if enabled
+ // Inexact sort-pushdown markers: surface both flags so
+ // readers can see the optimization fired.
+ if let Some(sort_order) = &self.sort_order_for_reorder {
+ write!(f, ", sort_order_for_reorder=[{sort_order}]")?;
+ }
if self.reverse_row_groups {
write!(f, ", reverse_row_groups=true")?;
}
@@ -744,25 +765,41 @@ impl FileSource for ParquetSource {
/// Try to optimize the scan to produce data in the requested sort order.
///
- /// This method receives:
+ /// Inputs:
/// 1. The query's required ordering (`order` parameter)
- /// 2. The file's natural ordering (via `self.file_ordering`, set by FileScanConfig)
+ /// 2. The source's equivalence properties (`eq_properties`)
///
- /// With both pieces of information, ParquetSource can decide what optimizations to apply.
+ /// # Returns
+ /// - `Exact`: the source's natural ordering already satisfies the
+ /// request. The surrounding `SortExec` can be eliminated provided
+ /// files within each group are non-overlapping (verified by
+ /// `FileScanConfig`).
+ /// - `Inexact`: the source can approximate the request via two
+ /// composable runtime steps — stats-based row-group reorder
+ /// (skipped when the leading sort key isn't a plain `Column`
+ /// in the file schema) and row-group iteration reverse. A
+ /// `SortExec` is still required for full correctness, but limit
+ /// pushdown and `TopK` benefit immediately.
+ /// - `Unsupported`: no approximation is available.
///
- /// # Behavior
- /// - Returns `Exact` when the file's natural ordering (from Parquet metadata) already
- /// satisfies the requested ordering. This allows the Sort operator to be eliminated
- /// if the files within each group are also non-overlapping (checked by FileScanConfig).
- /// - Returns `Inexact` when reversing the row group scan order would help satisfy the
- /// requested ordering. We still need a Sort operator at a higher level because:
- /// - We only reverse row group read order, not rows within row groups
- /// - This provides approximate ordering that benefits limit pushdown
+ /// # How the Inexact result is communicated
///
- /// # Returns
- /// - `Exact`: The file's natural ordering satisfies the request (within-file ordering guaranteed)
- /// - `Inexact`: Created an optimized source (e.g., reversed scan) that approximates the order
- /// - `Unsupported`: Cannot optimize for this ordering
+ /// The result is carried through two fields on `ParquetSource`:
+ ///
+ /// - `sort_order_for_reorder`: set to the request's `LexOrdering`
+ /// whenever the pushdown fires, regardless of whether the
+ /// leading expression is a plain `Column`. The opener invokes
+ /// `PreparedAccessPlan::reorder_by_statistics`, which skips
+ /// when the expression can't be looked up in parquet metadata.
+ /// Exposing the field unconditionally keeps `EXPLAIN` honest
+ /// about what the source was asked to approximate.
+ /// - `reverse_row_groups`: drives the opener's iteration flip.
+ /// When stats reorder applies (column-in-schema), this is just
+ /// the request's direction — the reorder produces ASC-by-min,
+ /// so reverse iff the query asks for DESC. When stats reorder
+ /// doesn't apply but the reversed source ordering satisfies
+ /// the request (function-wrapped case), this is always `true`
+ /// because we're flipping the file's natural order.
fn try_pushdown_sort(
&self,
order: &[PhysicalSortExpr],
@@ -782,24 +819,68 @@ impl FileSource for ParquetSource {
});
}
- // Build new equivalence properties with the reversed ordering.
- // This allows us to check if the reversed ordering satisfies the request
- // by leveraging:
- // - Function monotonicity (e.g., extract_year_month preserves ordering)
- // - Constant columns (from filters)
- // - Other equivalence relationships
+ // If the source's declared ordering is a non-empty *proper* prefix
+ // of the request (e.g. source `[a DESC, b ASC]`, request
+ // `[a DESC, b ASC, c DESC]`), decline pushdown so the outer
+ // `SortExec`'s `sort_prefix` optimisation — prefix-aware early
+ // termination in `TopK` — can still fire. Firing the Inexact
+ // pipeline below would invalidate the source's `output_ordering`
+ // (the runtime row-group reorder is approximate, so we can't
+ // honour the declared ordering anymore), which is exactly what
+ // `EnforceSorting` needs to derive `sort_prefix`. On data that
+ // is already in prefix order the stats-based reorder is mostly
+ // a no-op anyway, so the trade-off is plainly bad.
+ for prefix_len in 1..order.len() {
+ let prefix = order[..prefix_len].to_vec();
+ if eq_properties.ordering_satisfy(prefix.iter().cloned())? {
+ return Ok(SortOrderPushdownResult::Unsupported);
+ }
+ }
+
+ // Inexact pushdown. Two independent signals; either is enough
+ // to produce an approximate ordering, and they compose when
+ // both apply:
//
- // Example flow:
- // 1. File ordering: [extract_year_month(ws) DESC, ws DESC]
- // 2. After reversal: [extract_year_month(ws) ASC, ws ASC]
- // 3. Requested: [ws ASC]
- // 4. Through extract_year_month's monotonicity property, the reversed
- // ordering satisfies [ws ASC] even though it has additional prefix
+ // 1. `column_in_file_schema`: the request's leading sort key is
+ // a plain `Column` present in the file schema. The opener
+ // can sort row groups by that column's `min` via parquet
+ // statistics. Drives `sort_order_for_reorder`'s actual use.
+ //
+ // 2. `reversed_satisfies`: the source's declared ordering,
+ // when reversed, satisfies the request. This is strictly
+ // more powerful than the column-in-schema check because it
+ // runs the request through `EquivalenceProperties`'s full
+ // reasoning machinery:
+ //
+ // - Function monotonicity: e.g. file declares
+ // `[extract_year_month(ws) DESC, ws DESC]`, request is
+ // `[ws ASC]`; the reversed ordering still satisfies the
+ // request via `extract_year_month`'s monotonicity even
+ // though parquet has no stats keyed by the function
+ // expression itself.
+ // - Constant columns from filters: equivalence classes can
+ // mark columns as constant under a predicate, allowing
+ // requested orderings on those columns to be trivially
+ // satisfied.
+ // - Other equivalence relationships (e.g. `a = b` transfers
+ // ordering between `a` and `b`).
+ //
+ // `reorder_by_statistics` can't substitute for any of the
+ // above because it can only look up min/max for a plain
+ // physical column.
+ //
+ // `sort_order_for_reorder` is set in both cases so EXPLAIN
+ // shows what the source was asked to approximate; the opener
+ // skips stats-based reorder when the leading expression isn't
+ // a plain `Column`.
+ //
+ // The reversal flips each `PhysicalSortExpr` (both descending
+ // and nulls_first) and rebuilds an `EquivalenceProperties` so
+ // the request can be tested against the reversed orderings
+ // via the same `ordering_satisfy` API.
let reversed_eq_properties = {
let mut new = eq_properties.clone();
new.clear_orderings();
-
- // Reverse each ordering in the equivalence properties
let reversed_orderings = eq_properties
.oeq_class()
.iter()
@@ -810,19 +891,44 @@ impl FileSource for ParquetSource {
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
-
new.add_orderings(reversed_orderings);
new
};
+ let reversed_satisfies =
+ reversed_eq_properties.ordering_satisfy(order.iter().cloned())?;
+ let sort_order = LexOrdering::new(order.iter().cloned());
+ let column_in_file_schema = sort_order.as_ref().is_some_and(|s| {
+ s.first()
+ .expr
+ .downcast_ref::<datafusion_physical_expr::expressions::Column>()
+ .is_some_and(|col| {
+ self.table_schema
+ .file_schema()
+ .field_with_name(col.name())
+ .is_ok()
+ })
+ });
- // Check if the reversed ordering satisfies the requested ordering
- if !reversed_eq_properties.ordering_satisfy(order.iter().cloned())? {
+ if !column_in_file_schema && !reversed_satisfies {
return Ok(SortOrderPushdownResult::Unsupported);
}
- // Return Inexact because we're only reversing row group order,
- // not guaranteeing perfect row-level ordering
- let new_source = self.clone().with_reverse_row_groups(true);
+ // `reverse_row_groups` has different starting points in the
+ // two cases:
+ // - With stats reorder (column-in-schema): the reorder produces
+ // ASC-by-min, so reverse iff the request is DESC.
+ // - Without stats reorder (reversed-eq fallback): we flip the
+ // file's natural order, so always reverse.
+ let is_descending = sort_order
+ .as_ref()
+ .is_some_and(|s| s.first().options.descending);
+ let mut new_source = self.clone();
+ new_source.sort_order_for_reorder = sort_order;
+ new_source.reverse_row_groups = if column_in_file_schema {
+ is_descending
+ } else {
+ true
+ };
Ok(SortOrderPushdownResult::Inexact {
inner: Arc::new(new_source) as Arc<dyn FileSource>,
})
@@ -947,4 +1053,525 @@ mod tests {
assert!(source.reverse_row_groups());
assert!(source.filter().is_some());
}
+
+ /// Helpers for the `try_pushdown_sort` regression tests below.
+ mod pushdown_sort_helpers {
+ use super::*;
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+
+ pub(super) fn schema_with_a_int() -> Arc<Schema> {
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]))
+ }
+
+ pub(super) fn sort_expr_on(
+ schema: &Arc<Schema>,
+ name: &str,
+ descending: bool,
+ ) -> PhysicalSortExpr {
+ let idx = schema.index_of(name).unwrap();
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new(name, idx)),
+ options: SortOptions {
+ descending,
+ nulls_first: true,
+ },
+ }
+ }
+ }
+
+ /// When neither natural nor reversed ordering matches the request,
+ /// but the sort column is a plain `Column` present in the file
+ /// schema, `try_pushdown_sort` returns `Inexact` with
+ /// `sort_order_for_reorder` set so the opener can reorder row
+ /// groups by min/max statistics at runtime.
+ #[test]
+ fn try_pushdown_sort_returns_inexact_when_column_in_schema_asc() {
+ use datafusion_physical_expr::EquivalenceProperties;
+ use pushdown_sort_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(Arc::clone(&schema));
+ let order = vec![sort_expr_on(&schema, "a", false)];
+ // No declared natural ordering on the source.
+ let eq = EquivalenceProperties::new(Arc::clone(&schema));
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+
+ let SortOrderPushdownResult::Inexact { inner } = result else {
+ panic!("expected Inexact, got a different variant");
+ };
+ // Downcast back to `ParquetSource` to inspect the fields the
+ // opener reads to drive `reorder_by_statistics` / `reverse`.
+ let inner_parquet = inner
+ .downcast_ref::<ParquetSource>()
+ .expect("inner is ParquetSource");
+ let sort_order = inner_parquet
+ .sort_order_for_reorder
+ .as_ref()
+ .expect("sort_order_for_reorder must be set so the opener can reorder");
+ assert_eq!(sort_order.first().expr.to_string(), "a@0");
+ // ASC request must not flip RG iteration order.
+ assert!(
+ !inner_parquet.reverse_row_groups(),
+ "ASC request must not set reverse_row_groups",
+ );
+ }
+
+ /// Same as above but for DESC. `reverse_row_groups` must also be
+ /// `true` so the opener flips iteration order.
+ #[test]
+ fn try_pushdown_sort_returns_inexact_when_column_in_schema_desc() {
+ use datafusion_physical_expr::EquivalenceProperties;
+ use pushdown_sort_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(Arc::clone(&schema));
+ let order = vec![sort_expr_on(&schema, "a", true)];
+ let eq = EquivalenceProperties::new(Arc::clone(&schema));
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+
+ let SortOrderPushdownResult::Inexact { inner } = result else {
+ panic!("expected Inexact, got a different variant");
+ };
+ let inner_parquet = inner
+ .downcast_ref::<ParquetSource>()
+ .expect("inner is ParquetSource");
+ assert!(inner_parquet.sort_order_for_reorder.is_some());
+ assert!(
+ inner_parquet.reverse_row_groups(),
+ "DESC request must set reverse_row_groups",
+ );
+ }
+
+ /// A non-`Column` leading sort expression (e.g. `a + 1`,
+ /// `date_trunc('hour', ts)`) with no declared source ordering
+ /// yields `Unsupported` — parquet stats need a column name to
+ /// look up min/max, and there's no source ordering to reverse.
+ #[test]
+ fn try_pushdown_sort_returns_unsupported_for_non_column_sort_expr() {
+ use arrow::compute::SortOptions;
+ use datafusion_physical_expr::EquivalenceProperties;
+ use datafusion_physical_expr::expressions::{BinaryExpr, Column, lit};
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+ use pushdown_sort_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(Arc::clone(&schema));
+
+ // `a + 1` — not a plain Column.
+ let order = vec![PhysicalSortExpr {
+ expr: Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ datafusion_expr::Operator::Plus,
+ lit(1i32),
+ )),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }];
+ let eq = EquivalenceProperties::new(Arc::clone(&schema));
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+ assert!(
+ matches!(result, SortOrderPushdownResult::Unsupported),
+ "non-Column sort expression must yield Unsupported",
+ );
+ }
+
+ /// A sort column missing from the file schema with no declared
+ /// source ordering yields `Unsupported` — there are no parquet
+ /// stats for that column and no source ordering to reverse.
+ #[test]
+ fn try_pushdown_sort_returns_unsupported_when_column_not_in_file_schema() {
+ use arrow::compute::SortOptions;
+ use datafusion_physical_expr::EquivalenceProperties;
+ use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+ use pushdown_sort_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(Arc::clone(&schema));
+
+ // Reference a column ("b") that does not exist in the file
+ // schema (which only has "a"). The Column expression itself is
+ // well-formed; only its name is unknown to the file.
+ let order = vec![PhysicalSortExpr {
+ expr: Arc::new(Column::new("b", 0)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: true,
+ },
+ }];
+ let eq = EquivalenceProperties::new(Arc::clone(&schema));
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+ assert!(
+ matches!(result, SortOrderPushdownResult::Unsupported),
+ "column not in file schema must yield Unsupported",
+ );
+ }
+
+ /// Regression: when the source declares `[a DESC]` and the request is
+ /// `[a ASC]`, both `column_in_file_schema` and `reversed_satisfies`
+ /// are true. `reverse_row_groups` must follow the *request's*
+ /// direction (false for ASC) — not the source's, and not the OR of
+ /// both signals. The opener's stats-based reorder produces
+ /// ASC-by-min, so an ASC request needs no further flip; flipping
+ /// would incorrectly emit DESC.
+ #[test]
+ fn try_pushdown_sort_source_desc_request_asc_does_not_reverse() {
+ use datafusion_physical_expr::EquivalenceProperties;
+ use pushdown_sort_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(Arc::clone(&schema));
+ // Source declares `[a DESC]`.
+ let mut eq = EquivalenceProperties::new(Arc::clone(&schema));
+ eq.add_ordering(vec![sort_expr_on(&schema, "a", true)]);
+ // Request `[a ASC]`.
+ let order = vec![sort_expr_on(&schema, "a", false)];
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+
+ let SortOrderPushdownResult::Inexact { inner } = result else {
+ panic!("expected Inexact, got a different variant");
+ };
+ let inner_parquet = inner
+ .downcast_ref::<ParquetSource>()
+ .expect("inner is ParquetSource");
+ assert!(
+ inner_parquet.sort_order_for_reorder.is_some(),
+ "sort_order_for_reorder must be set",
+ );
+ assert!(
+ !inner_parquet.reverse_row_groups(),
+ "ASC request on source-DESC must not set reverse_row_groups; \
+ a stale `reversed_satisfies || is_descending` formula would \
+ incorrectly flip iteration to DESC after the stats reorder",
+ );
+ }
+
+ /// A sort column that is *not* in the file schema (here: a partition
+ /// column `b`) but the source's *reversed* declared ordering does
+ /// satisfy the request. Pushdown fires via the reversed-equivalence
+ /// path; `sort_order_for_reorder` is still set (so EXPLAIN reflects
+ /// what the source was asked to approximate, even though the opener
+ /// will skip its stats reorder because `b` has no per-RG min/max in
+ /// the parquet file), and `reverse_row_groups` is `true` because we
+ /// flip the file's natural order rather than re-sort by stats.
+ #[test]
+ fn try_pushdown_sort_returns_inexact_via_reversed_eq_when_column_not_in_file_schema()
+ {
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_datasource::TableSchema;
+ use datafusion_physical_expr::EquivalenceProperties;
+ use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+
+ // File schema is just `[a]`; `b` lives as a partition column on
+ // top, so it appears in the table schema but not the file schema
+ // — the same shape `column_in_file_schema` discards.
+ let file_schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+ let partition_b = Arc::new(Field::new("b", DataType::Int32, true));
+ let table_schema = TableSchema::new(file_schema, vec![partition_b]);
+ let source = ParquetSource::new(table_schema);
+
+ // EquivalenceProperties is built on the *full* table schema so
+ // it can carry an ordering on `b`.
+ let full_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ ]));
+ // Construct the request first, then derive the declared
+ // ordering as its reverse, so `ordering_satisfy` on the
+ // reversed-eq holds exactly. `PhysicalSortExpr::reverse` flips
+ // both `descending` and `nulls_first`, so spelling the
+ // declared ordering directly is error-prone.
+ let request_expr = PhysicalSortExpr {
+ expr: Arc::new(Column::new("b", 1)),
+ options: SortOptions {
+ descending: true,
+ nulls_first: true,
+ },
+ };
+ let declared = request_expr.reverse();
+ let mut eq = EquivalenceProperties::new(Arc::clone(&full_schema));
+ eq.add_ordering(vec![declared]);
+ let order = vec![request_expr];
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+
+ let SortOrderPushdownResult::Inexact { inner } = result else {
+ panic!("expected Inexact, got a different variant");
+ };
+ let inner_parquet = inner
+ .downcast_ref::<ParquetSource>()
+ .expect("inner is ParquetSource");
+ assert!(
+ inner_parquet.sort_order_for_reorder.is_some(),
+ "sort_order_for_reorder must be set so EXPLAIN reflects the request",
+ );
+ assert!(
+ inner_parquet.reverse_row_groups(),
+ "request reached via reversed_satisfies (column-not-in-file-schema) \
+ must set reverse_row_groups to flip the file's natural order",
+ );
+ }
+
+ /// Regression: when the source's declared ordering is a non-empty
+ /// *proper* prefix of the request, `try_pushdown_sort` must return
+ /// `Unsupported` so that the outer `SortExec` can keep its
+ /// `sort_prefix` annotation and `TopK` can early-terminate within
+ /// each prefix block. Letting the Phase 3 Inexact pipeline fire
+ /// would drop the source's `output_ordering`, destroying the
+ /// information `EnforceSorting` needs to compute `sort_prefix`.
+ #[test]
+ fn try_pushdown_sort_preserves_sort_prefix_when_source_declares_prefix_ordering() {
+ use arrow::compute::SortOptions;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_physical_expr::EquivalenceProperties;
+ use datafusion_physical_expr::expressions::Column;
+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, true),
+ Field::new("c", DataType::Int32, true),
+ ]));
+ let source = ParquetSource::new(Arc::clone(&schema));
+
+ // Source declares `[a DESC, b ASC NULLS LAST]` — the same prefix
+ // the SortExec input will see.
+ let mut eq = EquivalenceProperties::new(Arc::clone(&schema));
+ eq.add_ordering(vec![
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new("a", 0)),
+ options: SortOptions {
+ descending: true,
+ nulls_first: true,
+ },
+ },
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new("b", 1)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ },
+ ]);
+
+ // Request `[a DESC, b ASC NULLS LAST, c DESC]` — three columns;
+ // source's two-column declaration is a strict prefix.
+ let order = vec![
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new("a", 0)),
+ options: SortOptions {
+ descending: true,
+ nulls_first: true,
+ },
+ },
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new("b", 1)),
+ options: SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ },
+ PhysicalSortExpr {
+ expr: Arc::new(Column::new("c", 2)),
+ options: SortOptions {
+ descending: true,
+ nulls_first: true,
+ },
+ },
+ ];
+
+ let result = source.try_pushdown_sort(&order, &eq).unwrap();
+ assert!(
+ matches!(result, SortOrderPushdownResult::Unsupported),
+ "source ordering [a DESC, b ASC NULLS LAST] is a proper prefix \
+ of the request — `try_pushdown_sort` must return Unsupported so \
+ the SortExec sort_prefix optimisation can fire",
+ );
+ }
+
+ /// Helpers for the `reorder_files` regression tests below.
+ mod reorder_files_helpers {
+ use super::*;
+ use datafusion_common::stats::Precision;
+ use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};
+ use datafusion_datasource::PartitionedFile;
+
+ pub(super) fn file_with_min(name: &str, min: Option<i32>) -> PartitionedFile {
+ let mut pf = PartitionedFile::new(name.to_string(), 0);
+ let min_value = min
+ .map(|v| Precision::Exact(ScalarValue::Int32(Some(v))))
+ .unwrap_or(Precision::Absent);
+ pf.statistics = Some(Arc::new(Statistics {
+ num_rows: Precision::Absent,
+ total_byte_size: Precision::Absent,
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Absent,
+ max_value: Precision::Absent,
+ min_value,
+ sum_value: Precision::Absent,
+ distinct_count: Precision::Absent,
+ byte_size: Precision::Absent,
+ }],
+ }));
+ pf
+ }
+
+ pub(super) fn names(files: &[PartitionedFile]) -> Vec<&str> {
+ files
+ .iter()
+ .map(|f| f.object_meta.location.as_ref())
+ .collect()
+ }
+ }
+
+ /// ASC TopK: `reorder_files` keys off file `min` and sorts ASC,
+ /// so the file with the smallest `min` is read first. This
+ /// matches `PreparedAccessPlan::reorder_by_statistics` at the
+ /// row-group level (also `min ASC`).
+ #[test]
+ fn reorder_files_sorts_asc_by_min_for_asc_request() {
+ use pushdown_sort_helpers::*;
+ use reorder_files_helpers::*;
+
+ let schema = schema_with_a_int();
+ let mut source = ParquetSource::new(Arc::clone(&schema));
+ source.sort_order_for_reorder =
+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap());
+ // ASC request → `reverse_row_groups` left at its default `false`.
+
+ let reordered = source.reorder_files(vec![
+ file_with_min("middle", Some(50)),
+ file_with_min("small", Some(10)),
+ file_with_min("large", Some(100)),
+ ]);
+
+ assert_eq!(names(&reordered), vec!["small", "middle", "large"]);
+ }
+
+ /// DESC TopK: same `min` key, but sorted DESC — file with the
+ /// largest `min` first. Mirrors the row-group strategy of
+ /// "ASC-by-min then `reverse`".
+ #[test]
+ fn reorder_files_sorts_desc_by_min_for_desc_request() {
+ use pushdown_sort_helpers::*;
+ use reorder_files_helpers::*;
+
+ let schema = schema_with_a_int();
+ let mut source =
+ ParquetSource::new(Arc::clone(&schema)).with_reverse_row_groups(true);
+ source.sort_order_for_reorder =
+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", true)]).unwrap());
+
+ let reordered = source.reorder_files(vec![
+ file_with_min("middle", Some(50)),
+ file_with_min("small", Some(10)),
+ file_with_min("large", Some(100)),
+ ]);
+
+ assert_eq!(names(&reordered), vec!["large", "middle", "small"]);
+ }
+
+ /// Files without statistics sort to the *end* so present-stats
+ /// files run first regardless of direction. Verified for both
+ /// ASC and DESC.
+ #[test]
+ fn reorder_files_pushes_missing_stats_to_the_end() {
+ use pushdown_sort_helpers::*;
+ use reorder_files_helpers::*;
+
+ let schema = schema_with_a_int();
+ let mut source = ParquetSource::new(Arc::clone(&schema));
+ source.sort_order_for_reorder =
+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap());
+
+ let reordered = source.reorder_files(vec![
+ file_with_min("no_stats", None),
+ file_with_min("has_min", Some(10)),
+ ]);
+ assert_eq!(names(&reordered), vec!["has_min", "no_stats"]);
+
+ // Same for DESC.
+ let mut source =
+ ParquetSource::new(Arc::clone(&schema)).with_reverse_row_groups(true);
+ source.sort_order_for_reorder =
+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", true)]).unwrap());
+ let reordered = source.reorder_files(vec![
+ file_with_min("no_stats", None),
+ file_with_min("has_min", Some(10)),
+ ]);
+ assert_eq!(names(&reordered), vec!["has_min", "no_stats"]);
+ }
+
+ /// When no sort pushdown has fired (`sort_order_for_reorder` is
+ /// `None`), `reorder_files` is a no-op and preserves input order.
+ #[test]
+ fn reorder_files_is_a_no_op_without_pushdown() {
+ use pushdown_sort_helpers::*;
+ use reorder_files_helpers::*;
+
+ let schema = schema_with_a_int();
+ let source = ParquetSource::new(schema);
+ // No `sort_order_for_reorder` set on the source.
+
+ let input = vec![
+ file_with_min("c", Some(30)),
+ file_with_min("a", Some(10)),
+ file_with_min("b", Some(20)),
+ ];
+ let reordered = source.reorder_files(input.clone());
+ assert_eq!(names(&reordered), names(&input));
+ }
+
+ /// `sort_order_for_reorder` is surfaced in both `EXPLAIN` (Default)
+ /// and `EXPLAIN VERBOSE` / `EXPLAIN ANALYZE` (Verbose) so readers
+ /// and snapshot tests can see the inexact sort-pushdown fired.
+ #[test]
+ fn sort_order_for_reorder_shown_in_explain() {
+ use pushdown_sort_helpers::*;
+
+ // `std::fmt::Formatter` can't be constructed outside core fmt
+ // machinery, so we drive `fmt_extra` through a Display adapter
+ // and read the rendered string back with `format!`.
+ struct DisplayHelper<'a> {
+ source: &'a ParquetSource,
+ mode: DisplayFormatType,
+ }
+ impl std::fmt::Display for DisplayHelper<'_> {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ self.source.fmt_extra(self.mode, f)
+ }
+ }
+
+ let schema = schema_with_a_int();
+ let mut source = ParquetSource::new(Arc::clone(&schema));
+ let order = LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap();
+ source.sort_order_for_reorder = Some(order);
+
+ for mode in [DisplayFormatType::Default, DisplayFormatType::Verbose] {
+ let out = format!(
+ "{}",
+ DisplayHelper {
+ source: &source,
+ mode,
+ },
+ );
+ assert!(
+ out.contains("sort_order_for_reorder=[a@0 ASC]"),
+ "{mode:?} display must surface sort_order_for_reorder, got: {out}",
+ );
+ }
+ }
}
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs
index 9b4ae5827a..32bee63b54 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -280,6 +280,19 @@ pub trait FileSource: Any + Send + Sync {
Ok(SortOrderPushdownResult::Unsupported)
}
+ /// Reorder files in the shared work queue to optimize query performance.
+ ///
+ /// For example, TopK queries benefit from reading files with the best
+ /// statistics first, so the dynamic filter threshold tightens quickly.
+ ///
+ /// The default implementation returns files unchanged (no reordering).
+ fn reorder_files(
+ &self,
+ files: Vec<crate::PartitionedFile>,
+ ) -> Vec<crate::PartitionedFile> {
+ files
+ }
+
/// Try to push down a projection into this FileSource.
///
/// `FileSource` implementations that support projection pushdown should
diff --git a/datafusion/datasource/src/file_stream/work_source.rs b/datafusion/datasource/src/file_stream/work_source.rs
index 7f31dacca9..c00048453b 100644
--- a/datafusion/datasource/src/file_stream/work_source.rs
+++ b/datafusion/datasource/src/file_stream/work_source.rs
@@ -85,8 +85,19 @@ impl SharedWorkSource {
}
/// Create a shared work source for the unopened files in `config`.
+ ///
+ /// Files are reordered by the file source (e.g. by statistics for TopK)
+ /// before being placed in the shared queue, so the most promising files
+ /// are processed first across all partitions.
pub(crate) fn from_config(config: &FileScanConfig) -> Self {
- Self::new(config.file_groups.iter().flat_map(FileGroup::iter).cloned())
+ let files: Vec<_> = config
+ .file_groups
+ .iter()
+ .flat_map(FileGroup::iter)
+ .cloned()
+ .collect();
+ let files = config.file_source.reorder_files(files);
+ Self::new(files)
}
/// Pop the next file from the shared work queue.
diff --git a/datafusion/sqllogictest/test_files/create_external_table.slt b/datafusion/sqllogictest/test_files/create_external_table.slt
index 0b15a7f8ec..f56cff2a2a 100644
--- a/datafusion/sqllogictest/test_files/create_external_table.slt
+++ b/datafusion/sqllogictest/test_files/create_external_table.slt
@@ -264,7 +264,7 @@ logical_plan
02)--TableScan: t projection=[id]
physical_plan
01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], file_type=parquet, reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
statement ok
DROP TABLE t;
diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
index befc443303..e779ce2cbf 100644
--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt
@@ -90,7 +90,7 @@ logical_plan
02)--TableScan: test_parquet projection=[id, value, name]
physical_plan
01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[value@1 DESC], reverse_row_groups=true
statement ok
set datafusion.explain.analyze_level = summary;
@@ -128,7 +128,7 @@ logical_plan
02)--TableScan: test_parquet projection=[id, value, name]
physical_plan
01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, sort_order_for_reorder=[value@1 DESC], reverse_row_groups=true
query IIT
SELECT id, value AS v, name FROM (SELECT * FROM test_parquet UNION ALL SELECT * FROM test_parquet) ORDER BY v DESC LIMIT 3;
@@ -338,7 +338,7 @@ physical_plan
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet
03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]
04)----FilterExec: DynamicFilter [ empty ]
-05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]
+05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[data@1 DESC], reverse_row_groups=true
statement count 0
SET datafusion.execution.parquet.pushdown_filters = true;
@@ -361,7 +361,7 @@ physical_plan
01)HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(id@0, id@0)]
02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet
03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]
-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]
+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[data@1 DESC], reverse_row_groups=true
statement count 0
RESET datafusion.execution.parquet.pushdown_filters;
diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.slt
index cad36d4fa6..7460148bab 100644
--- a/datafusion/sqllogictest/test_files/explain_analyze.slt
+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt
@@ -231,7 +231,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[]
statement ok
reset datafusion.explain.analyze_categories;
@@ -247,7 +247,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]
statement ok
reset datafusion.explain.analyze_categories;
@@ -262,7 +262,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
statement ok
reset datafusion.explain.analyze_categories;
@@ -277,7 +277,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]
statement ok
reset datafusion.explain.analyze_categories;
@@ -292,7 +292,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[elapsed_compute=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[elapsed_compute=<slt:ignore>, metadata_load_time=<slt:ignore>]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[elapsed_compute=<slt:ignore>, metadata_load_time=<slt:ignore>]
statement ok
reset datafusion.explain.analyze_categories;
diff --git a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt b/datafusion/sqllogictest/test_files/join_limit_pushdown.slt
index ea9c971065..933b03e7eb 100644
--- a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/join_limit_pushdown.slt
@@ -259,7 +259,7 @@ SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 100;
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt
index 8e931ca52f..fc62584dc3 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -868,7 +868,7 @@ physical_plan
01)ProjectionExec: expr=[1 as foo]
02)--SortPreservingMergeExec: [part_key@0 ASC NULLS LAST], fetch=1
03)----SortExec: TopK(fetch=1), expr=[part_key@0 ASC NULLS LAST], preserve_partitioning=[true]
-04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-2.parquet]]}, projection=[part_key], file_type=parquet, predicate=DynamicFilter [ empty ]
+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-2.parquet]]}, projection=[part_key], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[part_key@0 ASC NULLS LAST]
query I
with selection as (
diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.slt
index 676306e1f0..373e1636a2 100644
--- a/datafusion/sqllogictest/test_files/limit_pruning.slt
+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt
@@ -120,7 +120,7 @@ explain analyze select * from tracking_data where species > 'M' AND s >= 50 orde
----
Plan with Metrics
01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]
statement ok
drop table tracking_data;
diff --git a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
index 6b5368620c..441adce9a3 100644
--- a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
+++ b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
@@ -22,7 +22,7 @@ SELECT COUNT(*) FROM (SELECT i FROM filter_limit WHERE i <> 0 LIMIT 1);
statement ok
reset datafusion.execution.batch_size;
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.slt
index 781d0b00a5..8026e98d86 100644
--- a/datafusion/sqllogictest/test_files/parquet.slt
+++ b/datafusion/sqllogictest/test_files/parquet.slt
@@ -95,7 +95,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST]
02)--SortExec: expr=[string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/1.parquet]]}, projection=[int_col, string_col], file_type=parquet
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/1.parquet]]}, projection=[int_col, string_col], file_type=parquet, sort_order_for_reorder=[string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST]
# Tear down test_table:
statement ok
@@ -889,7 +889,7 @@ WHERE b = 2;
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
index 85f9549357..cb3be93191 100644
--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt
@@ -110,7 +110,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
query T
select a from t where b = 2 ORDER BY b;
@@ -181,7 +181,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
query TT
EXPLAIN select a from t_pushdown where b > 2 ORDER BY a;
@@ -194,7 +194,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
query T
select a from t where b = 2 ORDER BY b;
@@ -278,7 +278,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]
query T
select a from t where b = 2 ORDER BY b;
@@ -377,7 +377,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]
02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]
query I
@@ -396,7 +396,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]
02)--SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]
# should not push down volatile predicates such as RANDOM
@@ -603,7 +603,7 @@ logical_plan
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust), sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test array_has_all predicate pushdown
query I?
@@ -620,7 +620,7 @@ logical_plan
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance]), sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test array_has_any predicate pushdown
query I?
@@ -637,7 +637,7 @@ logical_plan
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go]), sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test complex predicate with OR
query I?
@@ -656,7 +656,7 @@ logical_plan
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go]), sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test array function with other predicates
query I?
@@ -673,7 +673,7 @@ logical_plan
03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), sort_order_for_reorder=[id@0 ASC NULLS LAST], pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]
###
# Test filter pushdown through UNION with mixed support
diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
index a4a613e383..b1e76a555d 100644
--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
@@ -278,7 +278,7 @@ physical_plan
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
set datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.slt
index a40c1b8c7e..344aef1f92 100644
--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt
@@ -320,7 +320,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -345,7 +345,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -413,7 +413,7 @@ logical_plan
03)----TableScan: three_cols projection=[col_a, col_b, col_c]
physical_plan
01)SortExec: expr=[col_a@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c, col_b@1 as col_b_dup], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c, col_b@1 as col_b_dup], file_type=parquet, sort_order_for_reorder=[col_a@0 ASC NULLS LAST]
# Verify correctness
query IIII
@@ -444,7 +444,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -467,7 +467,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -490,7 +490,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query IIT
@@ -513,7 +513,7 @@ logical_plan
03)----TableScan: nested_struct projection=[id, nested]
physical_plan
01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/nested.parquet]]}, projection=[id, get_field(nested@1, outer, inner) as nested_struct.nested[outer][inner]], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/nested.parquet]]}, projection=[id, get_field(nested@1, outer, inner) as nested_struct.nested[outer][inner]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -535,7 +535,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, label) || _suffix as simple_struct.s[label] || Utf8("_suffix")], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, label) || _suffix as simple_struct.s[label] || Utf8("_suffix")], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query IT
@@ -637,7 +637,7 @@ SELECT id, s['value'] + 1 FROM simple_struct WHERE id > 1 ORDER BY id LIMIT 2;
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
SET datafusion.execution.target_partitions = 4;
@@ -687,7 +687,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]
02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet
+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -713,7 +713,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3
02)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ]
+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -737,7 +737,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3
02)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) + 1 as multi_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ]
+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) + 1 as multi_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -874,7 +874,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, value) + 10 as simple_struct.s[value] + Int64(10), get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, value) + 10 as simple_struct.s[value] + Int64(10), get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query IIIT
@@ -897,7 +897,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as constant], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as constant], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -919,7 +919,7 @@ logical_plan
02)--TableScan: simple_struct projection=[id]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query I
@@ -947,7 +947,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, id@0 + 100 as computed], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, id@0 + 100 as computed], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -1039,7 +1039,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + id@0 as combined], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + id@0 as combined], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -1095,7 +1095,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as answer, get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as answer, get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query IIT
@@ -1118,7 +1118,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 100 as simple_struct.s[value] + Int64(100), get_field(s@1, label) || _test as simple_struct.s[label] || Utf8("_test")], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 100 as simple_struct.s[value] + Int64(100), get_field(s@1, label) || _test as simple_struct.s[label] || Utf8("_test")], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query IIT
@@ -1288,7 +1288,7 @@ logical_plan
physical_plan
01)ProjectionExec: expr=[id@0 as id, simple_struct.s[value]@1 as simple_struct.s[value]]
02)--SortExec: expr=[id@0 ASC NULLS LAST, __datafusion_extracted_1@2 ASC NULLS LAST], preserve_partitioning=[false]
-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as __datafusion_extracted_1], file_type=parquet
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as __datafusion_extracted_1], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST, __datafusion_extracted_1@2 ASC NULLS LAST]
# Verify correctness
query II
@@ -1369,7 +1369,7 @@ logical_plan
03)----TableScan: simple_struct projection=[id, s]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST, simple_struct.s[value]@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST, simple_struct.s[value]@1 ASC NULLS LAST]
# Verify correctness
query II
diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
index 8469c32a17..c2fae88903 100644
--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt
@@ -257,7 +257,7 @@ EXPLAIN SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;
----
physical_plan
01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true
statement ok
set datafusion.explain.analyze_categories = 'rows';
@@ -268,7 +268,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;
----
Plan with Metrics
01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false], filter=[b@1 IS NULL OR b@1 > bd], metrics=[output_rows=1, output_batches=1, row_replacements=1]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=22.37% (240/1.07 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true, pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=22.37% (240/1.07 K)]
statement ok
reset datafusion.explain.analyze_categories;
@@ -319,7 +319,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_multi_col ORDER BY b ASC NULLS LAST, a DESC L
----
Plan with Metrics
01)SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false], filter=[b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac)], metrics=[output_rows=2, output_batches=1, row_replacements=2]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=22.37% (240/1.07 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 DESC], pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=22.37% (240/1.07 K)]
statement ok
reset datafusion.explain.analyze_categories;
@@ -655,7 +655,7 @@ EXPLAIN ANALYZE SELECT b, a FROM topk_proj ORDER BY a LIMIT 2;
----
Plan with Metrics
01)SortExec: TopK(fetch=2), expr=[a@1 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@1 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@1 ASC NULLS LAST], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]
# Case 2: prune — `SELECT a` — filter stays as `a < 2` on the scan.
query TT
@@ -663,7 +663,7 @@ EXPLAIN ANALYZE SELECT a FROM topk_proj ORDER BY a LIMIT 2;
----
Plan with Metrics
01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=7.09% (79/1.11 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=7.09% (79/1.11 K)]
# Case 3: expression — `SELECT a+1 AS a_plus_1` — the TopK filter is on
# `a_plus_1`, the scan predicate must read `a@0 + 1`.
@@ -682,7 +682,7 @@ EXPLAIN ANALYZE SELECT a + 1 AS a, b FROM topk_proj ORDER BY a LIMIT 2;
----
Plan with Metrics
01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]
statement ok
reset datafusion.explain.analyze_categories;
diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.slt
index b6c75f3977..540562eb3b 100644
--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt
@@ -43,7 +43,7 @@ logical_plan
02)--TableScan: sorted_parquet projection=[id, value, name]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
# Test 1.2: Verify results are correct
query IIT
@@ -91,7 +91,7 @@ logical_plan
physical_plan
01)GlobalLimitExec: skip=2, fetch=3
02)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
query IIT
SELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;
@@ -155,7 +155,7 @@ logical_plan
03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]
physical_plan
01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]
# Verify the results are correct despite reverse scanning with row selection
# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id
@@ -272,7 +272,7 @@ logical_plan
physical_plan
01)SortPreservingMergeExec: [id@0 DESC], fetch=3
02)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]
-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
# Verify correctness with repartitioning and multiple files
query IIT
@@ -381,7 +381,7 @@ logical_plan
03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]
physical_plan
01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]
# Test 2.2: Verify the results are correct
query TIR
@@ -491,7 +491,7 @@ logical_plan
03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")]
physical_plan
01)SortExec: TopK(fetch=3), expr=[period_end@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]
# Test 2.9: Complex case - literal constant in sort expression itself
# The literal 'constant' is ignored in sort analysis
@@ -511,7 +511,7 @@ logical_plan
03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("monthly")]
physical_plan
01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]
# Verify results
query TIR
@@ -540,7 +540,7 @@ logical_plan
03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.period_end = Int32(2)]
physical_plan
01)SortExec: expr=[timeframe@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, sort_order_for_reorder=[timeframe@0 DESC], reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]
# Cleanup
statement ok
@@ -600,7 +600,7 @@ logical_plan
02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]
physical_plan
01)SortExec: TopK(fetch=3), expr=[ts@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ts@1 DESC], reverse_row_groups=true
# Verify results
query IPIR
@@ -626,7 +626,7 @@ logical_plan
02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]
physical_plan
01)SortExec: TopK(fetch=3), expr=[date_trunc(day, ts@1) DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[date_trunc(day, ts@1) DESC], reverse_row_groups=true
# Verify results (descending day)
query IPIR
@@ -686,7 +686,7 @@ logical_plan
02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]
physical_plan
01)SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ts@1 DESC], reverse_row_groups=true
query IPIR
SELECT * FROM multi_month_parquet
@@ -712,7 +712,7 @@ logical_plan
02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]
physical_plan
01)SortExec: TopK(fetch=2), expr=[date_trunc(month, ts@1) DESC, ts@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[date_trunc(month, ts@1) DESC, ts@1 DESC], reverse_row_groups=true
query IPIR
SELECT * FROM multi_month_parquet
@@ -754,7 +754,7 @@ logical_plan
02)--TableScan: int_parquet projection=[id, small_val, big_val]
physical_plan
01)SortExec: TopK(fetch=2), expr=[CAST(small_val@1 AS Int64) DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[CAST(small_val@1 AS Int64) DESC], reverse_row_groups=true
query III
SELECT * FROM int_parquet
@@ -796,7 +796,7 @@ logical_plan
02)--TableScan: float_parquet projection=[id, value]
physical_plan
01)SortExec: TopK(fetch=3), expr=[ceil(value@1) DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ceil(value@1) DESC], reverse_row_groups=true
query IR
SELECT * FROM float_parquet
@@ -1109,7 +1109,9 @@ EXPLAIN SELECT * FROM reversed_parquet ORDER BY id ASC;
logical_plan
01)Sort: reversed_parquet.id ASC NULLS LAST
02)--TableScan: reversed_parquet projection=[id, value]
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test 4.2: Results must be correct
query II
@@ -1160,7 +1162,7 @@ logical_plan
02)--TableScan: overlap_parquet projection=[id, value]
physical_plan
01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_y.parquet]]}, projection=[id, value], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_y.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test 5.2: Results must be correct
query II
@@ -1192,7 +1194,9 @@ EXPLAIN SELECT * FROM reversed_with_order_parquet ORDER BY id ASC;
logical_plan
01)Sort: reversed_with_order_parquet.id ASC NULLS LAST
02)--TableScan: reversed_with_order_parquet projection=[id, value]
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
+physical_plan
+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Test 6.2: Results must be correct
query II
@@ -1278,7 +1282,7 @@ logical_plan
02)--TableScan: correct_parquet projection=[id, value]
physical_plan
01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet]]}, projection=[id, value], file_type=parquet, reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
query II
SELECT * FROM correct_parquet ORDER BY id DESC;
@@ -1329,7 +1333,9 @@ EXPLAIN SELECT * FROM desc_reversed_parquet ORDER BY id DESC;
logical_plan
01)Sort: desc_reversed_parquet.id DESC NULLS FIRST
02)--TableScan: desc_reversed_parquet projection=[id, value]
-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet]]}, projection=[id, value], output_ordering=[id@0 DESC], file_type=parquet
+physical_plan
+01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
# Test 8.2: Results must be correct
query II
@@ -1384,7 +1390,7 @@ logical_plan
02)--TableScan: multi_col_parquet projection=[category, id, value]
physical_plan
01)SortExec: expr=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/a_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/b_second.parquet]]}, projection=[category, id, value], file_type=parquet
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/a_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/b_second.parquet]]}, projection=[category, id, value], file_type=parquet, sort_order_for_reorder=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST]
# Test 9.2: Results must be correct
query TII
@@ -1519,7 +1525,7 @@ SELECT * FROM multi_partition_parquet ORDER BY id ASC;
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
SET datafusion.execution.target_partitions = 4;
@@ -1772,7 +1778,7 @@ logical_plan
02)--TableScan: tb_overlap projection=[id, value]
physical_plan
01)SortExec: TopK(fetch=5), expr=[id@0 DESC, value@1 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC, value@1 DESC], reverse_row_groups=true
query II
SELECT * FROM tb_overlap ORDER BY id DESC, value DESC LIMIT 5;
@@ -1857,7 +1863,7 @@ logical_plan
02)--TableScan: tc_limit projection=[id, value]
physical_plan
01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
query II
SELECT * FROM tc_limit ORDER BY id DESC LIMIT 3;
@@ -2221,8 +2227,8 @@ logical_plan
02)--TableScan: tg_buffer projection=[id, value]
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]
-02)--BufferExec: capacity=1073741824
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
+02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]
# Verify correctness
query II
@@ -2248,8 +2254,8 @@ logical_plan
02)--TableScan: tg_buffer projection=[id, value]
physical_plan
01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3
-02)--BufferExec: capacity=1073741824
-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet
+02)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
query II
SELECT * FROM tg_buffer ORDER BY id ASC LIMIT 3;
@@ -2280,3 +2286,466 @@ SET datafusion.execution.collect_statistics = true;
statement ok
SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# ===========================================================
+# Test H: Row group reorder by statistics for TopK queries.
+# When a file has multiple row groups with overlapping or
+# out-of-order statistics, sort pushdown returns Inexact and
+# `reorder_by_statistics` reorders row groups within the file
+# so TopK finds optimal values first.
+# ===========================================================
+
+# Create a table with 9 rows and write to parquet with small row groups
+# so we get multiple row groups per file. Rows are inserted in a mixed
+# order so row groups span overlapping ranges (forcing Inexact path).
+statement ok
+CREATE TABLE th_mixed(id INT, value INT) AS VALUES
+ (15, 150), (5, 50), (25, 250),
+ (10, 100), (20, 200), (1, 10),
+ (30, 300), (3, 30), (18, 180);
+
+# Write with row_group_size=3 → 3 rows per RG, 3 RGs total
+# RG statistics (unsorted order): RG0(5-25), RG1(1-20), RG2(3-30)
+# Note: files are overlapping → Inexact path → TopK retained
+query I
+COPY (SELECT * FROM th_mixed)
+TO 'test_files/scratch/sort_pushdown/th_reorder/data.parquet'
+STORED AS PARQUET
+OPTIONS ('format.max_row_group_size' '3');
+----
+9
+
+statement ok
+SET datafusion.execution.target_partitions = 1;
+
+statement ok
+CREATE EXTERNAL TABLE th_reorder(id INT, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/th_reorder/data.parquet';
+
+# Test H.1: ASC ORDER BY with LIMIT — Inexact path (file has no declared ordering)
+# Plan: SortExec(TopK) preserved. RG reorder happens inside DataSourceExec
+# (not visible in EXPLAIN, but verified by unit tests in access_plan.rs).
+query TT
+EXPLAIN SELECT * FROM th_reorder ORDER BY id ASC LIMIT 3;
+----
+logical_plan
+01)Sort: th_reorder.id ASC NULLS LAST, fetch=3
+02)--TableScan: th_reorder projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/th_reorder/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]
+
+# Results must be correct regardless of RG reorder.
+query II
+SELECT * FROM th_reorder ORDER BY id ASC LIMIT 3;
+----
+1 10
+3 30
+5 50
+
+# Test H.2: DESC ORDER BY with LIMIT — reorder + reverse compose
+query TT
+EXPLAIN SELECT * FROM th_reorder ORDER BY id DESC LIMIT 3;
+----
+logical_plan
+01)Sort: th_reorder.id DESC NULLS FIRST, fetch=3
+02)--TableScan: th_reorder projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/th_reorder/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
+
+query II
+SELECT * FROM th_reorder ORDER BY id DESC LIMIT 3;
+----
+30 300
+25 250
+20 200
+
+# Test H.3: Full sort (no LIMIT) — output must still be correctly sorted
+query II
+SELECT * FROM th_reorder ORDER BY id ASC;
+----
+1 10
+3 30
+5 50
+10 100
+15 150
+18 180
+20 200
+25 250
+30 300
+
+# Test H.4: ORDER BY expression (not a simple column) — reorder should
+# gracefully skip, results still correct
+query II
+SELECT id, value FROM th_reorder ORDER BY id + 1 ASC LIMIT 3;
+----
+1 10
+3 30
+5 50
+
+# Cleanup Test H
+statement ok
+DROP TABLE th_mixed;
+
+statement ok
+DROP TABLE th_reorder;
+
+# ===========================================================
+# Test J: Non-overlapping RGs without WITH ORDER —
+# RG reorder via DynamicFilter sort_options
+# ===========================================================
+
+# Create data written in scrambled order but with non-overlapping RGs
+statement ok
+CREATE TABLE tj_high(id INT, value INT) AS VALUES (10,100),(11,110),(12,120);
+
+statement ok
+CREATE TABLE tj_mid(id INT, value INT) AS VALUES (4,40),(5,50),(6,60);
+
+statement ok
+CREATE TABLE tj_low(id INT, value INT) AS VALUES (1,10),(2,20),(3,30);
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 3;
+
+# Write in scrambled order: high, low, mid → RGs are non-overlapping but not sorted
+query I
+COPY (
+ SELECT * FROM tj_high
+ UNION ALL SELECT * FROM tj_low
+ UNION ALL SELECT * FROM tj_mid
+)
+TO 'test_files/scratch/sort_pushdown/tj_scrambled/data.parquet';
+----
+9
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 1048576;
+
+# No WITH ORDER — tests non-sort-pushdown TopK path
+statement ok
+CREATE EXTERNAL TABLE tj_scrambled(id INT, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/tj_scrambled/data.parquet';
+
+# Test J.1: DESC LIMIT — RG reorder + reverse from DynamicFilter
+query TT
+EXPLAIN SELECT * FROM tj_scrambled ORDER BY id DESC LIMIT 3;
+----
+logical_plan
+01)Sort: tj_scrambled.id DESC NULLS FIRST, fetch=3
+02)--TableScan: tj_scrambled projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tj_scrambled/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true
+
+# Test J.2: Results must be correct
+query II
+SELECT * FROM tj_scrambled ORDER BY id DESC LIMIT 3;
+----
+12 120
+11 110
+10 100
+
+# Test J.3: ASC LIMIT
+query II
+SELECT * FROM tj_scrambled ORDER BY id ASC LIMIT 3;
+----
+1 10
+2 20
+3 30
+
+# Test J.4: DESC with WHERE
+query II
+SELECT * FROM tj_scrambled WHERE value >= 50 ORDER BY id DESC LIMIT 2;
+----
+12 120
+11 110
+
+# Test J.5: Full sort without LIMIT (no TopK, no reorder)
+query II
+SELECT * FROM tj_scrambled ORDER BY id ASC;
+----
+1 10
+2 20
+3 30
+4 40
+5 50
+6 60
+10 100
+11 110
+12 120
+
+# Cleanup Test J
+statement ok
+DROP TABLE tj_high;
+
+statement ok
+DROP TABLE tj_mid;
+
+statement ok
+DROP TABLE tj_low;
+
+statement ok
+DROP TABLE tj_scrambled;
+
+# ===========================================================
+# Test K: Overlapping RGs without WITH ORDER —
+# reorder still helps TopK find good values faster
+# ===========================================================
+
+# Create data with overlapping ranges
+statement ok
+CREATE TABLE tk_data(id INT, value INT) AS VALUES
+(1,10),(5,50),(9,90),
+(3,30),(7,70),(11,110),
+(2,20),(6,60),(10,100);
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 3;
+
+query I
+COPY (SELECT * FROM tk_data)
+TO 'test_files/scratch/sort_pushdown/tk_overlap/data.parquet';
+----
+9
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 1048576;
+
+statement ok
+CREATE EXTERNAL TABLE tk_overlap(id INT, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/tk_overlap/data.parquet';
+
+# Test K.1: DESC LIMIT on overlapping RGs — results must be correct
+query II
+SELECT * FROM tk_overlap ORDER BY id DESC LIMIT 3;
+----
+11 110
+10 100
+9 90
+
+# Test K.2: ASC LIMIT on overlapping RGs
+query II
+SELECT * FROM tk_overlap ORDER BY id ASC LIMIT 3;
+----
+1 10
+2 20
+3 30
+
+# Test K.3: Full sort (verify all data present)
+query II
+SELECT * FROM tk_overlap ORDER BY id ASC;
+----
+1 10
+2 20
+3 30
+5 50
+6 60
+7 70
+9 90
+10 100
+11 110
+
+# Cleanup Test K
+statement ok
+DROP TABLE tk_data;
+
+statement ok
+DROP TABLE tk_overlap;
+
+# ===========================================================
+# Test L: Multi-key ORDER BY — uses first key for RG reorder
+# ===========================================================
+
+statement ok
+CREATE TABLE tl_data(id INT, name VARCHAR, value INT) AS VALUES
+(10, 'a', 100), (11, 'b', 110), (12, 'c', 120),
+(4, 'd', 40), (5, 'e', 50), (6, 'f', 60),
+(1, 'g', 10), (2, 'h', 20), (3, 'i', 30);
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 3;
+
+query I
+COPY (SELECT * FROM tl_data)
+TO 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet';
+----
+9
+
+statement ok
+SET datafusion.execution.parquet.max_row_group_size = 1048576;
+
+statement ok
+CREATE EXTERNAL TABLE tl_multikey(id INT, name VARCHAR, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet';
+
+# Test L.1: Multi-key DESC — first key determines RG reorder
+query IIT
+SELECT id, value, name FROM tl_multikey ORDER BY id DESC, name ASC LIMIT 3;
+----
+12 120 c
+11 110 b
+10 100 a
+
+# Test L.2: Multi-key ASC
+query IIT
+SELECT id, value, name FROM tl_multikey ORDER BY id ASC, name DESC LIMIT 3;
+----
+1 10 g
+2 20 h
+3 30 i
+
+# Test L.3: Multi-key with 3 sort columns
+query IIT
+SELECT id, value, name FROM tl_multikey ORDER BY id DESC, value DESC, name ASC LIMIT 5;
+----
+12 120 c
+11 110 b
+10 100 a
+6 60 f
+5 50 e
+
+# Test L.4: WITH ORDER + multi-key DESC LIMIT
+statement ok
+CREATE EXTERNAL TABLE tl_sorted(id INT, name VARCHAR, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet'
+WITH ORDER (id ASC);
+
+statement ok
+SET datafusion.optimizer.enable_sort_pushdown = true;
+
+# Multi-key with sort pushdown — first key triggers reverse
+query TT
+EXPLAIN SELECT id, value FROM tl_sorted ORDER BY id DESC, value ASC LIMIT 3;
+----
+logical_plan
+01)Sort: tl_sorted.id DESC NULLS FIRST, tl_sorted.value ASC NULLS LAST, fetch=3
+02)--TableScan: tl_sorted projection=[id, value]
+physical_plan
+01)SortExec: TopK(fetch=3), expr=[id@0 DESC, value@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tl_multikey/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC, value@1 ASC NULLS LAST], reverse_row_groups=true
+
+query II
+SELECT id, value FROM tl_sorted ORDER BY id DESC, value ASC LIMIT 3;
+----
+12 120
+11 110
+10 100
+
+# Cleanup Test L
+statement ok
+DROP TABLE tl_data;
+
+statement ok
+DROP TABLE tl_multikey;
+
+statement ok
+DROP TABLE tl_sorted;
+
+# ===========================================================
+# Test P: Multi-file reorder in shared work queue for TopK.
+# Files are reordered by statistics so the most promising
+# files are processed first across all partitions.
+# ===========================================================
+
+statement ok
+SET datafusion.execution.target_partitions = 2;
+
+statement ok
+SET datafusion.execution.collect_statistics = true;
+
+# Create 3 separate files with non-overlapping ranges in "wrong" order:
+# file_high (id 10-12), file_low (id 1-3), file_mid (id 4-6)
+statement ok
+CREATE TABLE tp_high(id INT, value INT) AS VALUES (10,100),(11,110),(12,120);
+
+statement ok
+CREATE TABLE tp_mid(id INT, value INT) AS VALUES (4,40),(5,50),(6,60);
+
+statement ok
+CREATE TABLE tp_low(id INT, value INT) AS VALUES (1,10),(2,20),(3,30);
+
+query I
+COPY (SELECT * FROM tp_high)
+TO 'test_files/scratch/sort_pushdown/tp_multifile/high.parquet';
+----
+3
+
+query I
+COPY (SELECT * FROM tp_mid)
+TO 'test_files/scratch/sort_pushdown/tp_multifile/mid.parquet';
+----
+3
+
+query I
+COPY (SELECT * FROM tp_low)
+TO 'test_files/scratch/sort_pushdown/tp_multifile/low.parquet';
+----
+3
+
+# External table over the directory — files discovered in FS order
+statement ok
+CREATE EXTERNAL TABLE tp_multifile(id INT, value INT)
+STORED AS PARQUET
+LOCATION 'test_files/scratch/sort_pushdown/tp_multifile/';
+
+# Test P.1: DESC LIMIT — reorder_files puts high.parquet first
+query II
+SELECT * FROM tp_multifile ORDER BY id DESC LIMIT 3;
+----
+12 120
+11 110
+10 100
+
+# Test P.2: ASC LIMIT — reorder_files puts low.parquet first
+query II
+SELECT * FROM tp_multifile ORDER BY id ASC LIMIT 3;
+----
+1 10
+2 20
+3 30
+
+# Test P.3: Full sort — all data present and correct
+query II
+SELECT * FROM tp_multifile ORDER BY id ASC;
+----
+1 10
+2 20
+3 30
+4 40
+5 50
+6 60
+10 100
+11 110
+12 120
+
+# Test P.4: DESC LIMIT with WHERE
+query II
+SELECT * FROM tp_multifile WHERE value >= 40 ORDER BY id DESC LIMIT 3;
+----
+12 120
+11 110
+10 100
+
+# Cleanup Test P
+statement ok
+DROP TABLE tp_high;
+
+statement ok
+DROP TABLE tp_mid;
+
+statement ok
+DROP TABLE tp_low;
+
+statement ok
+DROP TABLE tp_multifile;
+
+# Restore settings
+statement ok
+SET datafusion.execution.target_partitions = 4;
diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.slt
index 8a1fef0722..8cab67dac0 100644
--- a/datafusion/sqllogictest/test_files/topk.slt
+++ b/datafusion/sqllogictest/test_files/topk.slt
@@ -326,7 +326,7 @@ explain select number, letter, age from partial_sorted order by age desc limit 3
----
physical_plan
01)SortExec: TopK(fetch=3), expr=[age@2 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[age@2 DESC], reverse_row_groups=true
query TT
explain select number, letter, age from partial_sorted order by number desc, letter desc limit 3;
@@ -340,14 +340,14 @@ explain select number, letter, age from partial_sorted order by number asc limit
----
physical_plan
01)SortExec: TopK(fetch=3), expr=[number@0 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[number@0 ASC NULLS LAST]
query TT
explain select number, letter, age from partial_sorted order by letter asc, number desc limit 3;
----
physical_plan
01)SortExec: TopK(fetch=3), expr=[letter@1 ASC NULLS LAST, number@0 DESC], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[letter@1 ASC NULLS LAST, number@0 DESC]
# Explicit NULLS ordering cases (reversing the order of the NULLS on the number and letter orderings)
query TT
@@ -362,7 +362,7 @@ explain select number, letter, age from partial_sorted order by number desc NULL
----
physical_plan
01)SortExec: TopK(fetch=3), expr=[number@0 DESC NULLS LAST, letter@1 ASC NULLS LAST], preserve_partitioning=[false]
-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[number@0 DESC NULLS LAST, letter@1 ASC NULLS LAST], reverse_row_groups=true
# Verify that the sort prefix is correctly computed on the normalized ordering (removing redundant aliased columns)
diff --git a/datafusion/sqllogictest/test_files/window_topk_pushdown.slt b/datafusion/sqllogictest/test_files/window_topk_pushdown.slt
index 54e907af5c..20f746d1fd 100644
--- a/datafusion/sqllogictest/test_files/window_topk_pushdown.slt
+++ b/datafusion/sqllogictest/test_files/window_topk_pushdown.slt
@@ -49,7 +49,7 @@ OPTIONS ('format.has_header' 'true');
# Config reset
-# The SLT runner sets `target_partitions` to 4 instead of using the default, so
+# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
SET datafusion.execution.target_partitions = 4;