Apache DataFusion SQL Query Engine
feat: globally reorder files and row groups by statistics for TopK queries (#21956)
Qi Zhu102da39c7fce
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 1418dcad11..505ccb3a1b 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -3268,7 +3268,7 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartiUnionExecDataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquetSortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]- DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet+ DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]");Ok(())}@@ -3286,7 +3286,7 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartiUnionExecDataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquetSortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]- DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet+ DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]");Ok(())diff --git a/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs b/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rsindex d14afaf1b3..316cb177fa 100644--- a/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs+++ b/datafusion/core/tests/fuzz_cases/topk_filter_pushdown.rs@@ -317,10 +317,23 @@ async fn test_fuzz_topk_filter_pushdown() {.map(|col| orders.get(**col).unwrap()).multi_cartesian_product(){+ // Add remaining columns as tiebreakers (ASC NULLS LAST)+ // to ensure deterministic results when RG reorder changes+ // the read order of rows with equal sort key values.+ let tiebreakers: Vec<String> = ["id", "name", "department"]+ .iter()+ .filter(|c| {+ !order_columns+ .iter()+ .take(num_order_by_columns)+ .any(|oc| **oc == **c)+ })+ .map(|c| format!("{c} ASC NULLS LAST"))+ .collect();+ let all_orderings =+ orderings.into_iter().chain(tiebreakers.iter()).join(", ");let query = format!(- "SELECT * FROM test_table ORDER BY {} LIMIT {}",- orderings.into_iter().join(", "),- limit+ "SELECT * FROM test_table ORDER BY {all_orderings} LIMIT {limit}",);queries.push(query);}diff --git a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs b/datafusion/core/tests/physical_optimizer/pushdown_sort.rsindex e2700c3174..6a5833f43d 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_sort.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_sort.rs@@ -85,7 +85,7 @@ fn test_sort_pushdown_basic_phase1() {output:Ok:- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -114,7 +114,7 @@ fn test_sort_with_limit_phase1() {output:Ok:- SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -145,7 +145,7 @@ fn test_sort_multiple_columns_phase1() {output:Ok:- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC, b@1 DESC NULLS LAST]");}@@ -180,7 +180,7 @@ fn test_prefix_match_single_column() {output:Ok:- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]");}@@ -214,7 +214,7 @@ fn test_prefix_match_with_limit() {output:Ok:- SortExec: TopK(fetch=100), expr=[a@0 DESC NULLS LAST, b@1 ASC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST, b@1 ASC], reverse_row_groups=true");}@@ -249,7 +249,7 @@ fn test_prefix_match_through_transparent_nodes() {Ok:- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]");}@@ -285,8 +285,13 @@ fn test_exact_prefix_match_same_direction() {}#[test]-fn test_no_prefix_match_longer_than_source() {- // Test that prefix matching does NOT work if requested is longer than source+fn test_inexact_pushdown_when_prefix_longer_than_source() {+ // Source has [a DESC] ordering, request is [a ASC, b DESC] — longer+ // than the source ordering so the prefix can't be matched. The+ // primary sort column 'a' is in the file schema, so sort pushdown+ // returns `Inexact` with `sort_order_for_reorder` set, drops the+ // source's `output_ordering` (the runtime reorder invalidates it),+ // and leaves the outer `SortExec` to enforce the full ordering.let schema = schema();// Source has [a DESC] ordering (single column)@@ -310,7 +315,7 @@ fn test_no_prefix_match_longer_than_source() {output:Ok:- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 DESC NULLS LAST], file_type=parquet+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC, b@1 DESC NULLS LAST]");}@@ -343,7 +348,7 @@ fn test_sort_through_repartition() {Ok:- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -375,7 +380,7 @@ fn test_nested_sorts() {Ok:- SortExec: expr=[b@1 ASC], preserve_partitioning=[false]- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -435,7 +440,7 @@ fn test_sort_through_coalesce_partitions() {- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -467,7 +472,7 @@ fn test_complex_plan_with_multiple_operators() {- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec- RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -501,14 +506,19 @@ fn test_multiple_sorts_different_columns() {Ok:- SortExec: expr=[c@2 ASC], preserve_partitioning=[false]- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}#[test]-fn test_no_pushdown_for_unordered_source() {- // Verify pushdown does NOT happen for sources without ordering+fn test_inexact_pushdown_for_unordered_source() {+ // Source has no declared `output_ordering`, request is `[a ASC]`.+ // The reversed-equivalence check can't fire (nothing to reverse),+ // but 'a' is in the file schema — sort pushdown returns `Inexact`+ // with `sort_order_for_reorder` set so the opener can sort row+ // groups by `min(a)` at scan time. The surrounding `SortExec`+ // stays in place to enforce the full ordering.let schema = schema();let source = parquet_exec(schema.clone()); // No output_orderinglet sort_exprs = LexOrdering::new(vec![sort_expr("a", &schema)]).unwrap();@@ -524,14 +534,20 @@ fn test_no_pushdown_for_unordered_source() {output:Ok:- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 ASC]");}#[test]-fn test_no_pushdown_for_non_reverse_sort() {- // Verify pushdown does NOT happen when sort doesn't reverse source ordering+fn test_inexact_pushdown_when_request_doesnt_match_source_ordering() {+ // The requested sort column ('b') doesn't match the source's natural+ // ordering ('a' ASC). Neither natural nor reversed satisfies the+ // request, but 'b' is in the file schema — so sort pushdown returns+ // `Inexact` with `sort_order_for_reorder` set, drops the source's+ // claimed `output_ordering` (the runtime row-group reorder+ // invalidates it), and keeps the surrounding `SortExec` for+ // correctness.let schema = schema();// Source sorted by 'a' ASC@@ -554,7 +570,7 @@ fn test_no_pushdown_for_non_reverse_sort() {output:Ok:- SortExec: expr=[b@1 ASC], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[b@1 ASC]");}@@ -630,7 +646,7 @@ fn test_pushdown_through_blocking_node() {- SortExec: expr=[a@0 ASC], preserve_partitioning=[false]- AggregateExec: mode=Final, gby=[a@0 as a], aggr=[COUNT(b)], ordering_mode=Sorted- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -668,7 +684,7 @@ fn test_sort_pushdown_through_simple_projection() {Ok:- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -703,7 +719,7 @@ fn test_sort_pushdown_through_projection_with_alias() {Ok:- SortExec: expr=[id@0 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[a@0 as id, b@1 as value]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -792,7 +808,7 @@ fn test_sort_pushdown_projection_reordered_columns() {Ok:- SortExec: expr=[a@2 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[c@2 as c, b@1 as b, a@0 as a]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -826,7 +842,7 @@ fn test_sort_pushdown_projection_with_limit() {Ok:- SortExec: TopK(fetch=10), expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -860,7 +876,7 @@ fn test_sort_pushdown_through_projection() {Ok:- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[a@0 as a, b@1 as b]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}@@ -895,7 +911,7 @@ fn test_sort_pushdown_projection_subset_of_columns() {Ok:- SortExec: expr=[a@0 DESC NULLS LAST], preserve_partitioning=[false]- ProjectionExec: expr=[a@0 as a]- - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, reverse_row_groups=true+ - DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet, sort_order_for_reorder=[a@0 DESC NULLS LAST], reverse_row_groups=true");}diff --git a/datafusion/datasource-parquet/src/access_plan.rs b/datafusion/datasource-parquet/src/access_plan.rsindex 295665055b..edbea39948 100644--- a/datafusion/datasource-parquet/src/access_plan.rs+++ b/datafusion/datasource-parquet/src/access_plan.rs@@ -16,7 +16,12 @@// under the License.use crate::sort::reverse_row_selection;+use arrow::datatypes::Schema;use datafusion_common::{Result, assert_eq_or_internal_err};+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr_common::sort_expr::LexOrdering;+use log::debug;+use parquet::arrow::arrow_reader::statistics::StatisticsConverter;use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::file::metadata::{ParquetMetaData, RowGroupMetaData};@@ -485,6 +490,127 @@ impl PreparedAccessPlan {})}+ /// Reorder row groups by their min statistics for the given sort order.+ ///+ /// This helps TopK queries find optimal values first. Row groups are+ /// always sorted by min values in ASC order — direction (DESC) is+ /// handled separately by `reverse()` which is applied after reorder.+ ///+ /// Gracefully skips reordering when:+ /// - There is a row_selection (too complex to remap)+ /// - 0 or 1 row groups (nothing to reorder)+ /// - Sort expression is not a simple column reference+ /// - Statistics are unavailable+ pub(crate) fn reorder_by_statistics(+ mut self,+ sort_order: &LexOrdering,+ file_metadata: &ParquetMetaData,+ arrow_schema: &Schema,+ ) -> Result<Self> {+ // Skip if row_selection present (too complex to remap)+ if self.row_selection.is_some() {+ debug!("Skipping RG reorder: row_selection present");+ return Ok(self);+ }++ // Nothing to reorder+ if self.row_group_indexes.len() <= 1 {+ return Ok(self);+ }++ let first_sort_expr = sort_order.first();++ // Extract column name from sort expression+ let column: &Column = match first_sort_expr.expr.downcast_ref::<Column>() {+ Some(col) => col,+ None => {+ debug!("Skipping RG reorder: sort expr is not a simple column");+ return Ok(self);+ }+ };++ // Expected graceful skip: the sort column lives outside the+ // file schema (e.g. a partition column whose ordering came+ // through `reversed_satisfies` rather than `column_in_file_schema`).+ // Parquet has no per-RG stats for it. Bail out quietly — no+ // `debug_assert!` because this is a normal pushdown shape.+ if arrow_schema.field_with_name(column.name()).is_err() {+ debug!(+ "Skipping RG reorder: column `{}` not in file schema",+ column.name()+ );+ return Ok(self);+ }++ // From here, any `StatisticsConverter` / stats read / sort+ // failure is unexpected — the column exists in the file+ // schema, so building the converter and pulling typed mins+ // should succeed on any well-formed parquet file. Trip a+ // `debug_assert!` so CI catches regressions, but stay graceful+ // in release so a single odd file can't take down a scan.+ let converter = match StatisticsConverter::try_new(+ column.name(),+ arrow_schema,+ file_metadata.file_metadata().schema_descr(),+ ) {+ Ok(c) => c,+ Err(e) => {+ debug_assert!(+ false,+ "RG reorder: cannot create stats converter for `{}`: {e}",+ column.name(),+ );+ return Ok(self);+ }+ };++ // Always sort ASC by min values — direction is handled by reverse+ let rg_metadata: Vec<&RowGroupMetaData> = self+ .row_group_indexes+ .iter()+ .map(|&idx| file_metadata.row_group(idx))+ .collect();++ let stat_mins = match converter.row_group_mins(rg_metadata.iter().copied()) {+ Ok(vals) => vals,+ Err(e) => {+ debug_assert!(+ false,+ "RG reorder: cannot get min values for `{}`: {e}",+ column.name(),+ );+ return Ok(self);+ }+ };++ let sort_options = arrow::compute::SortOptions {+ descending: false,+ nulls_first: first_sort_expr.options.nulls_first,+ };+ let sorted_indices =+ match arrow::compute::sort_to_indices(&stat_mins, Some(sort_options), None) {+ Ok(indices) => indices,+ Err(e) => {+ debug_assert!(+ false,+ "RG reorder: arrow sort_to_indices failed for `{}`: {e}",+ column.name(),+ );+ return Ok(self);+ }+ };++ // Apply the reordering+ let original_indexes = self.row_group_indexes.clone();+ self.row_group_indexes = sorted_indices+ .values()+ .iter()+ .map(|&i| original_indexes[i as usize])+ .collect();++ Ok(self)+ }+/// Reverse the access plan for reverse scanningpub(crate) fn reverse(mut self, file_metadata: &ParquetMetaData) -> Result<Self> {// Get the row group indexes before reversing@@ -722,4 +848,182 @@ mod test {.unwrap();Arc::new(SchemaDescriptor::new(Arc::new(schema)))}++ // ----------------------------------------------------------------+ // `reorder_by_statistics` tests+ // ----------------------------------------------------------------++ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_expr::Operator;+ use datafusion_physical_expr::expressions::{BinaryExpr, lit};+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+ use parquet::file::metadata::FileMetaData;+ use parquet::file::statistics::Statistics as ParquetStatistics;++ /// Single-column int32 schema named "a".+ fn int_schema_descr() -> SchemaDescPtr {+ use parquet::basic::Type as PhysicalType;+ use parquet::schema::types::Type as SchemaType;+ let field = SchemaType::primitive_type_builder("a", PhysicalType::INT32)+ .build()+ .unwrap();+ let schema = SchemaType::group_type_builder("schema")+ .with_fields(vec![Arc::new(field)])+ .build()+ .unwrap();+ Arc::new(SchemaDescriptor::new(Arc::new(schema)))+ }++ /// Build a `ParquetMetaData` with one row group per element of+ /// `mins`. Each row group declares int32 statistics with+ /// `min == max == mins[i]` so the reorder key is unambiguous.+ fn parquet_metadata_with_int_mins(mins: &[i32]) -> ParquetMetaData {+ let schema_descr = int_schema_descr();+ let row_groups: Vec<RowGroupMetaData> = mins+ .iter()+ .map(|&m| {+ let stats =+ ParquetStatistics::int32(Some(m), Some(m), None, Some(0), false);+ let column = ColumnChunkMetaData::builder(schema_descr.column(0))+ .set_statistics(stats)+ .set_num_values(100)+ .build()+ .unwrap();+ RowGroupMetaData::builder(schema_descr.clone())+ .set_num_rows(100)+ .set_column_metadata(vec![column])+ .build()+ .unwrap()+ })+ .collect();+ let file_metadata =+ FileMetaData::new(0, 0, None, None, schema_descr.clone(), None);+ ParquetMetaData::new(file_metadata, row_groups)+ }++ fn arrow_schema_a_int() -> Schema {+ Schema::new(vec![Field::new("a", DataType::Int32, true)])+ }++ fn lex_ordering_a_asc() -> LexOrdering {+ LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::new(Column::new("a", 0)),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }])+ .unwrap()+ }++ /// Happy path: three row groups with mins 50/10/100. After+ /// `reorder_by_statistics` the indexes are ordered ASC by `min`,+ /// i.e. RG 1 (min=10) first, then RG 0 (min=50), then RG 2+ /// (min=100).+ #[test]+ fn reorder_by_statistics_sorts_row_groups_asc_by_min() {+ let metadata = parquet_metadata_with_int_mins(&[50, 10, 100]);+ let plan = PreparedAccessPlan::new(vec![0, 1, 2], None).unwrap();++ let result = plan+ .reorder_by_statistics(+ &lex_ordering_a_asc(),+ &metadata,+ &arrow_schema_a_int(),+ )+ .unwrap();++ assert_eq!(result.row_group_indexes, vec![1, 0, 2]);+ }++ /// A `row_selection` is "too complex to remap" through reorder,+ /// so the function short-circuits and returns the input untouched.+ #[test]+ fn reorder_by_statistics_skips_when_row_selection_present() {+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);+ let selection = RowSelection::from(vec![RowSelector::select(100)]);+ let plan = PreparedAccessPlan::new(vec![0, 1], Some(selection)).unwrap();++ let result = plan+ .reorder_by_statistics(+ &lex_ordering_a_asc(),+ &metadata,+ &arrow_schema_a_int(),+ )+ .unwrap();++ assert_eq!(result.row_group_indexes, vec![0, 1]);+ }++ /// One row group means nothing to reorder.+ #[test]+ fn reorder_by_statistics_skips_when_at_most_one_row_group() {+ let metadata = parquet_metadata_with_int_mins(&[50]);+ let plan = PreparedAccessPlan::new(vec![0], None).unwrap();++ let result = plan+ .reorder_by_statistics(+ &lex_ordering_a_asc(),+ &metadata,+ &arrow_schema_a_int(),+ )+ .unwrap();++ assert_eq!(result.row_group_indexes, vec![0]);+ }++ /// Non-`Column` sort expressions (e.g. `a + 1`,+ /// `date_trunc(...)`) can't drive a stats lookup, so reorder is+ /// skipped. The opener falls back to whatever order it received.+ #[test]+ fn reorder_by_statistics_skips_for_non_column_sort_expr() {+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);+ let plan = PreparedAccessPlan::new(vec![0, 1], None).unwrap();+ let arrow_schema = arrow_schema_a_int();+ let order = LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Plus,+ lit(1i32),+ )),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }])+ .unwrap();++ let result = plan+ .reorder_by_statistics(&order, &metadata, &arrow_schema)+ .unwrap();++ assert_eq!(result.row_group_indexes, vec![0, 1]);+ }++ /// When the sort column lives outside the file's arrow schema+ /// (e.g. a partition column that reached this method through+ /// `try_pushdown_sort`'s reversed-equivalence branch), reorder is+ /// an expected graceful skip — no `debug_assert!` should fire.+ #[test]+ fn reorder_by_statistics_skips_when_column_not_in_arrow_schema() {+ let metadata = parquet_metadata_with_int_mins(&[50, 10]);+ let plan = PreparedAccessPlan::new(vec![0, 1], None).unwrap();+ // Arrow schema only has "a"; the sort references "b".+ let arrow_schema = arrow_schema_a_int();+ let order = LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::new(Column::new("b", 0)),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }])+ .unwrap();++ let result = plan+ .reorder_by_statistics(&order, &metadata, &arrow_schema)+ .unwrap();++ assert_eq!(result.row_group_indexes, vec![0, 1]);+ }}diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rsindex 7f9c51de8a..e41bc82eb7 100644--- a/datafusion/datasource-parquet/src/opener.rs+++ b/datafusion/datasource-parquet/src/opener.rs@@ -51,6 +51,7 @@ use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, is_dynamic_physical_expr,};+use datafusion_physical_expr_common::sort_expr::LexOrdering;use datafusion_physical_plan::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder,MetricCategory, PruningMetrics,@@ -76,9 +77,7 @@ use parquet::arrow::parquet_column;use parquet::arrow::push_decoder::{ParquetPushDecoder, ParquetPushDecoderBuilder};use parquet::basic::Type;use parquet::bloom_filter::Sbbf;-use parquet::file::metadata::{- PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData,-};+use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader};/// Stateless Parquet morselizer implementation.///@@ -139,6 +138,8 @@ pub(super) struct ParquetMorselizer {pub max_predicate_cache_size: Option<usize>,/// Whether to read row groups in reverse orderpub reverse_row_groups: bool,+ /// Optional sort order used to reorder row groups by their min/max statistics.+ pub sort_order_for_reorder: Option<LexOrdering>,}impl fmt::Debug for ParquetMorselizer {@@ -289,6 +290,7 @@ struct PreparedParquetOpen {predicate_creation_errors: Count,max_predicate_cache_size: Option<usize>,reverse_row_groups: bool,+ sort_order_for_reorder: Option<LexOrdering>,preserve_order: bool,#[cfg(feature = "parquet_encryption")]file_decryption_properties: Option<Arc<FileDecryptionProperties>>,@@ -658,6 +660,7 @@ impl ParquetMorselizer {predicate_creation_errors,max_predicate_cache_size: self.max_predicate_cache_size,reverse_row_groups: self.reverse_row_groups,+ sort_order_for_reorder: self.sort_order_for_reorder.clone(),preserve_order: self.preserve_order,#[cfg(feature = "parquet_encryption")]file_decryption_properties: None,@@ -1112,6 +1115,41 @@ impl RowGroupsPrunedParquetOpen {);}+ // Prepare access plans, then apply row-group ordering tweaks per+ // run. Two composable steps:+ //+ // 1. `reorder_by_statistics`: sort row groups by `min(col)` ASC.+ // Fixes out-of-order row groups (e.g. append-heavy workloads).+ // Skipped gracefully when statistics aren't available or the+ // sort expression isn't a plain column.+ //+ // 2. `reverse`: flip the iteration order for DESC requests, applied+ // AFTER any reorder so the reversed order is correct whether or+ // not reorder changed anything. Also handles `row_selection`+ // remapping.+ //+ // For sorted data: reorder is a no-op, reverse gives perfect DESC.+ // For unsorted data: reorder fixes the order, reverse flips for DESC.+ //+ // Both inputs come from the sort-pushdown channel —+ // `ParquetSource::try_pushdown_sort` sets `sort_order_for_reorder`+ // and/or `reverse_row_groups`.+ let prepare_access_plan =+ |plan: ParquetAccessPlan| -> Result<PreparedAccessPlan> {+ let mut prepared_plan = plan.prepare(rg_metadata)?;+ if let Some(sort_order) = prepared.sort_order_for_reorder.as_ref() {+ prepared_plan = prepared_plan.reorder_by_statistics(+ sort_order,+ file_metadata.as_ref(),+ &prepared.physical_file_schema,+ )?;+ }+ if prepared.reverse_row_groups {+ prepared_plan = prepared_plan.reverse(file_metadata.as_ref())?;+ }+ Ok(prepared_plan)+ };+let arrow_reader_metrics = ArrowReaderMetrics::enabled();let read_plan = build_projection_read_plan(prepared.projection.expr_iter(),@@ -1146,12 +1184,7 @@ impl RowGroupsPrunedParquetOpen {// Build a decoder per run.let mut decoders = VecDeque::with_capacity(runs.len());for run in runs {- let prepared_access_plan = prepare_access_plan(- run.access_plan,- rg_metadata,- file_metadata.as_ref(),- prepared.reverse_row_groups,- )?;+ let prepared_access_plan = prepare_access_plan(run.access_plan)?;let mut builder =decoder_config.build(prepared_access_plan, reader_metadata.clone());if run.needs_filter {@@ -1291,19 +1324,6 @@ impl<'a> RowFilterGenerator<'a> {}}-fn prepare_access_plan(- plan: ParquetAccessPlan,- rg_metadata: &[RowGroupMetaData],- file_metadata: &ParquetMetaData,- reverse_row_groups: bool,-) -> Result<PreparedAccessPlan> {- let mut prepared_access_plan = plan.prepare(rg_metadata)?;- if reverse_row_groups {- prepared_access_plan = prepared_access_plan.reverse(file_metadata)?;- }- Ok(prepared_access_plan)-}-/// State shared while building [`ParquetPushDecoder`]s for one file scan.////// A scan can be split into multiple decoder runs when row groups have@@ -1971,6 +1991,7 @@ mod test {encryption_factory: None,max_predicate_cache_size: self.max_predicate_cache_size,reverse_row_groups: self.reverse_row_groups,+ sort_order_for_reorder: None,}}}diff --git a/datafusion/datasource-parquet/src/sort.rs b/datafusion/datasource-parquet/src/sort.rsindex db22363aa3..c1cf4e8b78 100644--- a/datafusion/datasource-parquet/src/sort.rs+++ b/datafusion/datasource-parquet/src/sort.rs@@ -17,7 +17,11 @@//! Sort-related utilities for Parquet scanning-use datafusion_common::Result;+use arrow::datatypes::Schema;+use datafusion_common::{Result, ScalarValue};+use datafusion_datasource::PartitionedFile;+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr_common::sort_expr::LexOrdering;use parquet::arrow::arrow_reader::{RowSelection, RowSelector};use parquet::file::metadata::ParquetMetaData;use std::collections::HashMap;@@ -118,6 +122,89 @@ pub fn reverse_row_selection(Ok(RowSelection::from(reversed_selectors))}+/// Reorder a file list so the most "promising" files are read first,+/// matching `PreparedAccessPlan::reorder_by_statistics` at the+/// row-group level: key off the file's `min(col)`, and let the sort+/// direction follow the request (ASC by `min` for ASC requests, DESC+/// by `min` for DESC requests).+///+/// Keeping both layers consistent matters because they share the same+/// convergence story for TopK's dynamic filter: file `i`'s `min` is a+/// lower bound on every row group inside it, so the order chosen here+/// is a natural prefix of the order `reorder_by_statistics` will+/// produce within each file.+///+/// No-op when:+/// * `sort_order` is `None` (sort pushdown didn't fire);+/// * the leading sort expression is not a plain `Column`; or+/// * the column is not in `table_schema`.+///+/// Files missing statistics sort to the end so present-stats files+/// run first.+pub(crate) fn reorder_files_by_min_statistics(+ mut files: Vec<PartitionedFile>,+ sort_order: Option<&LexOrdering>,+ reverse_row_groups: bool,+ table_schema: &Schema,+) -> Vec<PartitionedFile> {+ let Some((col_name, descending)) =+ extract_topk_sort_info(sort_order, reverse_row_groups)+ else {+ return files;+ };++ let Ok(col_idx) = table_schema.index_of(&col_name) else {+ return files;+ };++ files.sort_by(|a, b| {+ let key_a = file_min_value(a, col_idx);+ let key_b = file_min_value(b, col_idx);+ match (key_a, key_b) {+ (Some(va), Some(vb)) => {+ let cmp = va.partial_cmp(&vb).unwrap_or(std::cmp::Ordering::Equal);+ if descending { cmp.reverse() } else { cmp }+ }+ (Some(_), None) => std::cmp::Ordering::Less,+ (None, Some(_)) => std::cmp::Ordering::Greater,+ (None, None) => std::cmp::Ordering::Equal,+ }+ });++ log::debug!(+ "Reordered {} files by min({}) {} for TopK optimization",+ files.len(),+ col_name,+ if descending { "DESC" } else { "ASC" }+ );++ files+}++/// Extract the `(column name, descending)` tuple used by file-level+/// reordering. Returns `None` when the sort order isn't set or the+/// leading sort expression isn't a plain `Column`.+fn extract_topk_sort_info(+ sort_order: Option<&LexOrdering>,+ reverse_row_groups: bool,+) -> Option<(String, bool)> {+ let sort_order = sort_order?;+ let first = sort_order.first();+ let col = first.expr.downcast_ref::<Column>()?;+ Some((col.name().to_string(), reverse_row_groups))+}++/// File's per-column `min` for the reorder key.+fn file_min_value(file: &PartitionedFile, col_idx: usize) -> Option<ScalarValue> {+ let stats = file.statistics.as_ref()?;+ stats+ .column_statistics+ .get(col_idx)?+ .min_value+ .get_value()+ .cloned()+}+#[cfg(test)]mod tests {use crate::ParquetAccessPlan;diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 91e3d22749..cf1fcea005 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -55,7 +55,7 @@ use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;#[cfg(feature = "parquet_encryption")]use datafusion_execution::parquet_encryption::EncryptionFactory;-use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};use itertools::Itertools;use object_store::ObjectStore;#[cfg(feature = "parquet_encryption")]@@ -289,11 +289,13 @@ pub struct ParquetSource {pub(crate) projection: ProjectionExprs,#[cfg(feature = "parquet_encryption")]pub(crate) encryption_factory: Option<Arc<dyn EncryptionFactory>>,- /// If true, read files in reverse order and reverse row groups within files.- /// But it's not guaranteed that rows within row groups are in reverse order,- /// so we still need to sort them after reading, so the reverse scan is inexact.- /// Used to optimize ORDER BY ... DESC on sorted data.+ /// If true, the opener flips row-group iteration order. Within-+ /// row-group order is on-disk order, so the scan is `Inexact` and+ /// a `SortExec` is kept in the plan.reverse_row_groups: bool,+ /// Sort order driving `PreparedAccessPlan::reorder_by_statistics`+ /// in the opener.+ sort_order_for_reorder: Option<LexOrdering>,}impl ParquetSource {@@ -319,6 +321,7 @@ impl ParquetSource {#[cfg(feature = "parquet_encryption")]encryption_factory: None,reverse_row_groups: false,+ sort_order_for_reorder: None,}}@@ -474,6 +477,7 @@ impl ParquetSource {}}+ #[cfg(test)]pub(crate) fn with_reverse_row_groups(mut self, reverse_row_groups: bool) -> Self {self.reverse_row_groups = reverse_row_groups;self@@ -581,9 +585,22 @@ impl FileSource for ParquetSource {encryption_factory: self.get_encryption_factory_with_config(),max_predicate_cache_size: self.max_predicate_cache_size(),reverse_row_groups: self.reverse_row_groups,+ sort_order_for_reorder: self.sort_order_for_reorder.clone(),}))}+ fn reorder_files(+ &self,+ files: Vec<datafusion_datasource::PartitionedFile>,+ ) -> Vec<datafusion_datasource::PartitionedFile> {+ crate::sort::reorder_files_by_min_statistics(+ files,+ self.sort_order_for_reorder.as_ref(),+ self.reverse_row_groups,+ self.table_schema.table_schema(),+ )+ }+fn table_schema(&self) -> &TableSchema {&self.table_schema}@@ -629,7 +646,11 @@ impl FileSource for ParquetSource {write!(f, "{predicate_string}")?;- // Add reverse_scan info if enabled+ // Inexact sort-pushdown markers: surface both flags so+ // readers can see the optimization fired.+ if let Some(sort_order) = &self.sort_order_for_reorder {+ write!(f, ", sort_order_for_reorder=[{sort_order}]")?;+ }if self.reverse_row_groups {write!(f, ", reverse_row_groups=true")?;}@@ -744,25 +765,41 @@ impl FileSource for ParquetSource {/// Try to optimize the scan to produce data in the requested sort order.///- /// This method receives:+ /// Inputs:/// 1. The query's required ordering (`order` parameter)- /// 2. The file's natural ordering (via `self.file_ordering`, set by FileScanConfig)+ /// 2. The source's equivalence properties (`eq_properties`)///- /// With both pieces of information, ParquetSource can decide what optimizations to apply.+ /// # Returns+ /// - `Exact`: the source's natural ordering already satisfies the+ /// request. The surrounding `SortExec` can be eliminated provided+ /// files within each group are non-overlapping (verified by+ /// `FileScanConfig`).+ /// - `Inexact`: the source can approximate the request via two+ /// composable runtime steps — stats-based row-group reorder+ /// (skipped when the leading sort key isn't a plain `Column`+ /// in the file schema) and row-group iteration reverse. A+ /// `SortExec` is still required for full correctness, but limit+ /// pushdown and `TopK` benefit immediately.+ /// - `Unsupported`: no approximation is available.///- /// # Behavior- /// - Returns `Exact` when the file's natural ordering (from Parquet metadata) already- /// satisfies the requested ordering. This allows the Sort operator to be eliminated- /// if the files within each group are also non-overlapping (checked by FileScanConfig).- /// - Returns `Inexact` when reversing the row group scan order would help satisfy the- /// requested ordering. We still need a Sort operator at a higher level because:- /// - We only reverse row group read order, not rows within row groups- /// - This provides approximate ordering that benefits limit pushdown+ /// # How the Inexact result is communicated///- /// # Returns- /// - `Exact`: The file's natural ordering satisfies the request (within-file ordering guaranteed)- /// - `Inexact`: Created an optimized source (e.g., reversed scan) that approximates the order- /// - `Unsupported`: Cannot optimize for this ordering+ /// The result is carried through two fields on `ParquetSource`:+ ///+ /// - `sort_order_for_reorder`: set to the request's `LexOrdering`+ /// whenever the pushdown fires, regardless of whether the+ /// leading expression is a plain `Column`. The opener invokes+ /// `PreparedAccessPlan::reorder_by_statistics`, which skips+ /// when the expression can't be looked up in parquet metadata.+ /// Exposing the field unconditionally keeps `EXPLAIN` honest+ /// about what the source was asked to approximate.+ /// - `reverse_row_groups`: drives the opener's iteration flip.+ /// When stats reorder applies (column-in-schema), this is just+ /// the request's direction — the reorder produces ASC-by-min,+ /// so reverse iff the query asks for DESC. When stats reorder+ /// doesn't apply but the reversed source ordering satisfies+ /// the request (function-wrapped case), this is always `true`+ /// because we're flipping the file's natural order.fn try_pushdown_sort(&self,order: &[PhysicalSortExpr],@@ -782,24 +819,68 @@ impl FileSource for ParquetSource {});}- // Build new equivalence properties with the reversed ordering.- // This allows us to check if the reversed ordering satisfies the request- // by leveraging:- // - Function monotonicity (e.g., extract_year_month preserves ordering)- // - Constant columns (from filters)- // - Other equivalence relationships+ // If the source's declared ordering is a non-empty *proper* prefix+ // of the request (e.g. source `[a DESC, b ASC]`, request+ // `[a DESC, b ASC, c DESC]`), decline pushdown so the outer+ // `SortExec`'s `sort_prefix` optimisation — prefix-aware early+ // termination in `TopK` — can still fire. Firing the Inexact+ // pipeline below would invalidate the source's `output_ordering`+ // (the runtime row-group reorder is approximate, so we can't+ // honour the declared ordering anymore), which is exactly what+ // `EnforceSorting` needs to derive `sort_prefix`. On data that+ // is already in prefix order the stats-based reorder is mostly+ // a no-op anyway, so the trade-off is plainly bad.+ for prefix_len in 1..order.len() {+ let prefix = order[..prefix_len].to_vec();+ if eq_properties.ordering_satisfy(prefix.iter().cloned())? {+ return Ok(SortOrderPushdownResult::Unsupported);+ }+ }++ // Inexact pushdown. Two independent signals; either is enough+ // to produce an approximate ordering, and they compose when+ // both apply://- // Example flow:- // 1. File ordering: [extract_year_month(ws) DESC, ws DESC]- // 2. After reversal: [extract_year_month(ws) ASC, ws ASC]- // 3. Requested: [ws ASC]- // 4. Through extract_year_month's monotonicity property, the reversed- // ordering satisfies [ws ASC] even though it has additional prefix+ // 1. `column_in_file_schema`: the request's leading sort key is+ // a plain `Column` present in the file schema. The opener+ // can sort row groups by that column's `min` via parquet+ // statistics. Drives `sort_order_for_reorder`'s actual use.+ //+ // 2. `reversed_satisfies`: the source's declared ordering,+ // when reversed, satisfies the request. This is strictly+ // more powerful than the column-in-schema check because it+ // runs the request through `EquivalenceProperties`'s full+ // reasoning machinery:+ //+ // - Function monotonicity: e.g. file declares+ // `[extract_year_month(ws) DESC, ws DESC]`, request is+ // `[ws ASC]`; the reversed ordering still satisfies the+ // request via `extract_year_month`'s monotonicity even+ // though parquet has no stats keyed by the function+ // expression itself.+ // - Constant columns from filters: equivalence classes can+ // mark columns as constant under a predicate, allowing+ // requested orderings on those columns to be trivially+ // satisfied.+ // - Other equivalence relationships (e.g. `a = b` transfers+ // ordering between `a` and `b`).+ //+ // `reorder_by_statistics` can't substitute for any of the+ // above because it can only look up min/max for a plain+ // physical column.+ //+ // `sort_order_for_reorder` is set in both cases so EXPLAIN+ // shows what the source was asked to approximate; the opener+ // skips stats-based reorder when the leading expression isn't+ // a plain `Column`.+ //+ // The reversal flips each `PhysicalSortExpr` (both descending+ // and nulls_first) and rebuilds an `EquivalenceProperties` so+ // the request can be tested against the reversed orderings+ // via the same `ordering_satisfy` API.let reversed_eq_properties = {let mut new = eq_properties.clone();new.clear_orderings();-- // Reverse each ordering in the equivalence propertieslet reversed_orderings = eq_properties.oeq_class().iter()@@ -810,19 +891,44 @@ impl FileSource for ParquetSource {.collect::<Vec<_>>()}).collect::<Vec<_>>();-new.add_orderings(reversed_orderings);new};+ let reversed_satisfies =+ reversed_eq_properties.ordering_satisfy(order.iter().cloned())?;+ let sort_order = LexOrdering::new(order.iter().cloned());+ let column_in_file_schema = sort_order.as_ref().is_some_and(|s| {+ s.first()+ .expr+ .downcast_ref::<datafusion_physical_expr::expressions::Column>()+ .is_some_and(|col| {+ self.table_schema+ .file_schema()+ .field_with_name(col.name())+ .is_ok()+ })+ });- // Check if the reversed ordering satisfies the requested ordering- if !reversed_eq_properties.ordering_satisfy(order.iter().cloned())? {+ if !column_in_file_schema && !reversed_satisfies {return Ok(SortOrderPushdownResult::Unsupported);}- // Return Inexact because we're only reversing row group order,- // not guaranteeing perfect row-level ordering- let new_source = self.clone().with_reverse_row_groups(true);+ // `reverse_row_groups` has different starting points in the+ // two cases:+ // - With stats reorder (column-in-schema): the reorder produces+ // ASC-by-min, so reverse iff the request is DESC.+ // - Without stats reorder (reversed-eq fallback): we flip the+ // file's natural order, so always reverse.+ let is_descending = sort_order+ .as_ref()+ .is_some_and(|s| s.first().options.descending);+ let mut new_source = self.clone();+ new_source.sort_order_for_reorder = sort_order;+ new_source.reverse_row_groups = if column_in_file_schema {+ is_descending+ } else {+ true+ };Ok(SortOrderPushdownResult::Inexact {inner: Arc::new(new_source) as Arc<dyn FileSource>,})@@ -947,4 +1053,525 @@ mod tests {assert!(source.reverse_row_groups());assert!(source.filter().is_some());}++ /// Helpers for the `try_pushdown_sort` regression tests below.+ mod pushdown_sort_helpers {+ use super::*;+ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_physical_expr::expressions::Column;+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;++ pub(super) fn schema_with_a_int() -> Arc<Schema> {+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]))+ }++ pub(super) fn sort_expr_on(+ schema: &Arc<Schema>,+ name: &str,+ descending: bool,+ ) -> PhysicalSortExpr {+ let idx = schema.index_of(name).unwrap();+ PhysicalSortExpr {+ expr: Arc::new(Column::new(name, idx)),+ options: SortOptions {+ descending,+ nulls_first: true,+ },+ }+ }+ }++ /// When neither natural nor reversed ordering matches the request,+ /// but the sort column is a plain `Column` present in the file+ /// schema, `try_pushdown_sort` returns `Inexact` with+ /// `sort_order_for_reorder` set so the opener can reorder row+ /// groups by min/max statistics at runtime.+ #[test]+ fn try_pushdown_sort_returns_inexact_when_column_in_schema_asc() {+ use datafusion_physical_expr::EquivalenceProperties;+ use pushdown_sort_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(Arc::clone(&schema));+ let order = vec![sort_expr_on(&schema, "a", false)];+ // No declared natural ordering on the source.+ let eq = EquivalenceProperties::new(Arc::clone(&schema));++ let result = source.try_pushdown_sort(&order, &eq).unwrap();++ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("expected Inexact, got a different variant");+ };+ // Downcast back to `ParquetSource` to inspect the fields the+ // opener reads to drive `reorder_by_statistics` / `reverse`.+ let inner_parquet = inner+ .downcast_ref::<ParquetSource>()+ .expect("inner is ParquetSource");+ let sort_order = inner_parquet+ .sort_order_for_reorder+ .as_ref()+ .expect("sort_order_for_reorder must be set so the opener can reorder");+ assert_eq!(sort_order.first().expr.to_string(), "a@0");+ // ASC request must not flip RG iteration order.+ assert!(+ !inner_parquet.reverse_row_groups(),+ "ASC request must not set reverse_row_groups",+ );+ }++ /// Same as above but for DESC. `reverse_row_groups` must also be+ /// `true` so the opener flips iteration order.+ #[test]+ fn try_pushdown_sort_returns_inexact_when_column_in_schema_desc() {+ use datafusion_physical_expr::EquivalenceProperties;+ use pushdown_sort_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(Arc::clone(&schema));+ let order = vec![sort_expr_on(&schema, "a", true)];+ let eq = EquivalenceProperties::new(Arc::clone(&schema));++ let result = source.try_pushdown_sort(&order, &eq).unwrap();++ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("expected Inexact, got a different variant");+ };+ let inner_parquet = inner+ .downcast_ref::<ParquetSource>()+ .expect("inner is ParquetSource");+ assert!(inner_parquet.sort_order_for_reorder.is_some());+ assert!(+ inner_parquet.reverse_row_groups(),+ "DESC request must set reverse_row_groups",+ );+ }++ /// A non-`Column` leading sort expression (e.g. `a + 1`,+ /// `date_trunc('hour', ts)`) with no declared source ordering+ /// yields `Unsupported` — parquet stats need a column name to+ /// look up min/max, and there's no source ordering to reverse.+ #[test]+ fn try_pushdown_sort_returns_unsupported_for_non_column_sort_expr() {+ use arrow::compute::SortOptions;+ use datafusion_physical_expr::EquivalenceProperties;+ use datafusion_physical_expr::expressions::{BinaryExpr, Column, lit};+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+ use pushdown_sort_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(Arc::clone(&schema));++ // `a + 1` — not a plain Column.+ let order = vec![PhysicalSortExpr {+ expr: Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ datafusion_expr::Operator::Plus,+ lit(1i32),+ )),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }];+ let eq = EquivalenceProperties::new(Arc::clone(&schema));++ let result = source.try_pushdown_sort(&order, &eq).unwrap();+ assert!(+ matches!(result, SortOrderPushdownResult::Unsupported),+ "non-Column sort expression must yield Unsupported",+ );+ }++ /// A sort column missing from the file schema with no declared+ /// source ordering yields `Unsupported` — there are no parquet+ /// stats for that column and no source ordering to reverse.+ #[test]+ fn try_pushdown_sort_returns_unsupported_when_column_not_in_file_schema() {+ use arrow::compute::SortOptions;+ use datafusion_physical_expr::EquivalenceProperties;+ use datafusion_physical_expr::expressions::Column;+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;+ use pushdown_sort_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(Arc::clone(&schema));++ // Reference a column ("b") that does not exist in the file+ // schema (which only has "a"). The Column expression itself is+ // well-formed; only its name is unknown to the file.+ let order = vec![PhysicalSortExpr {+ expr: Arc::new(Column::new("b", 0)),+ options: SortOptions {+ descending: false,+ nulls_first: true,+ },+ }];+ let eq = EquivalenceProperties::new(Arc::clone(&schema));++ let result = source.try_pushdown_sort(&order, &eq).unwrap();+ assert!(+ matches!(result, SortOrderPushdownResult::Unsupported),+ "column not in file schema must yield Unsupported",+ );+ }++ /// Regression: when the source declares `[a DESC]` and the request is+ /// `[a ASC]`, both `column_in_file_schema` and `reversed_satisfies`+ /// are true. `reverse_row_groups` must follow the *request's*+ /// direction (false for ASC) — not the source's, and not the OR of+ /// both signals. The opener's stats-based reorder produces+ /// ASC-by-min, so an ASC request needs no further flip; flipping+ /// would incorrectly emit DESC.+ #[test]+ fn try_pushdown_sort_source_desc_request_asc_does_not_reverse() {+ use datafusion_physical_expr::EquivalenceProperties;+ use pushdown_sort_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(Arc::clone(&schema));+ // Source declares `[a DESC]`.+ let mut eq = EquivalenceProperties::new(Arc::clone(&schema));+ eq.add_ordering(vec![sort_expr_on(&schema, "a", true)]);+ // Request `[a ASC]`.+ let order = vec![sort_expr_on(&schema, "a", false)];++ let result = source.try_pushdown_sort(&order, &eq).unwrap();++ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("expected Inexact, got a different variant");+ };+ let inner_parquet = inner+ .downcast_ref::<ParquetSource>()+ .expect("inner is ParquetSource");+ assert!(+ inner_parquet.sort_order_for_reorder.is_some(),+ "sort_order_for_reorder must be set",+ );+ assert!(+ !inner_parquet.reverse_row_groups(),+ "ASC request on source-DESC must not set reverse_row_groups; \+ a stale `reversed_satisfies || is_descending` formula would \+ incorrectly flip iteration to DESC after the stats reorder",+ );+ }++ /// A sort column that is *not* in the file schema (here: a partition+ /// column `b`) but the source's *reversed* declared ordering does+ /// satisfy the request. Pushdown fires via the reversed-equivalence+ /// path; `sort_order_for_reorder` is still set (so EXPLAIN reflects+ /// what the source was asked to approximate, even though the opener+ /// will skip its stats reorder because `b` has no per-RG min/max in+ /// the parquet file), and `reverse_row_groups` is `true` because we+ /// flip the file's natural order rather than re-sort by stats.+ #[test]+ fn try_pushdown_sort_returns_inexact_via_reversed_eq_when_column_not_in_file_schema()+ {+ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_datasource::TableSchema;+ use datafusion_physical_expr::EquivalenceProperties;+ use datafusion_physical_expr::expressions::Column;+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;++ // File schema is just `[a]`; `b` lives as a partition column on+ // top, so it appears in the table schema but not the file schema+ // — the same shape `column_in_file_schema` discards.+ let file_schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));+ let partition_b = Arc::new(Field::new("b", DataType::Int32, true));+ let table_schema = TableSchema::new(file_schema, vec![partition_b]);+ let source = ParquetSource::new(table_schema);++ // EquivalenceProperties is built on the *full* table schema so+ // it can carry an ordering on `b`.+ let full_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]));+ // Construct the request first, then derive the declared+ // ordering as its reverse, so `ordering_satisfy` on the+ // reversed-eq holds exactly. `PhysicalSortExpr::reverse` flips+ // both `descending` and `nulls_first`, so spelling the+ // declared ordering directly is error-prone.+ let request_expr = PhysicalSortExpr {+ expr: Arc::new(Column::new("b", 1)),+ options: SortOptions {+ descending: true,+ nulls_first: true,+ },+ };+ let declared = request_expr.reverse();+ let mut eq = EquivalenceProperties::new(Arc::clone(&full_schema));+ eq.add_ordering(vec![declared]);+ let order = vec![request_expr];++ let result = source.try_pushdown_sort(&order, &eq).unwrap();++ let SortOrderPushdownResult::Inexact { inner } = result else {+ panic!("expected Inexact, got a different variant");+ };+ let inner_parquet = inner+ .downcast_ref::<ParquetSource>()+ .expect("inner is ParquetSource");+ assert!(+ inner_parquet.sort_order_for_reorder.is_some(),+ "sort_order_for_reorder must be set so EXPLAIN reflects the request",+ );+ assert!(+ inner_parquet.reverse_row_groups(),+ "request reached via reversed_satisfies (column-not-in-file-schema) \+ must set reverse_row_groups to flip the file's natural order",+ );+ }++ /// Regression: when the source's declared ordering is a non-empty+ /// *proper* prefix of the request, `try_pushdown_sort` must return+ /// `Unsupported` so that the outer `SortExec` can keep its+ /// `sort_prefix` annotation and `TopK` can early-terminate within+ /// each prefix block. Letting the Phase 3 Inexact pipeline fire+ /// would drop the source's `output_ordering`, destroying the+ /// information `EnforceSorting` needs to compute `sort_prefix`.+ #[test]+ fn try_pushdown_sort_preserves_sort_prefix_when_source_declares_prefix_ordering() {+ use arrow::compute::SortOptions;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_physical_expr::EquivalenceProperties;+ use datafusion_physical_expr::expressions::Column;+ use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ Field::new("c", DataType::Int32, true),+ ]));+ let source = ParquetSource::new(Arc::clone(&schema));++ // Source declares `[a DESC, b ASC NULLS LAST]` — the same prefix+ // the SortExec input will see.+ let mut eq = EquivalenceProperties::new(Arc::clone(&schema));+ eq.add_ordering(vec![+ PhysicalSortExpr {+ expr: Arc::new(Column::new("a", 0)),+ options: SortOptions {+ descending: true,+ nulls_first: true,+ },+ },+ PhysicalSortExpr {+ expr: Arc::new(Column::new("b", 1)),+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ },+ ]);++ // Request `[a DESC, b ASC NULLS LAST, c DESC]` — three columns;+ // source's two-column declaration is a strict prefix.+ let order = vec![+ PhysicalSortExpr {+ expr: Arc::new(Column::new("a", 0)),+ options: SortOptions {+ descending: true,+ nulls_first: true,+ },+ },+ PhysicalSortExpr {+ expr: Arc::new(Column::new("b", 1)),+ options: SortOptions {+ descending: false,+ nulls_first: false,+ },+ },+ PhysicalSortExpr {+ expr: Arc::new(Column::new("c", 2)),+ options: SortOptions {+ descending: true,+ nulls_first: true,+ },+ },+ ];++ let result = source.try_pushdown_sort(&order, &eq).unwrap();+ assert!(+ matches!(result, SortOrderPushdownResult::Unsupported),+ "source ordering [a DESC, b ASC NULLS LAST] is a proper prefix \+ of the request — `try_pushdown_sort` must return Unsupported so \+ the SortExec sort_prefix optimisation can fire",+ );+ }++ /// Helpers for the `reorder_files` regression tests below.+ mod reorder_files_helpers {+ use super::*;+ use datafusion_common::stats::Precision;+ use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};+ use datafusion_datasource::PartitionedFile;++ pub(super) fn file_with_min(name: &str, min: Option<i32>) -> PartitionedFile {+ let mut pf = PartitionedFile::new(name.to_string(), 0);+ let min_value = min+ .map(|v| Precision::Exact(ScalarValue::Int32(Some(v))))+ .unwrap_or(Precision::Absent);+ pf.statistics = Some(Arc::new(Statistics {+ num_rows: Precision::Absent,+ total_byte_size: Precision::Absent,+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Absent,+ max_value: Precision::Absent,+ min_value,+ sum_value: Precision::Absent,+ distinct_count: Precision::Absent,+ byte_size: Precision::Absent,+ }],+ }));+ pf+ }++ pub(super) fn names(files: &[PartitionedFile]) -> Vec<&str> {+ files+ .iter()+ .map(|f| f.object_meta.location.as_ref())+ .collect()+ }+ }++ /// ASC TopK: `reorder_files` keys off file `min` and sorts ASC,+ /// so the file with the smallest `min` is read first. This+ /// matches `PreparedAccessPlan::reorder_by_statistics` at the+ /// row-group level (also `min ASC`).+ #[test]+ fn reorder_files_sorts_asc_by_min_for_asc_request() {+ use pushdown_sort_helpers::*;+ use reorder_files_helpers::*;++ let schema = schema_with_a_int();+ let mut source = ParquetSource::new(Arc::clone(&schema));+ source.sort_order_for_reorder =+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap());+ // ASC request → `reverse_row_groups` left at its default `false`.++ let reordered = source.reorder_files(vec![+ file_with_min("middle", Some(50)),+ file_with_min("small", Some(10)),+ file_with_min("large", Some(100)),+ ]);++ assert_eq!(names(&reordered), vec!["small", "middle", "large"]);+ }++ /// DESC TopK: same `min` key, but sorted DESC — file with the+ /// largest `min` first. Mirrors the row-group strategy of+ /// "ASC-by-min then `reverse`".+ #[test]+ fn reorder_files_sorts_desc_by_min_for_desc_request() {+ use pushdown_sort_helpers::*;+ use reorder_files_helpers::*;++ let schema = schema_with_a_int();+ let mut source =+ ParquetSource::new(Arc::clone(&schema)).with_reverse_row_groups(true);+ source.sort_order_for_reorder =+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", true)]).unwrap());++ let reordered = source.reorder_files(vec![+ file_with_min("middle", Some(50)),+ file_with_min("small", Some(10)),+ file_with_min("large", Some(100)),+ ]);++ assert_eq!(names(&reordered), vec!["large", "middle", "small"]);+ }++ /// Files without statistics sort to the *end* so present-stats+ /// files run first regardless of direction. Verified for both+ /// ASC and DESC.+ #[test]+ fn reorder_files_pushes_missing_stats_to_the_end() {+ use pushdown_sort_helpers::*;+ use reorder_files_helpers::*;++ let schema = schema_with_a_int();+ let mut source = ParquetSource::new(Arc::clone(&schema));+ source.sort_order_for_reorder =+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap());++ let reordered = source.reorder_files(vec![+ file_with_min("no_stats", None),+ file_with_min("has_min", Some(10)),+ ]);+ assert_eq!(names(&reordered), vec!["has_min", "no_stats"]);++ // Same for DESC.+ let mut source =+ ParquetSource::new(Arc::clone(&schema)).with_reverse_row_groups(true);+ source.sort_order_for_reorder =+ Some(LexOrdering::new(vec![sort_expr_on(&schema, "a", true)]).unwrap());+ let reordered = source.reorder_files(vec![+ file_with_min("no_stats", None),+ file_with_min("has_min", Some(10)),+ ]);+ assert_eq!(names(&reordered), vec!["has_min", "no_stats"]);+ }++ /// When no sort pushdown has fired (`sort_order_for_reorder` is+ /// `None`), `reorder_files` is a no-op and preserves input order.+ #[test]+ fn reorder_files_is_a_no_op_without_pushdown() {+ use pushdown_sort_helpers::*;+ use reorder_files_helpers::*;++ let schema = schema_with_a_int();+ let source = ParquetSource::new(schema);+ // No `sort_order_for_reorder` set on the source.++ let input = vec![+ file_with_min("c", Some(30)),+ file_with_min("a", Some(10)),+ file_with_min("b", Some(20)),+ ];+ let reordered = source.reorder_files(input.clone());+ assert_eq!(names(&reordered), names(&input));+ }++ /// `sort_order_for_reorder` is surfaced in both `EXPLAIN` (Default)+ /// and `EXPLAIN VERBOSE` / `EXPLAIN ANALYZE` (Verbose) so readers+ /// and snapshot tests can see the inexact sort-pushdown fired.+ #[test]+ fn sort_order_for_reorder_shown_in_explain() {+ use pushdown_sort_helpers::*;++ // `std::fmt::Formatter` can't be constructed outside core fmt+ // machinery, so we drive `fmt_extra` through a Display adapter+ // and read the rendered string back with `format!`.+ struct DisplayHelper<'a> {+ source: &'a ParquetSource,+ mode: DisplayFormatType,+ }+ impl std::fmt::Display for DisplayHelper<'_> {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ self.source.fmt_extra(self.mode, f)+ }+ }++ let schema = schema_with_a_int();+ let mut source = ParquetSource::new(Arc::clone(&schema));+ let order = LexOrdering::new(vec![sort_expr_on(&schema, "a", false)]).unwrap();+ source.sort_order_for_reorder = Some(order);++ for mode in [DisplayFormatType::Default, DisplayFormatType::Verbose] {+ let out = format!(+ "{}",+ DisplayHelper {+ source: &source,+ mode,+ },+ );+ assert!(+ out.contains("sort_order_for_reorder=[a@0 ASC]"),+ "{mode:?} display must surface sort_order_for_reorder, got: {out}",+ );+ }+ }}diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex 9b4ae5827a..32bee63b54 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -280,6 +280,19 @@ pub trait FileSource: Any + Send + Sync {Ok(SortOrderPushdownResult::Unsupported)}+ /// Reorder files in the shared work queue to optimize query performance.+ ///+ /// For example, TopK queries benefit from reading files with the best+ /// statistics first, so the dynamic filter threshold tightens quickly.+ ///+ /// The default implementation returns files unchanged (no reordering).+ fn reorder_files(+ &self,+ files: Vec<crate::PartitionedFile>,+ ) -> Vec<crate::PartitionedFile> {+ files+ }+/// Try to push down a projection into this FileSource.////// `FileSource` implementations that support projection pushdown shoulddiff --git a/datafusion/datasource/src/file_stream/work_source.rs b/datafusion/datasource/src/file_stream/work_source.rsindex 7f31dacca9..c00048453b 100644--- a/datafusion/datasource/src/file_stream/work_source.rs+++ b/datafusion/datasource/src/file_stream/work_source.rs@@ -85,8 +85,19 @@ impl SharedWorkSource {}/// Create a shared work source for the unopened files in `config`.+ ///+ /// Files are reordered by the file source (e.g. by statistics for TopK)+ /// before being placed in the shared queue, so the most promising files+ /// are processed first across all partitions.pub(crate) fn from_config(config: &FileScanConfig) -> Self {- Self::new(config.file_groups.iter().flat_map(FileGroup::iter).cloned())+ let files: Vec<_> = config+ .file_groups+ .iter()+ .flat_map(FileGroup::iter)+ .cloned()+ .collect();+ let files = config.file_source.reorder_files(files);+ Self::new(files)}/// Pop the next file from the shared work queue.diff --git a/datafusion/sqllogictest/test_files/create_external_table.slt b/datafusion/sqllogictest/test_files/create_external_table.sltindex 0b15a7f8ec..f56cff2a2a 100644--- a/datafusion/sqllogictest/test_files/create_external_table.slt+++ b/datafusion/sqllogictest/test_files/create_external_table.slt@@ -264,7 +264,7 @@ logical_plan02)--TableScan: t projection=[id]physical_plan01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], file_type=parquet, reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=truestatement okDROP TABLE t;diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex befc443303..e779ce2cbf 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -90,7 +90,7 @@ logical_plan02)--TableScan: test_parquet projection=[id, value, name]physical_plan01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[value@1 DESC], reverse_row_groups=truestatement okset datafusion.explain.analyze_level = summary;@@ -128,7 +128,7 @@ logical_plan02)--TableScan: test_parquet projection=[id, value, name]physical_plan01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, sort_order_for_reorder=[value@1 DESC], reverse_row_groups=truequery IITSELECT id, value AS v, name FROM (SELECT * FROM test_parquet UNION ALL SELECT * FROM test_parquet) ORDER BY v DESC LIMIT 3;@@ -338,7 +338,7 @@ physical_plan02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]04)----FilterExec: DynamicFilter [ empty ]-05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[data@1 DESC], reverse_row_groups=truestatement count 0SET datafusion.execution.parquet.pushdown_filters = true;@@ -361,7 +361,7 @@ physical_plan01)HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(id@0, id@0)]02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id], file_type=parquet03)--SortExec: expr=[data@1 DESC], preserve_partitioning=[false]-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[data@1 DESC], reverse_row_groups=truestatement count 0RESET datafusion.execution.parquet.pushdown_filters;diff --git a/datafusion/sqllogictest/test_files/explain_analyze.slt b/datafusion/sqllogictest/test_files/explain_analyze.sltindex cad36d4fa6..7460148bab 100644--- a/datafusion/sqllogictest/test_files/explain_analyze.slt+++ b/datafusion/sqllogictest/test_files/explain_analyze.slt@@ -231,7 +231,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[]statement okreset datafusion.explain.analyze_categories;@@ -247,7 +247,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, scan_efficiency_ratio=22.13% (521/2.35 K)]statement okreset datafusion.explain.analyze_categories;@@ -262,7 +262,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]statement okreset datafusion.explain.analyze_categories;@@ -277,7 +277,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, scan_efficiency_ratio=<slt:ignore>]statement okreset datafusion.explain.analyze_categories;@@ -292,7 +292,7 @@ explain analyze select * from cat_tracking where species > 'M' AND s >= 50 order----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[elapsed_compute=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[elapsed_compute=<slt:ignore>, metadata_load_time=<slt:ignore>]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/explain_analyze/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[elapsed_compute=<slt:ignore>, metadata_load_time=<slt:ignore>]statement okreset datafusion.explain.analyze_categories;diff --git a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt b/datafusion/sqllogictest/test_files/join_limit_pushdown.sltindex ea9c971065..933b03e7eb 100644--- a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt+++ b/datafusion/sqllogictest/test_files/join_limit_pushdown.slt@@ -259,7 +259,7 @@ SELECT t1.a, t2.x FROM t1 INNER JOIN t2 ON t1.a = t2.x LIMIT 100;# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okset datafusion.execution.target_partitions = 4;diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.sltindex 8e931ca52f..fc62584dc3 100644--- a/datafusion/sqllogictest/test_files/limit.slt+++ b/datafusion/sqllogictest/test_files/limit.slt@@ -868,7 +868,7 @@ physical_plan01)ProjectionExec: expr=[1 as foo]02)--SortPreservingMergeExec: [part_key@0 ASC NULLS LAST], fetch=103)----SortExec: TopK(fetch=1), expr=[part_key@0 ASC NULLS LAST], preserve_partitioning=[true]-04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-2.parquet]]}, projection=[part_key], file_type=parquet, predicate=DynamicFilter [ empty ]+04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/test_limit_with_partitions/part-2.parquet]]}, projection=[part_key], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[part_key@0 ASC NULLS LAST]query Iwith selection as (diff --git a/datafusion/sqllogictest/test_files/limit_pruning.slt b/datafusion/sqllogictest/test_files/limit_pruning.sltindex 676306e1f0..373e1636a2 100644--- a/datafusion/sqllogictest/test_files/limit_pruning.slt+++ b/datafusion/sqllogictest/test_files/limit_pruning.slt@@ -120,7 +120,7 @@ explain analyze select * from tracking_data where species > 'M' AND s >= 50 orde----Plan with Metrics01)SortExec: TopK(fetch=3), expr=[species@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[species@0 < Nlpine Sheep], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit_pruning/data.parquet]]}, projection=[species, s], file_type=parquet, predicate=species@0 > M AND s@1 >= 50 AND DynamicFilter [ species@0 < Nlpine Sheep ], sort_order_for_reorder=[species@0 ASC NULLS LAST], pruning_predicate=species_null_count@1 != row_count@2 AND species_max@0 > M AND s_null_count@4 != row_count@2 AND s_max@3 >= 50 AND species_null_count@1 != row_count@2 AND species_min@5 < Nlpine Sheep, required_guarantees=[], metrics=[output_rows=3, elapsed_compute=<slt:ignore>, output_bytes=<slt:ignore>, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=4 total → 3 matched -> 1 fully matched, row_groups_pruned_bloom_filter=3 total → 3 matched, page_index_pages_pruned=2 total → 2 matched, page_index_pages_skipped_by_fully_matched=1, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=<slt:ignore>, metadata_load_time=<slt:ignore>, scan_efficiency_ratio=<slt:ignore> (521/2.35 K)]statement okdrop table tracking_data;diff --git a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt b/datafusion/sqllogictest/test_files/limit_single_row_batches.sltindex 6b5368620c..441adce9a3 100644--- a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt+++ b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt@@ -22,7 +22,7 @@ SELECT COUNT(*) FROM (SELECT i FROM filter_limit WHERE i <> 0 LIMIT 1);statement okreset datafusion.execution.batch_size;-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okset datafusion.execution.target_partitions = 4;diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.sltindex 781d0b00a5..8026e98d86 100644--- a/datafusion/sqllogictest/test_files/parquet.slt+++ b/datafusion/sqllogictest/test_files/parquet.slt@@ -95,7 +95,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST]02)--SortExec: expr=[string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/1.parquet]]}, projection=[int_col, string_col], file_type=parquet+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_table/1.parquet]]}, projection=[int_col, string_col], file_type=parquet, sort_order_for_reorder=[string_col@1 ASC NULLS LAST, int_col@0 ASC NULLS LAST]# Tear down test_table:statement ok@@ -889,7 +889,7 @@ WHERE b = 2;# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okset datafusion.execution.target_partitions = 4;diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.sltindex 85f9549357..cb3be93191 100644--- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt+++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt@@ -110,7 +110,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]query Tselect a from t where b = 2 ORDER BY b;@@ -181,7 +181,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]query TTEXPLAIN select a from t_pushdown where b > 2 ORDER BY a;@@ -194,7 +194,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]query Tselect a from t where b = 2 ORDER BY b;@@ -278,7 +278,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[]query Tselect a from t where b = 2 ORDER BY b;@@ -377,7 +377,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [a@0 ASC NULLS LAST]02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2 AND a@0 IS NOT NULL, sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2 AND a_null_count@3 != row_count@2, required_guarantees=[]query I@@ -396,7 +396,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [b@0 ASC NULLS LAST]02)--SortExec: expr=[b@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[b], file_type=parquet, predicate=a@0 = bar, sort_order_for_reorder=[b@0 ASC NULLS LAST], pruning_predicate=a_null_count@2 != row_count@3 AND a_min@0 <= bar AND bar <= a_max@1, required_guarantees=[a in (bar)]# should not push down volatile predicates such as RANDOM@@ -603,7 +603,7 @@ logical_plan03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has(array_test.tags, Utf8("rust"))]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust)+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has(tags@1, rust), sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test array_has_all predicate pushdownquery I?@@ -620,7 +620,7 @@ logical_plan03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust, performance]))]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance])+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust, performance]), sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test array_has_any predicate pushdownquery I?@@ -637,7 +637,7 @@ logical_plan03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_any(array_test.tags, List([python, go]))]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go])+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_any(tags@1, [python, go]), sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test complex predicate with ORquery I?@@ -656,7 +656,7 @@ logical_plan03)----TableScan: array_test projection=[id, tags], partial_filters=[array_has_all(array_test.tags, List([rust])) OR array_has_any(array_test.tags, List([python, go]))]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go])+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=array_has_all(tags@1, [rust]) OR array_has_any(tags@1, [python, go]), sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test array function with other predicatesquery I?@@ -673,7 +673,7 @@ logical_plan03)----TableScan: array_test projection=[id, tags], partial_filters=[array_test.id > Int64(1), array_has(array_test.tags, Utf8("rust"))]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/array_data/data.parquet]]}, projection=[id, tags], file_type=parquet, predicate=id@0 > 1 AND array_has(tags@1, rust), sort_order_for_reorder=[id@0 ASC NULLS LAST], pruning_predicate=id_null_count@1 != row_count@2 AND id_max@0 > 1, required_guarantees=[]#### Test filter pushdown through UNION with mixed supportdiff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.sltindex a4a613e383..b1e76a555d 100644--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt@@ -278,7 +278,7 @@ physical_plan# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okset datafusion.execution.target_partitions = 4;diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex a40c1b8c7e..344aef1f92 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -320,7 +320,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -345,7 +345,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -413,7 +413,7 @@ logical_plan03)----TableScan: three_cols projection=[col_a, col_b, col_c]physical_plan01)SortExec: expr=[col_a@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c, col_b@1 as col_b_dup], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/three_cols.parquet]]}, projection=[col_a, col_b, col_c, col_b@1 as col_b_dup], file_type=parquet, sort_order_for_reorder=[col_a@0 ASC NULLS LAST]# Verify correctnessquery IIII@@ -444,7 +444,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -467,7 +467,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 1 as simple_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -490,7 +490,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery IIT@@ -513,7 +513,7 @@ logical_plan03)----TableScan: nested_struct projection=[id, nested]physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/nested.parquet]]}, projection=[id, get_field(nested@1, outer, inner) as nested_struct.nested[outer][inner]], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/nested.parquet]]}, projection=[id, get_field(nested@1, outer, inner) as nested_struct.nested[outer][inner]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -535,7 +535,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, label) || _suffix as simple_struct.s[label] || Utf8("_suffix")], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, label) || _suffix as simple_struct.s[label] || Utf8("_suffix")], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery IT@@ -637,7 +637,7 @@ SELECT id, s['value'] + 1 FROM simple_struct WHERE id > 1 ORDER BY id LIMIT 2;# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okSET datafusion.execution.target_partitions = 4;@@ -687,7 +687,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -713,7 +713,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=302)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ]+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) as multi_struct.s[value]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -737,7 +737,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=302)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) + 1 as multi_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ]+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part3.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part4.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/multi/part5.parquet]]}, projection=[id, get_field(s@1, value) + 1 as multi_struct.s[value] + Int64(1)], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -874,7 +874,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, value) + 10 as simple_struct.s[value] + Int64(10), get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, value) + 10 as simple_struct.s[value] + Int64(10), get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery IIIT@@ -897,7 +897,7 @@ logical_plan03)----TableScan: simple_struct projection=[id]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as constant], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as constant], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -919,7 +919,7 @@ logical_plan02)--TableScan: simple_struct projection=[id]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery I@@ -947,7 +947,7 @@ logical_plan03)----TableScan: simple_struct projection=[id]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, id@0 + 100 as computed], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, id@0 + 100 as computed], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -1039,7 +1039,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + id@0 as combined], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + id@0 as combined], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -1095,7 +1095,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as answer, get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, 42 as answer, get_field(s@1, label) as simple_struct.s[label]], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery IIT@@ -1118,7 +1118,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: TopK(fetch=2), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 100 as simple_struct.s[value] + Int64(100), get_field(s@1, label) || _test as simple_struct.s[label] || Utf8("_test")], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) + 100 as simple_struct.s[value] + Int64(100), get_field(s@1, label) || _test as simple_struct.s[label] || Utf8("_test")], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery IIT@@ -1288,7 +1288,7 @@ logical_planphysical_plan01)ProjectionExec: expr=[id@0 as id, simple_struct.s[value]@1 as simple_struct.s[value]]02)--SortExec: expr=[id@0 ASC NULLS LAST, __datafusion_extracted_1@2 ASC NULLS LAST], preserve_partitioning=[false]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as __datafusion_extracted_1], file_type=parquet+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value], get_field(s@1, label) as __datafusion_extracted_1], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST, __datafusion_extracted_1@2 ASC NULLS LAST]# Verify correctnessquery II@@ -1369,7 +1369,7 @@ logical_plan03)----TableScan: simple_struct projection=[id, s]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST, simple_struct.s[value]@1 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[id, get_field(s@1, value) as simple_struct.s[value]], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST, simple_struct.s[value]@1 ASC NULLS LAST]# Verify correctnessquery IIdiff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.sltindex 8469c32a17..c2fae88903 100644--- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt+++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt@@ -257,7 +257,7 @@ EXPLAIN SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;----physical_plan01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=truestatement okset datafusion.explain.analyze_categories = 'rows';@@ -268,7 +268,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_single_col ORDER BY b DESC LIMIT 1;----Plan with Metrics01)SortExec: TopK(fetch=1), expr=[b@1 DESC], preserve_partitioning=[false], filter=[b@1 IS NULL OR b@1 > bd], metrics=[output_rows=1, output_batches=1, row_replacements=1]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=22.37% (240/1.07 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_single_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 IS NULL OR b@1 > bd ], sort_order_for_reorder=[b@1 DESC], reverse_row_groups=true, pruning_predicate=b_null_count@0 > 0 OR b_null_count@0 != row_count@2 AND b_max@1 > bd, required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=4, predicate_cache_records=4, scan_efficiency_ratio=22.37% (240/1.07 K)]statement okreset datafusion.explain.analyze_categories;@@ -319,7 +319,7 @@ EXPLAIN ANALYZE SELECT * FROM topk_multi_col ORDER BY b ASC NULLS LAST, a DESC L----Plan with Metrics01)SortExec: TopK(fetch=2), expr=[b@1 ASC NULLS LAST, a@0 DESC], preserve_partitioning=[false], filter=[b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac)], metrics=[output_rows=2, output_batches=1, row_replacements=2]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=22.37% (240/1.07 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_multi_col.parquet]]}, projection=[a, b, c], file_type=parquet, predicate=DynamicFilter [ b@1 < bb OR b@1 = bb AND (a@0 IS NULL OR a@0 > ac) ], sort_order_for_reorder=[b@1 ASC NULLS LAST, a@0 DESC], pruning_predicate=b_null_count@1 != row_count@2 AND b_min@0 < bb OR b_null_count@1 != row_count@2 AND b_min@0 <= bb AND bb <= b_max@3 AND (a_null_count@4 > 0 OR a_null_count@4 != row_count@2 AND a_max@5 > ac), required_guarantees=[], metrics=[output_rows=4, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=4, pushdown_rows_pruned=0, predicate_cache_inner_records=8, predicate_cache_records=8, scan_efficiency_ratio=22.37% (240/1.07 K)]statement okreset datafusion.explain.analyze_categories;@@ -655,7 +655,7 @@ EXPLAIN ANALYZE SELECT b, a FROM topk_proj ORDER BY a LIMIT 2;----Plan with Metrics01)SortExec: TopK(fetch=2), expr=[a@1 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@1 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[b, a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@1 ASC NULLS LAST], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]# Case 2: prune — `SELECT a` — filter stays as `a < 2` on the scan.query TT@@ -663,7 +663,7 @@ EXPLAIN ANALYZE SELECT a FROM topk_proj ORDER BY a LIMIT 2;----Plan with Metrics01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 2], metrics=[output_rows=2, output_batches=1, row_replacements=2]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=7.09% (79/1.11 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[a], file_type=parquet, predicate=DynamicFilter [ a@0 < 2 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], pruning_predicate=a_null_count@1 != row_count@2 AND a_min@0 < 2, required_guarantees=[], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=7.09% (79/1.11 K)]# Case 3: expression — `SELECT a+1 AS a_plus_1` — the TopK filter is on# `a_plus_1`, the scan predicate must read `a@0 + 1`.@@ -682,7 +682,7 @@ EXPLAIN ANALYZE SELECT a + 1 AS a, b FROM topk_proj ORDER BY a LIMIT 2;----Plan with Metrics01)SortExec: TopK(fetch=2), expr=[a@0 ASC NULLS LAST], preserve_partitioning=[false], filter=[a@0 < 3], metrics=[output_rows=2, output_batches=1, row_replacements=2]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/topk_proj.parquet]]}, projection=[CAST(a@0 AS Int64) + 1 as a, b], file_type=parquet, predicate=DynamicFilter [ CAST(a@0 AS Int64) + 1 < 3 ], sort_order_for_reorder=[a@0 ASC NULLS LAST], metrics=[output_rows=3, output_batches=1, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=0 total → 0 matched, page_index_rows_pruned=0 total → 0 matched, limit_pruned_row_groups=0 total → 0 matched, batches_split=0, file_open_errors=0, file_scan_errors=0, files_opened=1, files_processed=1, num_predicate_creation_errors=0, predicate_evaluation_errors=0, pushdown_rows_matched=3, pushdown_rows_pruned=0, predicate_cache_inner_records=3, predicate_cache_records=3, scan_efficiency_ratio=13.72% (153/1.11 K)]statement okreset datafusion.explain.analyze_categories;diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltindex b6c75f3977..540562eb3b 100644--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -43,7 +43,7 @@ logical_plan02)--TableScan: sorted_parquet projection=[id, value, name]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true# Test 1.2: Verify results are correctquery IIT@@ -91,7 +91,7 @@ logical_planphysical_plan01)GlobalLimitExec: skip=2, fetch=302)--SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=truequery IITSELECT * FROM sorted_parquet ORDER BY id DESC LIMIT 3 OFFSET 2;@@ -155,7 +155,7 @@ logical_plan03)----TableScan: multi_rg_sorted projection=[id, category, value], partial_filters=[multi_rg_sorted.category = Utf8View("alpha") OR multi_rg_sorted.category = Utf8View("gamma")]physical_plan01)SortExec: TopK(fetch=5), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_rg_sorted.parquet]]}, projection=[id, category, value], file_type=parquet, predicate=(category@1 = alpha OR category@1 = gamma) AND DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true, pruning_predicate=category_null_count@2 != row_count@3 AND category_min@0 <= alpha AND alpha <= category_max@1 OR category_null_count@2 != row_count@3 AND category_min@0 <= gamma AND gamma <= category_max@1, required_guarantees=[category in (alpha, gamma)]# Verify the results are correct despite reverse scanning with row selection# Expected: gamma values (6, 5) then alpha values (2, 1), in DESC order by id@@ -272,7 +272,7 @@ logical_planphysical_plan01)SortPreservingMergeExec: [id@0 DESC], fetch=302)--SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[true]-03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/sorted_multi/part3.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true# Verify correctness with repartitioning and multiple filesquery IIT@@ -381,7 +381,7 @@ logical_plan03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("quarterly")]physical_plan01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = quarterly AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= quarterly AND quarterly <= timeframe_max@1, required_guarantees=[timeframe in (quarterly)]# Test 2.2: Verify the results are correctquery TIR@@ -491,7 +491,7 @@ logical_plan03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("daily") OR timeseries_parquet.timeframe = Utf8View("weekly")]physical_plan01)SortExec: TopK(fetch=3), expr=[period_end@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], output_ordering=[timeframe@0 ASC NULLS LAST, period_end@1 ASC NULLS LAST], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=(timeframe@0 = daily OR timeframe@0 = weekly) AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= daily AND daily <= timeframe_max@1 OR timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= weekly AND weekly <= timeframe_max@1, required_guarantees=[timeframe in (daily, weekly)]# Test 2.9: Complex case - literal constant in sort expression itself# The literal 'constant' is ignored in sort analysis@@ -511,7 +511,7 @@ logical_plan03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.timeframe = Utf8View("monthly")]physical_plan01)SortExec: TopK(fetch=2), expr=[period_end@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=timeframe@0 = monthly AND DynamicFilter [ empty ], sort_order_for_reorder=[period_end@1 DESC], reverse_row_groups=true, pruning_predicate=timeframe_null_count@2 != row_count@3 AND timeframe_min@0 <= monthly AND monthly <= timeframe_max@1, required_guarantees=[timeframe in (monthly)]# Verify resultsquery TIR@@ -540,7 +540,7 @@ logical_plan03)----TableScan: timeseries_parquet projection=[timeframe, period_end, value], partial_filters=[timeseries_parquet.period_end = Int32(2)]physical_plan01)SortExec: expr=[timeframe@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timeseries_sorted.parquet]]}, projection=[timeframe, period_end, value], file_type=parquet, predicate=period_end@1 = 2, sort_order_for_reorder=[timeframe@0 DESC], reverse_row_groups=true, pruning_predicate=period_end_null_count@2 != row_count@3 AND period_end_min@0 <= 2 AND 2 <= period_end_max@1, required_guarantees=[period_end in (2)]# Cleanupstatement ok@@ -600,7 +600,7 @@ logical_plan02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]physical_plan01)SortExec: TopK(fetch=3), expr=[ts@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ts@1 DESC], reverse_row_groups=true# Verify resultsquery IPIR@@ -626,7 +626,7 @@ logical_plan02)--TableScan: timestamp_parquet projection=[id, ts, volume, price]physical_plan01)SortExec: TopK(fetch=3), expr=[date_trunc(day, ts@1) DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/timestamp_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[date_trunc(day, ts@1) DESC], reverse_row_groups=true# Verify results (descending day)query IPIR@@ -686,7 +686,7 @@ logical_plan02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]physical_plan01)SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ts@1 DESC], reverse_row_groups=truequery IPIRSELECT * FROM multi_month_parquet@@ -712,7 +712,7 @@ logical_plan02)--TableScan: multi_month_parquet projection=[id, ts, volume, price]physical_plan01)SortExec: TopK(fetch=2), expr=[date_trunc(month, ts@1) DESC, ts@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_month_sorted.parquet]]}, projection=[id, ts, volume, price], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[date_trunc(month, ts@1) DESC, ts@1 DESC], reverse_row_groups=truequery IPIRSELECT * FROM multi_month_parquet@@ -754,7 +754,7 @@ logical_plan02)--TableScan: int_parquet projection=[id, small_val, big_val]physical_plan01)SortExec: TopK(fetch=2), expr=[CAST(small_val@1 AS Int64) DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/int_sorted.parquet]]}, projection=[id, small_val, big_val], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[CAST(small_val@1 AS Int64) DESC], reverse_row_groups=truequery IIISELECT * FROM int_parquet@@ -796,7 +796,7 @@ logical_plan02)--TableScan: float_parquet projection=[id, value]physical_plan01)SortExec: TopK(fetch=3), expr=[ceil(value@1) DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/float_sorted.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[ceil(value@1) DESC], reverse_row_groups=truequery IRSELECT * FROM float_parquet@@ -1109,7 +1109,9 @@ EXPLAIN SELECT * FROM reversed_parquet ORDER BY id ASC;logical_plan01)Sort: reversed_parquet.id ASC NULLS LAST02)--TableScan: reversed_parquet projection=[id, value]-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test 4.2: Results must be correctquery II@@ -1160,7 +1162,7 @@ logical_plan02)--TableScan: overlap_parquet projection=[id, value]physical_plan01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_y.parquet]]}, projection=[id, value], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_y.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test 5.2: Results must be correctquery II@@ -1192,7 +1194,9 @@ EXPLAIN SELECT * FROM reversed_with_order_parquet ORDER BY id ASC;logical_plan01)Sort: reversed_with_order_parquet.id ASC NULLS LAST02)--TableScan: reversed_with_order_parquet projection=[id, value]-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Test 6.2: Results must be correctquery II@@ -1278,7 +1282,7 @@ logical_plan02)--TableScan: correct_parquet projection=[id, value]physical_plan01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet]]}, projection=[id, value], file_type=parquet, reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=truequery IISELECT * FROM correct_parquet ORDER BY id DESC;@@ -1329,7 +1333,9 @@ EXPLAIN SELECT * FROM desc_reversed_parquet ORDER BY id DESC;logical_plan01)Sort: desc_reversed_parquet.id DESC NULLS FIRST02)--TableScan: desc_reversed_parquet projection=[id, value]-physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet]]}, projection=[id, value], output_ordering=[id@0 DESC], file_type=parquet+physical_plan+01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true# Test 8.2: Results must be correctquery II@@ -1384,7 +1390,7 @@ logical_plan02)--TableScan: multi_col_parquet projection=[category, id, value]physical_plan01)SortExec: expr=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/a_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/b_second.parquet]]}, projection=[category, id, value], file_type=parquet+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/a_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/b_second.parquet]]}, projection=[category, id, value], file_type=parquet, sort_order_for_reorder=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST]# Test 9.2: Results must be correctquery TII@@ -1519,7 +1525,7 @@ SELECT * FROM multi_partition_parquet ORDER BY id ASC;# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okSET datafusion.execution.target_partitions = 4;@@ -1772,7 +1778,7 @@ logical_plan02)--TableScan: tb_overlap projection=[id, value]physical_plan01)SortExec: TopK(fetch=5), expr=[id@0 DESC, value@1 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC, value@1 DESC], reverse_row_groups=truequery IISELECT * FROM tb_overlap ORDER BY id DESC, value DESC LIMIT 5;@@ -1857,7 +1863,7 @@ logical_plan02)--TableScan: tc_limit projection=[id, value]physical_plan01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=truequery IISELECT * FROM tc_limit ORDER BY id DESC LIMIT 3;@@ -2221,8 +2227,8 @@ logical_plan02)--TableScan: tg_buffer projection=[id, value]physical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]-02)--BufferExec: capacity=1073741824-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], file_type=parquet, sort_order_for_reorder=[id@0 ASC NULLS LAST]# Verify correctnessquery II@@ -2248,8 +2254,8 @@ logical_plan02)--TableScan: tg_buffer projection=[id, value]physical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST], fetch=3-02)--BufferExec: capacity=1073741824-03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+02)--SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tg_buffer/c_low.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]query IISELECT * FROM tg_buffer ORDER BY id ASC LIMIT 3;@@ -2280,3 +2286,466 @@ SET datafusion.execution.collect_statistics = true;statement okSET datafusion.optimizer.enable_sort_pushdown = true;++# ===========================================================+# Test H: Row group reorder by statistics for TopK queries.+# When a file has multiple row groups with overlapping or+# out-of-order statistics, sort pushdown returns Inexact and+# `reorder_by_statistics` reorders row groups within the file+# so TopK finds optimal values first.+# ===========================================================++# Create a table with 9 rows and write to parquet with small row groups+# so we get multiple row groups per file. Rows are inserted in a mixed+# order so row groups span overlapping ranges (forcing Inexact path).+statement ok+CREATE TABLE th_mixed(id INT, value INT) AS VALUES+ (15, 150), (5, 50), (25, 250),+ (10, 100), (20, 200), (1, 10),+ (30, 300), (3, 30), (18, 180);++# Write with row_group_size=3 → 3 rows per RG, 3 RGs total+# RG statistics (unsorted order): RG0(5-25), RG1(1-20), RG2(3-30)+# Note: files are overlapping → Inexact path → TopK retained+query I+COPY (SELECT * FROM th_mixed)+TO 'test_files/scratch/sort_pushdown/th_reorder/data.parquet'+STORED AS PARQUET+OPTIONS ('format.max_row_group_size' '3');+----+9++statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+CREATE EXTERNAL TABLE th_reorder(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/th_reorder/data.parquet';++# Test H.1: ASC ORDER BY with LIMIT — Inexact path (file has no declared ordering)+# Plan: SortExec(TopK) preserved. RG reorder happens inside DataSourceExec+# (not visible in EXPLAIN, but verified by unit tests in access_plan.rs).+query TT+EXPLAIN SELECT * FROM th_reorder ORDER BY id ASC LIMIT 3;+----+logical_plan+01)Sort: th_reorder.id ASC NULLS LAST, fetch=3+02)--TableScan: th_reorder projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/th_reorder/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 ASC NULLS LAST]++# Results must be correct regardless of RG reorder.+query II+SELECT * FROM th_reorder ORDER BY id ASC LIMIT 3;+----+1 10+3 30+5 50++# Test H.2: DESC ORDER BY with LIMIT — reorder + reverse compose+query TT+EXPLAIN SELECT * FROM th_reorder ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: th_reorder.id DESC NULLS FIRST, fetch=3+02)--TableScan: th_reorder projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/th_reorder/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true++query II+SELECT * FROM th_reorder ORDER BY id DESC LIMIT 3;+----+30 300+25 250+20 200++# Test H.3: Full sort (no LIMIT) — output must still be correctly sorted+query II+SELECT * FROM th_reorder ORDER BY id ASC;+----+1 10+3 30+5 50+10 100+15 150+18 180+20 200+25 250+30 300++# Test H.4: ORDER BY expression (not a simple column) — reorder should+# gracefully skip, results still correct+query II+SELECT id, value FROM th_reorder ORDER BY id + 1 ASC LIMIT 3;+----+1 10+3 30+5 50++# Cleanup Test H+statement ok+DROP TABLE th_mixed;++statement ok+DROP TABLE th_reorder;++# ===========================================================+# Test J: Non-overlapping RGs without WITH ORDER —+# RG reorder via DynamicFilter sort_options+# ===========================================================++# Create data written in scrambled order but with non-overlapping RGs+statement ok+CREATE TABLE tj_high(id INT, value INT) AS VALUES (10,100),(11,110),(12,120);++statement ok+CREATE TABLE tj_mid(id INT, value INT) AS VALUES (4,40),(5,50),(6,60);++statement ok+CREATE TABLE tj_low(id INT, value INT) AS VALUES (1,10),(2,20),(3,30);++statement ok+SET datafusion.execution.parquet.max_row_group_size = 3;++# Write in scrambled order: high, low, mid → RGs are non-overlapping but not sorted+query I+COPY (+ SELECT * FROM tj_high+ UNION ALL SELECT * FROM tj_low+ UNION ALL SELECT * FROM tj_mid+)+TO 'test_files/scratch/sort_pushdown/tj_scrambled/data.parquet';+----+9++statement ok+SET datafusion.execution.parquet.max_row_group_size = 1048576;++# No WITH ORDER — tests non-sort-pushdown TopK path+statement ok+CREATE EXTERNAL TABLE tj_scrambled(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tj_scrambled/data.parquet';++# Test J.1: DESC LIMIT — RG reorder + reverse from DynamicFilter+query TT+EXPLAIN SELECT * FROM tj_scrambled ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: tj_scrambled.id DESC NULLS FIRST, fetch=3+02)--TableScan: tj_scrambled projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tj_scrambled/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC], reverse_row_groups=true++# Test J.2: Results must be correct+query II+SELECT * FROM tj_scrambled ORDER BY id DESC LIMIT 3;+----+12 120+11 110+10 100++# Test J.3: ASC LIMIT+query II+SELECT * FROM tj_scrambled ORDER BY id ASC LIMIT 3;+----+1 10+2 20+3 30++# Test J.4: DESC with WHERE+query II+SELECT * FROM tj_scrambled WHERE value >= 50 ORDER BY id DESC LIMIT 2;+----+12 120+11 110++# Test J.5: Full sort without LIMIT (no TopK, no reorder)+query II+SELECT * FROM tj_scrambled ORDER BY id ASC;+----+1 10+2 20+3 30+4 40+5 50+6 60+10 100+11 110+12 120++# Cleanup Test J+statement ok+DROP TABLE tj_high;++statement ok+DROP TABLE tj_mid;++statement ok+DROP TABLE tj_low;++statement ok+DROP TABLE tj_scrambled;++# ===========================================================+# Test K: Overlapping RGs without WITH ORDER —+# reorder still helps TopK find good values faster+# ===========================================================++# Create data with overlapping ranges+statement ok+CREATE TABLE tk_data(id INT, value INT) AS VALUES+(1,10),(5,50),(9,90),+(3,30),(7,70),(11,110),+(2,20),(6,60),(10,100);++statement ok+SET datafusion.execution.parquet.max_row_group_size = 3;++query I+COPY (SELECT * FROM tk_data)+TO 'test_files/scratch/sort_pushdown/tk_overlap/data.parquet';+----+9++statement ok+SET datafusion.execution.parquet.max_row_group_size = 1048576;++statement ok+CREATE EXTERNAL TABLE tk_overlap(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tk_overlap/data.parquet';++# Test K.1: DESC LIMIT on overlapping RGs — results must be correct+query II+SELECT * FROM tk_overlap ORDER BY id DESC LIMIT 3;+----+11 110+10 100+9 90++# Test K.2: ASC LIMIT on overlapping RGs+query II+SELECT * FROM tk_overlap ORDER BY id ASC LIMIT 3;+----+1 10+2 20+3 30++# Test K.3: Full sort (verify all data present)+query II+SELECT * FROM tk_overlap ORDER BY id ASC;+----+1 10+2 20+3 30+5 50+6 60+7 70+9 90+10 100+11 110++# Cleanup Test K+statement ok+DROP TABLE tk_data;++statement ok+DROP TABLE tk_overlap;++# ===========================================================+# Test L: Multi-key ORDER BY — uses first key for RG reorder+# ===========================================================++statement ok+CREATE TABLE tl_data(id INT, name VARCHAR, value INT) AS VALUES+(10, 'a', 100), (11, 'b', 110), (12, 'c', 120),+(4, 'd', 40), (5, 'e', 50), (6, 'f', 60),+(1, 'g', 10), (2, 'h', 20), (3, 'i', 30);++statement ok+SET datafusion.execution.parquet.max_row_group_size = 3;++query I+COPY (SELECT * FROM tl_data)+TO 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet';+----+9++statement ok+SET datafusion.execution.parquet.max_row_group_size = 1048576;++statement ok+CREATE EXTERNAL TABLE tl_multikey(id INT, name VARCHAR, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet';++# Test L.1: Multi-key DESC — first key determines RG reorder+query IIT+SELECT id, value, name FROM tl_multikey ORDER BY id DESC, name ASC LIMIT 3;+----+12 120 c+11 110 b+10 100 a++# Test L.2: Multi-key ASC+query IIT+SELECT id, value, name FROM tl_multikey ORDER BY id ASC, name DESC LIMIT 3;+----+1 10 g+2 20 h+3 30 i++# Test L.3: Multi-key with 3 sort columns+query IIT+SELECT id, value, name FROM tl_multikey ORDER BY id DESC, value DESC, name ASC LIMIT 5;+----+12 120 c+11 110 b+10 100 a+6 60 f+5 50 e++# Test L.4: WITH ORDER + multi-key DESC LIMIT+statement ok+CREATE EXTERNAL TABLE tl_sorted(id INT, name VARCHAR, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tl_multikey/data.parquet'+WITH ORDER (id ASC);++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++# Multi-key with sort pushdown — first key triggers reverse+query TT+EXPLAIN SELECT id, value FROM tl_sorted ORDER BY id DESC, value ASC LIMIT 3;+----+logical_plan+01)Sort: tl_sorted.id DESC NULLS FIRST, tl_sorted.value ASC NULLS LAST, fetch=3+02)--TableScan: tl_sorted projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC, value@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tl_multikey/data.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[id@0 DESC, value@1 ASC NULLS LAST], reverse_row_groups=true++query II+SELECT id, value FROM tl_sorted ORDER BY id DESC, value ASC LIMIT 3;+----+12 120+11 110+10 100++# Cleanup Test L+statement ok+DROP TABLE tl_data;++statement ok+DROP TABLE tl_multikey;++statement ok+DROP TABLE tl_sorted;++# ===========================================================+# Test P: Multi-file reorder in shared work queue for TopK.+# Files are reordered by statistics so the most promising+# files are processed first across all partitions.+# ===========================================================++statement ok+SET datafusion.execution.target_partitions = 2;++statement ok+SET datafusion.execution.collect_statistics = true;++# Create 3 separate files with non-overlapping ranges in "wrong" order:+# file_high (id 10-12), file_low (id 1-3), file_mid (id 4-6)+statement ok+CREATE TABLE tp_high(id INT, value INT) AS VALUES (10,100),(11,110),(12,120);++statement ok+CREATE TABLE tp_mid(id INT, value INT) AS VALUES (4,40),(5,50),(6,60);++statement ok+CREATE TABLE tp_low(id INT, value INT) AS VALUES (1,10),(2,20),(3,30);++query I+COPY (SELECT * FROM tp_high)+TO 'test_files/scratch/sort_pushdown/tp_multifile/high.parquet';+----+3++query I+COPY (SELECT * FROM tp_mid)+TO 'test_files/scratch/sort_pushdown/tp_multifile/mid.parquet';+----+3++query I+COPY (SELECT * FROM tp_low)+TO 'test_files/scratch/sort_pushdown/tp_multifile/low.parquet';+----+3++# External table over the directory — files discovered in FS order+statement ok+CREATE EXTERNAL TABLE tp_multifile(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tp_multifile/';++# Test P.1: DESC LIMIT — reorder_files puts high.parquet first+query II+SELECT * FROM tp_multifile ORDER BY id DESC LIMIT 3;+----+12 120+11 110+10 100++# Test P.2: ASC LIMIT — reorder_files puts low.parquet first+query II+SELECT * FROM tp_multifile ORDER BY id ASC LIMIT 3;+----+1 10+2 20+3 30++# Test P.3: Full sort — all data present and correct+query II+SELECT * FROM tp_multifile ORDER BY id ASC;+----+1 10+2 20+3 30+4 40+5 50+6 60+10 100+11 110+12 120++# Test P.4: DESC LIMIT with WHERE+query II+SELECT * FROM tp_multifile WHERE value >= 40 ORDER BY id DESC LIMIT 3;+----+12 120+11 110+10 100++# Cleanup Test P+statement ok+DROP TABLE tp_high;++statement ok+DROP TABLE tp_mid;++statement ok+DROP TABLE tp_low;++statement ok+DROP TABLE tp_multifile;++# Restore settings+statement ok+SET datafusion.execution.target_partitions = 4;diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.sltindex 8a1fef0722..8cab67dac0 100644--- a/datafusion/sqllogictest/test_files/topk.slt+++ b/datafusion/sqllogictest/test_files/topk.slt@@ -326,7 +326,7 @@ explain select number, letter, age from partial_sorted order by age desc limit 3----physical_plan01)SortExec: TopK(fetch=3), expr=[age@2 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[age@2 DESC], reverse_row_groups=truequery TTexplain select number, letter, age from partial_sorted order by number desc, letter desc limit 3;@@ -340,14 +340,14 @@ explain select number, letter, age from partial_sorted order by number asc limit----physical_plan01)SortExec: TopK(fetch=3), expr=[number@0 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[number@0 ASC NULLS LAST]query TTexplain select number, letter, age from partial_sorted order by letter asc, number desc limit 3;----physical_plan01)SortExec: TopK(fetch=3), expr=[letter@1 ASC NULLS LAST, number@0 DESC], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[letter@1 ASC NULLS LAST, number@0 DESC]# Explicit NULLS ordering cases (reversing the order of the NULLS on the number and letter orderings)query TT@@ -362,7 +362,7 @@ explain select number, letter, age from partial_sorted order by number desc NULL----physical_plan01)SortExec: TopK(fetch=3), expr=[number@0 DESC NULLS LAST, letter@1 ASC NULLS LAST], preserve_partitioning=[false]-02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], output_ordering=[number@0 DESC, letter@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/topk/partial_sorted/1.parquet]]}, projection=[number, letter, age], file_type=parquet, predicate=DynamicFilter [ empty ], sort_order_for_reorder=[number@0 DESC NULLS LAST, letter@1 ASC NULLS LAST], reverse_row_groups=true# Verify that the sort prefix is correctly computed on the normalized ordering (removing redundant aliased columns)diff --git a/datafusion/sqllogictest/test_files/window_topk_pushdown.slt b/datafusion/sqllogictest/test_files/window_topk_pushdown.sltindex 54e907af5c..20f746d1fd 100644--- a/datafusion/sqllogictest/test_files/window_topk_pushdown.slt+++ b/datafusion/sqllogictest/test_files/window_topk_pushdown.slt@@ -49,7 +49,7 @@ OPTIONS ('format.has_header' 'true');# Config reset-# The SLT runner sets `target_partitions` to 4 instead of using the default, so+# The SLT runner sets `target_partitions` to 4 instead of using the default, so# reset it explicitly.statement okSET datafusion.execution.target_partitions = 4;