Apache DataFusion SQL Query Engine
refactor: extract sort pushdown logic from FileScanConfig into separate module (#21457)
Qi Zhu4389f14e70da
diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config/mod.rssimilarity index 83%rename from datafusion/datasource/src/file_scan_config.rsrename to datafusion/datasource/src/file_scan_config/mod.rsindex 976e1158f5..cb39e44e7b 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -18,6 +18,8 @@//! [`FileScanConfig`] to configure scanning of possibly partitioned//! file sources.+pub(crate) mod sort_pushdown;+use crate::file_groups::FileGroup;use crate::{PartitionedFile, display::FileGroupsDisplay, file::FileSource,@@ -27,7 +29,6 @@ use crate::{use arrow::datatypes::FieldRef;use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;-use datafusion_common::stats::Precision;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,@@ -37,7 +38,6 @@ use datafusion_execution::{};use datafusion_expr::Operator;-use datafusion_physical_expr::equivalence::project_orderings;use datafusion_physical_expr::expressions::{BinaryExpr, Column};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::utils::reassign_expr_columns;@@ -605,7 +605,8 @@ impl DataSource for FileScanConfig {match t {DisplayFormatType::Default | DisplayFormatType::Verbose => {let schema = self.projected_schema().map_err(|_| std::fmt::Error {})?;- let orderings = get_projected_output_ordering(self, &schema);+ let orderings =+ sort_pushdown::get_projected_output_ordering(self, &schema);write!(f, "file_groups=")?;FileGroupsDisplay(&self.file_groups).fmt_as(t, f)?;@@ -992,13 +993,6 @@ impl DataSource for FileScanConfig {}}-/// Result of sorting files within groups by their min/max statistics.-struct SortedFileGroups {- file_groups: Vec<FileGroup>,- any_reordered: bool,- all_non_overlapping: bool,-}-impl FileScanConfig {/// Returns only the output orderings that are validated against actual/// file group statistics.@@ -1031,7 +1025,12 @@ impl FileScanConfig {/// upstream e.g. by `SortPreservingMergeExec`.fn validated_output_ordering(&self) -> Vec<LexOrdering> {let schema = self.file_source.table_schema().table_schema();- validate_orderings(&self.output_ordering, schema, &self.file_groups, None)+ sort_pushdown::validate_orderings(+ &self.output_ordering,+ schema,+ &self.file_groups,+ None,+ )}/// Get the file schema (schema of the files without partition columns)@@ -1299,336 +1298,9 @@ impl FileScanConfig {&self.file_source}- /// Rebuild FileScanConfig after sort pushdown, applying file-level optimizations.- ///- /// This is the core of sort pushdown for file-based sources. It performs- /// three optimizations depending on the pushdown result:- ///- /// ```text- /// ┌─────────────────────────────────────────────────────────────┐- /// │ rebuild_with_source │- /// │ │- /// │ 1. Reverse file groups (if DESC matches reversed ordering) │- /// │ 2. Sort files within groups by min/max statistics │- /// │ 3. If Exact + non-overlapping: │- /// │ Keep output_ordering → SortExec eliminated │- /// │ Otherwise: clear output_ordering → SortExec stays │- /// └─────────────────────────────────────────────────────────────┘- /// ```- ///- /// # Why sort files by statistics?- ///- /// Files within a partition (file group) are read sequentially. By sorting- /// them so that file_i.max <= file_{i+1}.min, the combined output stream- /// is already in order — no SortExec needed for that partition.- ///- /// Even when files overlap (Inexact), statistics-based ordering helps- /// TopK/LIMIT queries: reading low-value files first lets dynamic filters- /// prune high-value files earlier.- fn rebuild_with_source(- &self,- new_file_source: Arc<dyn FileSource>,- is_exact: bool,- order: &[PhysicalSortExpr],- ) -> Result<FileScanConfig> {- let mut new_config = self.clone();-- // Reverse file order (within each group) if the caller is requesting a reversal of this- // scan's declared output ordering.- let reverse_file_groups = if self.output_ordering.is_empty() {- false- } else if let Some(requested) = LexOrdering::new(order.iter().cloned()) {- let projected_schema = self.projected_schema()?;- let orderings = project_orderings(&self.output_ordering, &projected_schema);- orderings- .iter()- .any(|ordering| ordering.is_reverse(&requested))- } else {- false- };-- if reverse_file_groups {- new_config.file_groups = new_config- .file_groups- .into_iter()- .map(|group| {- let mut files = group.into_inner();- files.reverse();- files.into()- })- .collect();- }-- new_config.file_source = new_file_source;-- // Sort files within groups by statistics when not reversing- let all_non_overlapping = if !reverse_file_groups {- if let Some(sort_order) = LexOrdering::new(order.iter().cloned()) {- let projected_schema = new_config.projected_schema()?;- let projection_indices = new_config- .file_source- .projection()- .as_ref()- .and_then(|p| ordered_column_indices_from_projection(p));- let result = Self::sort_files_within_groups_by_statistics(- &new_config.file_groups,- &sort_order,- &projected_schema,- projection_indices.as_deref(),- );- new_config.file_groups = result.file_groups;- result.all_non_overlapping- } else {- false- }- } else {- // When reversing, files are already reversed above. We skip- // statistics-based sorting here because it would undo the reversal.- // Note: reverse path is always Inexact, so all_non_overlapping- // is not used (is_exact is false).- false- };-- if is_exact && all_non_overlapping {- // Truly exact: within-file ordering guaranteed and files are non-overlapping.- // Keep output_ordering so SortExec can be eliminated for each partition.- //- // We intentionally do NOT redistribute files across groups here.- // The planning-phase bin-packing may interleave file ranges across groups:- //- // Group 0: [f1(1-10), f3(21-30)] ← interleaved with group 1- // Group 1: [f2(11-20), f4(31-40)]- //- // This interleaving is actually beneficial because SPM pulls from both- // partitions concurrently, keeping parallel I/O active:- //- // SPM: pull P0 [1-10] → pull P1 [11-20] → pull P0 [21-30] → pull P1 [31-40]- // ^^^^^^^^^^^^ ^^^^^^^^^^^^- // both partitions scanning files simultaneously- //- // If we were to redistribute files consecutively:- // Group 0: [f1(1-10), f2(11-20)] ← all values < group 1- // Group 1: [f3(21-30), f4(31-40)]- //- // SPM would read ALL of group 0 first (values always smaller), then group 1.- // This degrades to single-threaded sequential I/O — the other partition- // sits idle the entire time, losing the parallelism benefit.- } else {- new_config.output_ordering = vec![];- }-- Ok(new_config)- }-- /// Sort files within each file group by their min/max statistics.- ///- /// No files are moved between groups — parallelism and group composition- /// are unchanged. Groups where statistics are unavailable are kept as-is.- ///- /// ```text- /// Before: Group [file_c(20-30), file_a(0-9), file_b(10-19)]- /// After: Group [file_a(0-9), file_b(10-19), file_c(20-30)]- /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^- /// sorted by min value, non-overlapping → Exact- /// ```- fn sort_files_within_groups_by_statistics(- file_groups: &[FileGroup],- sort_order: &LexOrdering,- projected_schema: &SchemaRef,- projection_indices: Option<&[usize]>,- ) -> SortedFileGroups {- let mut any_reordered = false;- let mut confirmed_non_overlapping: usize = 0;- let mut new_groups = Vec::with_capacity(file_groups.len());-- for group in file_groups {- if group.len() <= 1 {- new_groups.push(group.clone());- confirmed_non_overlapping += 1;- continue;- }-- let files: Vec<_> = group.iter().collect();-- let statistics = match MinMaxStatistics::new_from_files(- sort_order,- projected_schema,- projection_indices,- files.iter().copied(),- ) {- Ok(stats) => stats,- Err(e) => {- log::trace!(- "Cannot sort file group by statistics: {e}. Keeping original order."- );- new_groups.push(group.clone());- continue;- }- };-- let sorted_indices = statistics.min_values_sorted();-- let already_sorted = sorted_indices- .iter()- .enumerate()- .all(|(pos, (idx, _))| pos == *idx);-- let sorted_group: FileGroup = if already_sorted {- group.clone()- } else {- any_reordered = true;- sorted_indices- .iter()- .map(|(idx, _)| files[*idx].clone())- .collect()- };-- let sorted_files: Vec<_> = sorted_group.iter().collect();- let is_non_overlapping = match MinMaxStatistics::new_from_files(- sort_order,- projected_schema,- projection_indices,- sorted_files.iter().copied(),- ) {- Ok(stats) => stats.is_sorted(),- Err(_) => false,- };-- if is_non_overlapping {- confirmed_non_overlapping += 1;- }-- new_groups.push(sorted_group);- }-- SortedFileGroups {- file_groups: new_groups,- any_reordered,- all_non_overlapping: confirmed_non_overlapping == file_groups.len(),- }- }-- /// Last-resort optimization when FileSource returns `Unsupported`.- ///- /// FileSource may return `Unsupported` because `eq_properties` had no- /// ordering — which happens when `validated_output_ordering()` stripped- /// the ordering because files were in the wrong order. After sorting- /// files by statistics, the ordering may become valid again.- ///- /// This method:- /// 1. Sorts files within groups by min/max statistics- /// 2. Re-checks if the sorted file order makes `output_ordering` valid- /// 3. If valid AND non-overlapping → `Exact` (SortExec eliminated!)- /// 4. If files were reordered but ordering not valid → `Inexact`- /// 5. If no files were reordered → `Unsupported`- ///- /// This handles the key case where files have correct within-file ordering- /// (e.g., Parquet sorting_columns metadata) but were listed in wrong order- /// (e.g., alphabetical order doesn't match sort key order).- fn try_sort_file_groups_by_statistics(- &self,- order: &[PhysicalSortExpr],- ) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {- let Some(sort_order) = LexOrdering::new(order.iter().cloned()) else {- return Ok(SortOrderPushdownResult::Unsupported);- };-- let projected_schema = self.projected_schema()?;- let projection_indices = self- .file_source- .projection()- .as_ref()- .and_then(|p| ordered_column_indices_from_projection(p));-- let result = Self::sort_files_within_groups_by_statistics(- &self.file_groups,- &sort_order,- &projected_schema,- projection_indices.as_deref(),- );-- if !result.any_reordered {- return Ok(SortOrderPushdownResult::Unsupported);- }-- let mut new_config = self.clone();- new_config.file_groups = result.file_groups;-- // Re-check: now that files are sorted, does output_ordering become valid?- // This handles the case where validated_output_ordering() previously- // stripped the ordering because files were in the wrong order.- //- // IMPORTANT: We cannot claim Exact if any file in a non-last position- // contains NULLs in the sort columns. With NULLS LAST, NULLs within- // a file are placed after all non-null values. If the next file has- // non-null values smaller than the previous file's max, those values- // would incorrectly appear after the NULLs. Similarly for NULLS FIRST.- //- // Conservative approach: if any file has nulls in the sort columns,- // do not claim Exact. The SortExec will handle NULL ordering correctly.- if result.all_non_overlapping- && !self.output_ordering.is_empty()- && !Self::any_file_has_nulls_in_sort_columns(- &new_config.file_groups,- order,- &projected_schema,- projection_indices.as_deref(),- )- {- // Files are now non-overlapping, no NULLs in sort columns.- // Re-ask the FileSource if this ordering satisfies the request,- // using eq_properties computed from the NEW (sorted) file groups.- let new_eq_props = new_config.eq_properties();- if new_eq_props.ordering_satisfy(order.iter().cloned())? {- // The sorted file order makes the ordering valid → Exact!- return Ok(SortOrderPushdownResult::Exact {- inner: Arc::new(new_config),- });- }- }-- new_config.output_ordering = vec![];- Ok(SortOrderPushdownResult::Inexact {- inner: Arc::new(new_config),- })- }-- /// Check if any file in any group has nulls in the sort columns.- fn any_file_has_nulls_in_sort_columns(- file_groups: &[FileGroup],- order: &[PhysicalSortExpr],- projected_schema: &SchemaRef,- projection_indices: Option<&[usize]>,- ) -> bool {- let Some(sort_columns) =- sort_columns_from_physical_sort_exprs_nullable(order, projected_schema)- else {- return true; // Can't determine, assume nulls exist- };-- for group in file_groups {- for file in group.iter() {- let Some(stats) = file.statistics.as_ref() else {- return true; // No stats, assume nulls exist- };- for col in &sort_columns {- let stat_idx = projection_indices- .map(|p| p[col.index()])- .unwrap_or_else(|| col.index());- if stat_idx >= stats.column_statistics.len() {- return true;- }- let col_stats = &stats.column_statistics[stat_idx];- match &col_stats.null_count {- Precision::Exact(0) => {} // No nulls, safe- Precision::Exact(_) => return true, // Has nulls- _ => return true, // Unknown null count, assume nulls- }- }- }- }- false- }+ // Sort pushdown methods (rebuild_with_source, try_sort_file_groups_by_statistics,+ // sort_files_within_groups_by_statistics, any_file_has_nulls_in_sort_columns)+ // are in crate::sort_pushdown module.}impl Debug for FileScanConfig {@@ -1646,7 +1318,7 @@ impl Debug for FileScanConfig {impl DisplayAs for FileScanConfig {fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> FmtResult {let schema = self.projected_schema().map_err(|_| std::fmt::Error {})?;- let orderings = get_projected_output_ordering(self, &schema);+ let orderings = sort_pushdown::get_projected_output_ordering(self, &schema);write!(f, "file_groups=")?;FileGroupsDisplay(&self.file_groups).fmt_as(t, f)?;@@ -1669,202 +1341,6 @@ impl DisplayAs for FileScanConfig {}}-/// Get the indices of columns in a projection if the projection is a simple-/// list of columns.-/// If there are any expressions other than columns, returns None.-fn ordered_column_indices_from_projection(- projection: &ProjectionExprs,-) -> Option<Vec<usize>> {- projection- .expr_iter()- .map(|e| {- let index = e.as_any().downcast_ref::<Column>()?.index();- Some(index)- })- .collect::<Option<Vec<usize>>>()-}--/// Extract Column references from sort expressions for null checking.-fn sort_columns_from_physical_sort_exprs_nullable(- order: &[PhysicalSortExpr],- _schema: &SchemaRef,-) -> Option<Vec<Column>> {- order- .iter()- .map(|expr| expr.expr.as_any().downcast_ref::<Column>().cloned())- .collect()-}--/// Check whether a given ordering is valid for all file groups by verifying-/// that files within each group are sorted according to their min/max statistics.-///-/// For single-file (or empty) groups, the ordering is trivially valid.-/// For multi-file groups, we check that the min/max statistics for the sort-/// columns are in order and non-overlapping (or touching at boundaries).-///-/// `projection` maps projected column indices back to table-schema indices-/// when validating after projection; pass `None` when validating at-/// table-schema level.-fn is_ordering_valid_for_file_groups(- file_groups: &[FileGroup],- ordering: &LexOrdering,- schema: &SchemaRef,- projection: Option<&[usize]>,-) -> bool {- file_groups.iter().all(|group| {- if group.len() <= 1 {- return true; // single-file groups are trivially sorted- }- match MinMaxStatistics::new_from_files(ordering, schema, projection, group.iter())- {- Ok(stats) => stats.is_sorted(),- Err(_) => false, // can't prove sorted → reject- }- })-}--/// Filters orderings to retain only those valid for all file groups,-/// verified via min/max statistics.-fn validate_orderings(- orderings: &[LexOrdering],- schema: &SchemaRef,- file_groups: &[FileGroup],- projection: Option<&[usize]>,-) -> Vec<LexOrdering> {- orderings- .iter()- .filter(|ordering| {- is_ordering_valid_for_file_groups(file_groups, ordering, schema, projection)- })- .cloned()- .collect()-}--/// The various listing tables does not attempt to read all files-/// concurrently, instead they will read files in sequence within a-/// partition. This is an important property as it allows plans to-/// run against 1000s of files and not try to open them all-/// concurrently.-///-/// However, it means if we assign more than one file to a partition-/// the output sort order will not be preserved as illustrated in the-/// following diagrams:-///-/// When only 1 file is assigned to each partition, each partition is-/// correctly sorted on `(A, B, C)`-///-/// ```text-/// ┏ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ┓-/// ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ┐-/// ┃ ┌───────────────┐ ┌──────────────┐ │ ┌──────────────┐ │ ┌─────────────┐ ┃-/// │ │ 1.parquet │ │ │ │ 2.parquet │ │ │ 3.parquet │ │ │ 4.parquet │ │-/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │ │Sort: A, B, C │ │ │Sort: A, B, C│ ┃-/// │ └───────────────┘ │ │ └──────────────┘ │ └──────────────┘ │ └─────────────┘ │-/// ┃ │ │ ┃-/// │ │ │ │ │ │-/// ┃ │ │ ┃-/// │ │ │ │ │ │-/// ┃ │ │ ┃-/// │ │ │ │ │ │-/// ┃ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┃-/// DataFusion DataFusion DataFusion DataFusion-/// ┃ Partition 1 Partition 2 Partition 3 Partition 4 ┃-/// ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━-///-/// DataSourceExec-/// ```-///-/// However, when more than 1 file is assigned to each partition, each-/// partition is NOT correctly sorted on `(A, B, C)`. Once the second-/// file is scanned, the same values for A, B and C can be repeated in-/// the same sorted stream-///-///```text-/// ┏ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━-/// ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┃-/// ┃ ┌───────────────┐ ┌──────────────┐ │-/// │ │ 1.parquet │ │ │ │ 2.parquet │ ┃-/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │-/// │ └───────────────┘ │ │ └──────────────┘ ┃-/// ┃ ┌───────────────┐ ┌──────────────┐ │-/// │ │ 3.parquet │ │ │ │ 4.parquet │ ┃-/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │-/// │ └───────────────┘ │ │ └──────────────┘ ┃-/// ┃ │-/// │ │ │ ┃-/// ┃ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘-/// DataFusion DataFusion ┃-/// ┃ Partition 1 Partition 2-/// ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ┛-///-/// DataSourceExec-/// ```-///-/// **Exception**: When files within a partition are **non-overlapping** (verified-/// via min/max statistics) and each file is internally sorted, the combined-/// output is still correctly sorted. Sort pushdown-/// ([`FileScanConfig::try_pushdown_sort`]) detects this case and preserves-/// `output_ordering`, allowing `SortExec` to be eliminated entirely.-///-/// ```text-/// Partition 1 (files sorted by stats, non-overlapping):-/// ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐-/// │ 1.parquet │ │ 2.parquet │ │ 3.parquet │-/// │ A: [1..100] │ │ A: [101..200] │ │ A: [201..300] │-/// │ Sort: A, B, C │ │ Sort: A, B, C │ │ Sort: A, B, C │-/// └──────────────────┘ └──────────────────┘ └──────────────────┘-/// max(1) <= min(2) ✓ max(2) <= min(3) ✓ → output_ordering preserved-/// ```-fn get_projected_output_ordering(- base_config: &FileScanConfig,- projected_schema: &SchemaRef,-) -> Vec<LexOrdering> {- let projected_orderings =- project_orderings(&base_config.output_ordering, projected_schema);-- let indices = base_config- .file_source- .projection()- .as_ref()- .map(|p| ordered_column_indices_from_projection(p));-- match indices {- Some(Some(indices)) => {- // Simple column projection — validate with statistics- validate_orderings(- &projected_orderings,- projected_schema,- &base_config.file_groups,- Some(indices.as_slice()),- )- }- None => {- // No projection — validate with statistics (no remapping needed)- validate_orderings(- &projected_orderings,- projected_schema,- &base_config.file_groups,- None,- )- }- Some(None) => {- // Complex projection (expressions, not simple columns) — can't- // determine column indices for statistics. Still valid if all- // file groups have at most one file.- if base_config.file_groups.iter().all(|g| g.len() <= 1) {- projected_orderings- } else {- debug!(- "Skipping specified output orderings. \- Some file groups couldn't be determined to be sorted: {:?}",- base_config.file_groups- );- vec![]- }- }- }-}-/// Convert type to a type suitable for use as a `ListingTable`/// partition column. Returns `Dictionary(UInt16, val_type)`, which is/// a reasonable trade off between a reasonable number of partitiondiff --git a/datafusion/datasource/src/file_scan_config/sort_pushdown.rs b/datafusion/datasource/src/file_scan_config/sort_pushdown.rsnew file mode 100644index 0000000000..db4019071f--- /dev/null+++ b/datafusion/datasource/src/file_scan_config/sort_pushdown.rs@@ -0,0 +1,577 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Sort pushdown helpers for [`FileScanConfig`].+//!+//! This module contains the statistics-based file sorting, non-overlapping+//! validation, and NULL handling logic used by+//! [`FileScanConfig::try_pushdown_sort`](super::FileScanConfig::try_pushdown_sort).+//!+//! Extracted from `file_scan_config.rs` to keep that module focused on+//! core configuration and data-source plumbing.++use super::FileScanConfig;+use crate::file::FileSource;+use crate::file_groups::FileGroup;+use crate::source::DataSource;+use crate::statistics::MinMaxStatistics;++use arrow::datatypes::SchemaRef;+use datafusion_common::Result;+use datafusion_common::stats::Precision;+use datafusion_physical_expr::equivalence::project_orderings;+use datafusion_physical_expr::expressions::Column;+use datafusion_physical_expr::projection::ProjectionExprs;+use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};+use datafusion_physical_plan::SortOrderPushdownResult;+use log::debug;+use std::sync::Arc;++/// Result of sorting files within groups by their min/max statistics.+pub(crate) struct SortedFileGroups {+ file_groups: Vec<FileGroup>,+ any_reordered: bool,+ all_non_overlapping: bool,+}++impl FileScanConfig {+ ///+ /// This is the core of sort pushdown for file-based sources. It performs+ /// three optimizations depending on the pushdown result:+ ///+ /// ```text+ /// ┌─────────────────────────────────────────────────────────────┐+ /// │ rebuild_with_source │+ /// │ │+ /// │ 1. Reverse file groups (if DESC matches reversed ordering) │+ /// │ 2. Sort files within groups by min/max statistics │+ /// │ 3. If Exact + non-overlapping: │+ /// │ Keep output_ordering → SortExec eliminated │+ /// │ Otherwise: clear output_ordering → SortExec stays │+ /// └─────────────────────────────────────────────────────────────┘+ /// ```+ ///+ /// # Why sort files by statistics?+ ///+ /// Files within a partition (file group) are read sequentially. By sorting+ /// them so that file_i.max <= file_{i+1}.min, the combined output stream+ /// is already in order — no SortExec needed for that partition.+ ///+ /// Even when files overlap (Inexact), statistics-based ordering helps+ /// TopK/LIMIT queries: reading low-value files first lets dynamic filters+ /// prune high-value files earlier.+ pub(crate) fn rebuild_with_source(+ &self,+ new_file_source: Arc<dyn FileSource>,+ is_exact: bool,+ order: &[PhysicalSortExpr],+ ) -> Result<FileScanConfig> {+ let mut new_config = self.clone();++ // Reverse file order (within each group) if the caller is requesting a reversal of this+ // scan's declared output ordering.+ let reverse_file_groups = if self.output_ordering.is_empty() {+ false+ } else if let Some(requested) = LexOrdering::new(order.iter().cloned()) {+ let projected_schema = self.projected_schema()?;+ let orderings = project_orderings(&self.output_ordering, &projected_schema);+ orderings+ .iter()+ .any(|ordering| ordering.is_reverse(&requested))+ } else {+ false+ };++ if reverse_file_groups {+ new_config.file_groups = new_config+ .file_groups+ .into_iter()+ .map(|group| {+ let mut files = group.into_inner();+ files.reverse();+ files.into()+ })+ .collect();+ }++ new_config.file_source = new_file_source;++ // Sort files within groups by statistics when not reversing+ let all_non_overlapping = if !reverse_file_groups {+ if let Some(sort_order) = LexOrdering::new(order.iter().cloned()) {+ let projected_schema = new_config.projected_schema()?;+ let projection_indices = new_config+ .file_source+ .projection()+ .as_ref()+ .and_then(|p| ordered_column_indices_from_projection(p));+ let result = sort_files_within_groups_by_statistics(+ &new_config.file_groups,+ &sort_order,+ &projected_schema,+ projection_indices.as_deref(),+ );+ new_config.file_groups = result.file_groups;+ result.all_non_overlapping+ } else {+ false+ }+ } else {+ // When reversing, files are already reversed above. We skip+ // statistics-based sorting here because it would undo the reversal.+ // Note: reverse path is always Inexact, so all_non_overlapping+ // is not used (is_exact is false).+ false+ };++ if is_exact && all_non_overlapping {+ // Truly exact: within-file ordering guaranteed and files are non-overlapping.+ // Keep output_ordering so SortExec can be eliminated for each partition.+ //+ // We intentionally do NOT redistribute files across groups here.+ // The planning-phase bin-packing may interleave file ranges across groups:+ //+ // Group 0: [f1(1-10), f3(21-30)] ← interleaved with group 1+ // Group 1: [f2(11-20), f4(31-40)]+ //+ // This interleaving is actually beneficial because SPM pulls from both+ // partitions concurrently, keeping parallel I/O active:+ //+ // SPM: pull P0 [1-10] → pull P1 [11-20] → pull P0 [21-30] → pull P1 [31-40]+ // ^^^^^^^^^^^^ ^^^^^^^^^^^^+ // both partitions scanning files simultaneously+ //+ // If we were to redistribute files consecutively:+ // Group 0: [f1(1-10), f2(11-20)] ← all values < group 1+ // Group 1: [f3(21-30), f4(31-40)]+ //+ // SPM would read ALL of group 0 first (values always smaller), then group 1.+ // This degrades to single-threaded sequential I/O — the other partition+ // sits idle the entire time, losing the parallelism benefit.+ } else {+ new_config.output_ordering = vec![];+ }++ Ok(new_config)+ }++ /// Last-resort optimization when FileSource returns `Unsupported`.+ ///+ /// FileSource may return `Unsupported` because `eq_properties` had no+ /// ordering — which happens when `validated_output_ordering()` stripped+ /// the ordering because files were in the wrong order. After sorting+ /// files by statistics, the ordering may become valid again.+ ///+ /// This method:+ /// 1. Sorts files within groups by min/max statistics+ /// 2. Re-checks if the sorted file order makes `output_ordering` valid+ /// 3. If valid AND non-overlapping → `Exact` (SortExec eliminated!)+ /// 4. If files were reordered but ordering not valid → `Inexact`+ /// 5. If no files were reordered → `Unsupported`+ ///+ /// This handles the key case where files have correct within-file ordering+ /// (e.g., Parquet sorting_columns metadata) but were listed in wrong order+ /// (e.g., alphabetical order doesn't match sort key order).+ pub(crate) fn try_sort_file_groups_by_statistics(+ &self,+ order: &[PhysicalSortExpr],+ ) -> Result<SortOrderPushdownResult<Arc<dyn DataSource>>> {+ let Some(sort_order) = LexOrdering::new(order.iter().cloned()) else {+ return Ok(SortOrderPushdownResult::Unsupported);+ };++ let projected_schema = self.projected_schema()?;+ let projection_indices = self+ .file_source+ .projection()+ .as_ref()+ .and_then(|p| ordered_column_indices_from_projection(p));++ let result = sort_files_within_groups_by_statistics(+ &self.file_groups,+ &sort_order,+ &projected_schema,+ projection_indices.as_deref(),+ );++ if !result.any_reordered {+ return Ok(SortOrderPushdownResult::Unsupported);+ }++ let mut new_config = self.clone();+ new_config.file_groups = result.file_groups;++ // Re-check: now that files are sorted, does output_ordering become valid?+ // This handles the case where validated_output_ordering() previously+ // stripped the ordering because files were in the wrong order.+ //+ // IMPORTANT: We cannot claim Exact if any file in a non-last position+ // contains NULLs in the sort columns. With NULLS LAST, NULLs within+ // a file are placed after all non-null values. If the next file has+ // non-null values smaller than the previous file's max, those values+ // would incorrectly appear after the NULLs. Similarly for NULLS FIRST.+ //+ // Conservative approach: if any file has nulls in the sort columns,+ // do not claim Exact. The SortExec will handle NULL ordering correctly.+ if result.all_non_overlapping+ && !self.output_ordering.is_empty()+ && !any_file_has_nulls_in_sort_columns(+ &new_config.file_groups,+ order,+ &projected_schema,+ projection_indices.as_deref(),+ )+ {+ // Files are now non-overlapping, no NULLs in sort columns.+ // Re-ask the FileSource if this ordering satisfies the request,+ // using eq_properties computed from the NEW (sorted) file groups.+ let new_eq_props = new_config.eq_properties();+ if new_eq_props.ordering_satisfy(order.iter().cloned())? {+ // The sorted file order makes the ordering valid → Exact!+ return Ok(SortOrderPushdownResult::Exact {+ inner: Arc::new(new_config),+ });+ }+ }++ new_config.output_ordering = vec![];+ Ok(SortOrderPushdownResult::Inexact {+ inner: Arc::new(new_config),+ })+ }+}++/// Sort files within each file group by their min/max statistics.+///+/// No files are moved between groups — parallelism and group composition+/// are unchanged. Groups where statistics are unavailable are kept as-is.+///+/// ```text+/// Before: Group [file_c(20-30), file_a(0-9), file_b(10-19)]+/// After: Group [file_a(0-9), file_b(10-19), file_c(20-30)]+/// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^+/// sorted by min value, non-overlapping → Exact+/// ```+pub(crate) fn sort_files_within_groups_by_statistics(+ file_groups: &[FileGroup],+ sort_order: &LexOrdering,+ projected_schema: &SchemaRef,+ projection_indices: Option<&[usize]>,+) -> SortedFileGroups {+ let mut any_reordered = false;+ let mut confirmed_non_overlapping: usize = 0;+ let mut new_groups = Vec::with_capacity(file_groups.len());++ for group in file_groups {+ if group.len() <= 1 {+ new_groups.push(group.clone());+ confirmed_non_overlapping += 1;+ continue;+ }++ let files: Vec<_> = group.iter().collect();++ let statistics = match MinMaxStatistics::new_from_files(+ sort_order,+ projected_schema,+ projection_indices,+ files.iter().copied(),+ ) {+ Ok(stats) => stats,+ Err(e) => {+ log::trace!(+ "Cannot sort file group by statistics: {e}. Keeping original order."+ );+ new_groups.push(group.clone());+ continue;+ }+ };++ let sorted_indices = statistics.min_values_sorted();++ let already_sorted = sorted_indices+ .iter()+ .enumerate()+ .all(|(pos, (idx, _))| pos == *idx);++ let sorted_group: FileGroup = if already_sorted {+ group.clone()+ } else {+ any_reordered = true;+ sorted_indices+ .iter()+ .map(|(idx, _)| files[*idx].clone())+ .collect()+ };++ let sorted_files: Vec<_> = sorted_group.iter().collect();+ let is_non_overlapping = match MinMaxStatistics::new_from_files(+ sort_order,+ projected_schema,+ projection_indices,+ sorted_files.iter().copied(),+ ) {+ Ok(stats) => stats.is_sorted(),+ Err(_) => false,+ };++ if is_non_overlapping {+ confirmed_non_overlapping += 1;+ }++ new_groups.push(sorted_group);+ }++ SortedFileGroups {+ file_groups: new_groups,+ any_reordered,+ all_non_overlapping: confirmed_non_overlapping == file_groups.len(),+ }+}++/// Check if any file in any group has nulls in the sort columns.+pub(crate) fn any_file_has_nulls_in_sort_columns(+ file_groups: &[FileGroup],+ order: &[PhysicalSortExpr],+ projected_schema: &SchemaRef,+ projection_indices: Option<&[usize]>,+) -> bool {+ let Some(sort_columns) =+ sort_columns_from_physical_sort_exprs_nullable(order, projected_schema)+ else {+ return true; // Can't determine, assume nulls exist+ };++ for group in file_groups {+ for file in group.iter() {+ let Some(stats) = file.statistics.as_ref() else {+ return true; // No stats, assume nulls exist+ };+ for col in &sort_columns {+ let stat_idx = projection_indices+ .map(|p| p[col.index()])+ .unwrap_or_else(|| col.index());+ if stat_idx >= stats.column_statistics.len() {+ return true;+ }+ let col_stats = &stats.column_statistics[stat_idx];+ match &col_stats.null_count {+ Precision::Exact(0) => {} // No nulls, safe+ Precision::Exact(_) => return true, // Has nulls+ _ => return true, // Unknown null count, assume nulls+ }+ }+ }+ }+ false+}++/// Get the indices of columns in a projection if the projection is a simple+/// list of columns.+/// If there are any expressions other than columns, returns None.+pub(crate) fn ordered_column_indices_from_projection(+ projection: &ProjectionExprs,+) -> Option<Vec<usize>> {+ projection+ .expr_iter()+ .map(|e| {+ let index = e.as_any().downcast_ref::<Column>()?.index();+ Some(index)+ })+ .collect::<Option<Vec<usize>>>()+}++/// Extract Column references from sort expressions for null checking.+fn sort_columns_from_physical_sort_exprs_nullable(+ order: &[PhysicalSortExpr],+ _schema: &SchemaRef,+) -> Option<Vec<Column>> {+ order+ .iter()+ .map(|expr| expr.expr.as_any().downcast_ref::<Column>().cloned())+ .collect()+}++/// Check whether a given ordering is valid for all file groups by verifying+/// that files within each group are sorted according to their min/max statistics.+///+/// For single-file (or empty) groups, the ordering is trivially valid.+/// For multi-file groups, we check that the min/max statistics for the sort+/// columns are in order and non-overlapping (or touching at boundaries).+///+/// `projection` maps projected column indices back to table-schema indices+/// when validating after projection; pass `None` when validating at+/// table-schema level.+pub(crate) fn is_ordering_valid_for_file_groups(+ file_groups: &[FileGroup],+ ordering: &LexOrdering,+ schema: &SchemaRef,+ projection: Option<&[usize]>,+) -> bool {+ file_groups.iter().all(|group| {+ if group.len() <= 1 {+ return true; // single-file groups are trivially sorted+ }+ match MinMaxStatistics::new_from_files(ordering, schema, projection, group.iter())+ {+ Ok(stats) => stats.is_sorted(),+ Err(_) => false, // can't prove sorted → reject+ }+ })+}++/// Filters orderings to retain only those valid for all file groups,+/// verified via min/max statistics.+pub(crate) fn validate_orderings(+ orderings: &[LexOrdering],+ schema: &SchemaRef,+ file_groups: &[FileGroup],+ projection: Option<&[usize]>,+) -> Vec<LexOrdering> {+ orderings+ .iter()+ .filter(|ordering| {+ is_ordering_valid_for_file_groups(file_groups, ordering, schema, projection)+ })+ .cloned()+ .collect()+}++/// The various listing tables do not attempt to read all files+/// concurrently, instead they will read files in sequence within a+/// partition. This is an important property as it allows plans to+/// run against 1000s of files and not try to open them all+/// concurrently.+///+/// However, it means if we assign more than one file to a partition+/// the output sort order will not be preserved as illustrated in the+/// following diagrams:+///+/// When only 1 file is assigned to each partition, each partition is+/// correctly sorted on `(A, B, C)`+///+/// ```text+/// ┏ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ┓+/// ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ┐+/// ┃ ┌───────────────┐ ┌──────────────┐ │ ┌──────────────┐ │ ┌─────────────┐ ┃+/// │ │ 1.parquet │ │ │ │ 2.parquet │ │ │ 3.parquet │ │ │ 4.parquet │ │+/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │ │Sort: A, B, C │ │ │Sort: A, B, C│ ┃+/// │ └───────────────┘ │ │ └──────────────┘ │ └──────────────┘ │ └─────────────┘ │+/// ┃ │ │ ┃+/// │ │ │ │ │ │+/// ┃ │ │ ┃+/// │ │ │ │ │ │+/// ┃ │ │ ┃+/// │ │ │ │ │ │+/// ┃ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┃+/// DataFusion DataFusion DataFusion DataFusion+/// ┃ Partition 1 Partition 2 Partition 3 Partition 4 ┃+/// ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━+///+/// DataSourceExec+/// ```+///+/// However, when more than 1 file is assigned to each partition, each+/// partition is NOT correctly sorted on `(A, B, C)`. Once the second+/// file is scanned, the same values for A, B and C can be repeated in+/// the same sorted stream+///+///```text+/// ┏ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━+/// ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐ ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┃+/// ┃ ┌───────────────┐ ┌──────────────┐ │+/// │ │ 1.parquet │ │ │ │ 2.parquet │ ┃+/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │+/// │ └───────────────┘ │ │ └──────────────┘ ┃+/// ┃ ┌───────────────┐ ┌──────────────┐ │+/// │ │ 3.parquet │ │ │ │ 4.parquet │ ┃+/// ┃ │ Sort: A, B, C │ │Sort: A, B, C │ │+/// │ └───────────────┘ │ │ └──────────────┘ ┃+/// ┃ │+/// │ │ │ ┃+/// ┃ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘+/// DataFusion DataFusion ┃+/// ┃ Partition 1 Partition 2+/// ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ━ ┛+///+/// DataSourceExec+/// ```+///+/// **Exception**: When files within a partition are **non-overlapping** (verified+/// via min/max statistics) and each file is internally sorted, the combined+/// output is still correctly sorted. Sort pushdown+/// ([`FileScanConfig::try_pushdown_sort`]) detects this case and preserves+/// `output_ordering`, allowing `SortExec` to be eliminated entirely.+///+/// ```text+/// Partition 1 (files sorted by stats, non-overlapping):+/// ┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐+/// │ 1.parquet │ │ 2.parquet │ │ 3.parquet │+/// │ A: [1..100] │ │ A: [101..200] │ │ A: [201..300] │+/// │ Sort: A, B, C │ │ Sort: A, B, C │ │ Sort: A, B, C │+/// └──────────────────┘ └──────────────────┘ └──────────────────┘+/// max(1) <= min(2) ✓ max(2) <= min(3) ✓ → output_ordering preserved+/// ```+pub(crate) fn get_projected_output_ordering(+ base_config: &FileScanConfig,+ projected_schema: &SchemaRef,+) -> Vec<LexOrdering> {+ let projected_orderings =+ project_orderings(&base_config.output_ordering, projected_schema);++ let indices = base_config+ .file_source+ .projection()+ .as_ref()+ .map(|p| ordered_column_indices_from_projection(p));++ match indices {+ Some(Some(indices)) => {+ // Simple column projection — validate with statistics+ validate_orderings(+ &projected_orderings,+ projected_schema,+ &base_config.file_groups,+ Some(indices.as_slice()),+ )+ }+ None => {+ // No projection — validate with statistics (no remapping needed)+ validate_orderings(+ &projected_orderings,+ projected_schema,+ &base_config.file_groups,+ None,+ )+ }+ Some(None) => {+ // Complex projection (expressions, not simple columns) — can't+ // determine column indices for statistics. Still valid if all+ // file groups have at most one file.+ if base_config.file_groups.iter().all(|g| g.len() <= 1) {+ projected_orderings+ } else {+ debug!(+ "Skipping specified output orderings. \+ Some file groups couldn't be determined to be sorted: {:?}",+ base_config.file_groups+ );+ vec![]+ }+ }+ }+}