Apache DataFusion SQL Query Engine
fix: ignore exact empty files in ordering analysis (#24648)
xudong.w221523813a5f
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rsindex eb5480dde6..6c294fe077 100644--- a/datafusion/catalog-listing/src/table.rs+++ b/datafusion/catalog-listing/src/table.rs@@ -23,7 +23,7 @@ use crate::{ListingOptions, ListingTableConfig};use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef};use async_trait::async_trait;use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};-use datafusion_common::stats::Precision;+use datafusion_common::stats::{Precision, is_known_empty};use datafusion_common::{Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err,project_schema,@@ -400,6 +400,9 @@ fn derive_common_ordering_from_files(file_groups: &[FileGroup]) -> Option<LexOrd// Collect file orderings and track countsfor group in file_groups {for file in group.iter() {+ if file.statistics.as_deref().is_some_and(is_known_empty) {+ continue;+ }state = match (&state, &file.ordering) {// If this is the first file with ordering, set it as current(CurrentOrderingState::FirstFile, Some(ordering)) => {@@ -1237,6 +1240,14 @@ mod tests {PartitionedFile::new(name.to_string(), 1024).with_ordering(ordering)}+ /// Helper to create an exact zero-row file with optional ordering+ fn create_empty_file(name: &str, ordering: Option<LexOrdering>) -> PartitionedFile {+ create_file(name, ordering).with_statistics(Arc::new(Statistics {+ num_rows: Precision::Exact(0),+ ..Default::default()+ }))+ }+#[test]fn test_derive_common_ordering_all_files_same_ordering() {// All files have the same ordering -> returns that ordering@@ -1308,6 +1319,19 @@ mod tests {assert_eq!(result, None);}+ #[test]+ fn test_derive_common_ordering_ignores_empty_files() {+ let ordering = lex_ordering(vec![sort_expr("a", 0, false, true)]);++ let file_groups = vec![FileGroup::new(vec![+ create_empty_file("empty.parquet", None),+ create_file("data.parquet", Some(ordering.clone())),+ ])];++ let result = derive_common_ordering_from_files(&file_groups);+ assert_eq!(result, Some(ordering));+ }+#[test]fn test_derive_common_ordering_all_none() {// No files have ordering -> returns Nonediff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rsindex 1a226c3698..1df71e15a2 100644--- a/datafusion/common/src/stats.rs+++ b/datafusion/common/src/stats.rs@@ -382,6 +382,13 @@ pub struct Statistics {pub column_statistics: Vec<ColumnStatistics>,}+/// Returns `true` when the statistics prove that the input contains no rows.+///+/// Inexact or absent row counts are not sufficient to treat an input as empty.+pub fn is_known_empty(statistics: &Statistics) -> bool {+ statistics.num_rows == Precision::Exact(0)+}+/// Fallback to use when NDV overlap can not be estimated from column bounds.#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]pub enum NdvFallback {@@ -3376,4 +3383,17 @@ mod tests {stats.calculate_total_byte_size(&non_primitive_schema);assert_eq!(stats.total_byte_size, Precision::Inexact(999));}++ #[test]+ fn test_is_known_empty() {+ let statistics = |num_rows| Statistics {+ num_rows,+ ..Default::default()+ };++ assert!(is_known_empty(&statistics(Precision::Exact(0))));+ assert!(!is_known_empty(&statistics(Precision::Inexact(0))));+ assert!(!is_known_empty(&statistics(Precision::Exact(1))));+ assert!(!is_known_empty(&statistics(Precision::Absent)));+ }}diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 73f8b8afa1..4e72b5e83b 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -48,7 +48,7 @@ use datafusion_execution::{use datafusion_expr::Operator;use crate::source::OpenArgs;-use datafusion_common::stats::Precision;+use datafusion_common::stats::{Precision, is_known_empty};use datafusion_physical_expr::expressions::{BinaryExpr, Column};use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};use datafusion_physical_expr::utils::reassign_expr_columns;@@ -1237,6 +1237,31 @@ impl DataSource for FileScanConfig {}}+/// Separates files that require statistics reasoning from files proven empty.+///+/// The latter must remain in the scan even though they cannot affect ordering.+fn partition_known_empty_files<'a>(+ files: impl IntoIterator<Item = &'a PartitionedFile>,+) -> (Vec<&'a PartitionedFile>, Vec<&'a PartitionedFile>) {+ files+ .into_iter()+ .partition(|file| !file.statistics.as_deref().is_some_and(is_known_empty))+}++/// Appends empty files to the smallest groups without affecting their row order.+fn append_known_empty_files(+ file_groups: &mut [FileGroup],+ empty_files: Vec<&PartitionedFile>,+) {+ for file in empty_files {+ file_groups+ .iter_mut()+ .min_by_key(|group| group.len())+ .expect("non-empty files must create at least one file group")+ .push((*file).clone());+ }+}+impl FileScanConfig {/// Returns only the output orderings that are validated against actual/// file group statistics.@@ -1417,11 +1442,16 @@ impl FileScanConfig {return Ok(vec![]);}+ let (files, empty_files) = partition_known_empty_files(flattened_files);+ if files.is_empty() {+ return Ok(file_groups.to_vec());+ }+let statistics = MinMaxStatistics::new_from_files(sort_order,table_schema,None,- flattened_files.iter().copied(),+ files.iter().copied(),)?;let indices_sorted_by_min = statistics.min_values_sorted();@@ -1452,17 +1482,20 @@ impl FileScanConfig {file_groups_indices.retain(|group| !group.is_empty());// Assemble indices back into groups of PartitionedFiles- Ok(file_groups_indices+ let mut file_groups = file_groups_indices.into_iter().map(|file_group_indices| {FileGroup::new(file_group_indices.into_iter()- .map(|idx| flattened_files[idx].clone())+ .map(|idx| files[idx].clone()).collect(),)})- .collect())+ .collect::<Vec<_>>();++ append_known_empty_files(&mut file_groups, empty_files);+ Ok(file_groups)}/// Attempts to do a bin-packing on files into file groups, such that any two files@@ -1492,11 +1525,16 @@ impl FileScanConfig {return Ok(vec![]);}+ let (files, empty_files) = partition_known_empty_files(flattened_files);+ if files.is_empty() {+ return Ok(file_groups.to_vec());+ }+let statistics = MinMaxStatistics::new_from_files(sort_order,table_schema,None,- flattened_files.iter().copied(),+ files.iter().copied(),).map_err(|e| {e.context("construct min/max statistics for split_groups_by_statistics")@@ -1522,15 +1560,18 @@ impl FileScanConfig {}// Assemble indices back into groups of PartitionedFiles- Ok(file_groups_indices+ let mut file_groups = file_groups_indices.into_iter().map(|file_group_indices| {file_group_indices.into_iter()- .map(|idx| flattened_files[idx].clone())- .collect()+ .map(|idx| files[idx].clone())+ .collect::<FileGroup>()})- .collect())+ .collect::<Vec<_>>();++ append_known_empty_files(&mut file_groups, empty_files);+ Ok(file_groups)}/// Write the data_type based on file_source@@ -2703,6 +2744,78 @@ mod tests {Ok(())}+ #[test]+ fn split_groups_by_statistics_preserves_exact_empty_files() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Float64,+ false,+ )]));+ let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(+ Column::new("value", 0),+ ))])+ .unwrap();++ let mixed_files = vec![FileGroup::new(vec![+ make_file_with_stats("high", 20.0, 29.0),+ make_exact_empty_file("empty"),+ make_file_with_stats("low", 0.0, 9.0),+ ])];+ let split = FileScanConfig::split_groups_by_statistics(+ &schema,+ &mixed_files,+ &sort_order,+ )?;+ let split_with_target =+ FileScanConfig::split_groups_by_statistics_with_target_partitions(+ &schema,+ &mixed_files,+ &sort_order,+ 2,+ )?;++ for result in [split, split_with_target] {+ assert!(verify_sort_integrity(&result));+ let mut file_names = result+ .iter()+ .flat_map(FileGroup::iter)+ .map(|file| file.object_meta.location.to_string())+ .collect::<Vec<_>>();+ file_names.sort();+ assert_eq!(file_names, ["empty", "high", "low"]);+ assert!(result.iter().all(|group| !group.is_empty()));+ }++ let empty_files = vec![FileGroup::new(vec![+ make_exact_empty_file("empty1"),+ make_exact_empty_file("empty2"),+ ])];+ let split = FileScanConfig::split_groups_by_statistics(+ &schema,+ &empty_files,+ &sort_order,+ )?;+ let split_with_target =+ FileScanConfig::split_groups_by_statistics_with_target_partitions(+ &schema,+ &empty_files,+ &sort_order,+ 2,+ )?;++ for result in [split, split_with_target] {+ assert!(verify_sort_integrity(&result));+ assert_eq!(result.len(), 1);+ let file_names = result[0]+ .iter()+ .map(|file| file.object_meta.location.to_string())+ .collect::<Vec<_>>();+ assert_eq!(file_names, ["empty1", "empty2"]);+ }++ Ok(())+ }+#[test]fn test_partition_statistics_projection() {// This test verifies that partition_statistics applies projection correctly.@@ -3141,6 +3254,15 @@ mod tests {))}+ fn make_exact_empty_file(name: &str) -> PartitionedFile {+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(+ Statistics {+ num_rows: Precision::Exact(0),+ ..Default::default()+ },+ ))+ }+#[derive(Clone)]struct ExactSortPushdownSource {metrics: ExecutionPlanMetricsSet,diff --git a/datafusion/datasource/src/file_scan_config/sort_pushdown.rs b/datafusion/datasource/src/file_scan_config/sort_pushdown.rsindex 3f5beed20f..bf2c3444fc 100644--- a/datafusion/datasource/src/file_scan_config/sort_pushdown.rs+++ b/datafusion/datasource/src/file_scan_config/sort_pushdown.rs@@ -24,7 +24,7 @@//! Extracted from `file_scan_config.rs` to keep that module focused on//! core configuration and data-source plumbing.-use super::FileScanConfig;+use super::{FileScanConfig, partition_known_empty_files};use crate::file::FileSource;use crate::file_groups::FileGroup;use crate::source::DataSource;@@ -32,7 +32,7 @@ use crate::statistics::MinMaxStatistics;use arrow::datatypes::SchemaRef;use datafusion_common::Result;-use datafusion_common::stats::Precision;+use datafusion_common::stats::{Precision, is_known_empty};use datafusion_physical_expr::equivalence::project_orderings;use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr::projection::ProjectionExprs;@@ -328,7 +328,12 @@ pub(crate) fn sort_files_within_groups_by_statistics(continue;}- let files: Vec<_> = group.iter().collect();+ let (files, empty_files) = partition_known_empty_files(group.iter());+ if files.len() <= 1 {+ new_groups.push(group.clone());+ confirmed_non_overlapping += 1;+ continue;+ }let statistics = match MinMaxStatistics::new_from_files(sort_order,@@ -353,17 +358,22 @@ pub(crate) fn sort_files_within_groups_by_statistics(.enumerate().all(|(pos, (idx, _))| pos == *idx);+ let sorted_files = sorted_indices+ .iter()+ .map(|(idx, _)| files[*idx])+ .collect::<Vec<_>>();+let sorted_group: FileGroup = if already_sorted {group.clone()} else {any_reordered = true;- sorted_indices+ sorted_files.iter()- .map(|(idx, _)| files[*idx].clone())+ .map(|file| (**file).clone())+ .chain(empty_files.iter().map(|file| (**file).clone())).collect()};- let sorted_files: Vec<_> = sorted_group.iter().collect();let is_non_overlapping = match MinMaxStatistics::new_from_files(sort_order,projected_schema,@@ -406,6 +416,9 @@ pub(crate) fn any_file_has_nulls_in_sort_columns(let Some(stats) = file.statistics.as_ref() else {return true; // No stats, assume nulls exist};+ if is_known_empty(stats) {+ continue;+ }for col in &sort_columns {let stat_idx = projection_indices.map(|p| p[col.index()])@@ -468,11 +481,11 @@ pub(crate) fn is_ordering_valid_for_file_groups(projection: Option<&[usize]>,) -> bool {file_groups.iter().all(|group| {- if group.len() <= 1 {+ let (files, _) = partition_known_empty_files(group.iter());+ if files.len() <= 1 {return true; // single-file groups are trivially sorted}- match MinMaxStatistics::new_from_files(ordering, schema, projection, group.iter())- {+ match MinMaxStatistics::new_from_files(ordering, schema, projection, files) {Ok(stats) => stats.is_sorted(),Err(_) => false, // can't prove sorted → reject}@@ -620,3 +633,86 @@ pub(crate) fn get_projected_output_ordering(}}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::PartitionedFile;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};++ fn file_with_stats(name: &str, min: f64, max: f64) -> PartitionedFile {+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(+ Statistics {+ num_rows: Precision::Exact(100),+ total_byte_size: Precision::Exact(1024),+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Exact(0),+ min_value: Precision::Exact(ScalarValue::Float64(Some(min))),+ max_value: Precision::Exact(ScalarValue::Float64(Some(max))),+ ..Default::default()+ }],+ },+ ))+ }++ fn exact_empty_file(name: &str) -> PartitionedFile {+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(+ Statistics {+ num_rows: Precision::Exact(0),+ ..Default::default()+ },+ ))+ }++ #[test]+ fn exact_empty_files_are_ignored_and_preserved() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "value",+ DataType::Float64,+ false,+ )]));+ let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(+ Column::new("value", 0),+ ))])+ .unwrap();+ let file_groups = vec![+ FileGroup::new(vec![+ file_with_stats("high", 20.0, 29.0),+ exact_empty_file("empty"),+ file_with_stats("low", 0.0, 9.0),+ ]),+ FileGroup::new(vec![exact_empty_file("empty1"), exact_empty_file("empty2")]),+ ];++ let sorted = sort_files_within_groups_by_statistics(+ &file_groups,+ &sort_order,+ &schema,+ None,+ );++ assert!(sorted.any_reordered);+ assert!(sorted.all_non_overlapping);+ assert_eq!(sorted.file_groups[0].len(), 3);+ assert_eq!(sorted.file_groups[1].len(), 2);+ let first_group_names = sorted.file_groups[0]+ .iter()+ .map(|file| file.object_meta.location.to_string())+ .collect::<Vec<_>>();+ assert_eq!(first_group_names, ["low", "high", "empty"]);++ assert!(!any_file_has_nulls_in_sort_columns(+ &sorted.file_groups,+ sort_order.as_ref(),+ &schema,+ None,+ ));+ assert!(is_ordering_valid_for_file_groups(+ &sorted.file_groups,+ &sort_order,+ &schema,+ None,+ ));+ }+}diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rsindex e415b3e48a..d0bf64d81c 100644--- a/datafusion/datasource/src/mod.rs+++ b/datafusion/datasource/src/mod.rs@@ -61,7 +61,7 @@ pub use self::url::ListingTableUrl;use crate::file_groups::FileGroup;use arrow::datatypes::SchemaRef;use chrono::TimeZone;-use datafusion_common::stats::Precision;+use datafusion_common::stats::{Precision, is_known_empty};use datafusion_common::{ColumnStatistics, Result, TableReference};use datafusion_common::{ScalarValue, Statistics};use datafusion_physical_expr::LexOrdering;@@ -504,7 +504,12 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGro/// Used by tests and benchmarkspub fn verify_sort_integrity(file_groups: &[FileGroup]) -> bool {for group in file_groups {- let files = group.iter().collect::<Vec<_>>();+ // Known-empty files contribute no rows and may not have min/max+ // statistics, so they cannot violate the ordering.+ let files = group+ .iter()+ .filter(|file| !file.statistics.as_deref().is_some_and(is_known_empty))+ .collect::<Vec<_>>();for i in 1..files.len() {let prev_file = files[i - 1];let curr_file = files[i];diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.sltindex b1e76a555d..30688539d8 100644--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt@@ -276,6 +276,70 @@ physical_plan01)SortPreservingMergeExec: [constant_col@0 ASC NULLS LAST]02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet]]}, projection=[constant_col], output_ordering=[constant_col@0 ASC NULLS LAST], file_type=parquet+# Add a zero-row file without ordering metadata or column min/max statistics.+query I+COPY (SELECT * FROM src_table WHERE FALSE)+TO 'test_files/scratch/parquet_sorted_statistics/test_table/partition_col=D/empty.parquet'+STORED AS PARQUET;+----+0++# Verify ordering can be inferred from the non-empty files while the empty file+# remains in the scan file groups.+statement ok+DROP TABLE test_table;++statement ok+CREATE EXTERNAL TABLE inferred_test_table (+ partition_col TEXT NOT NULL,+ int_col INT NOT NULL,+ descending_col INT NOT NULL,+ string_col TEXT NOT NULL,+ bigint_col BIGINT NOT NULL,+ date_col DATE NOT NULL,+ overlapping_col INT NOT NULL,+ constant_col INT NOT NULL,+ nulls_first_col INT,+ nulls_last_col INT+)+STORED AS PARQUET+PARTITIONED BY (partition_col)+LOCATION 'test_files/scratch/parquet_sorted_statistics/test_table';++query TT+EXPLAIN SELECT int_col+FROM inferred_test_table+ORDER BY int_col;+----+logical_plan+01)Sort: inferred_test_table.int_col ASC NULLS LAST+02)--TableScan: inferred_test_table projection=[int_col]+physical_plan+01)SortPreservingMergeExec: [int_col@0 ASC NULLS LAST]+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=D/empty.parquet]]}, projection=[int_col], output_ordering=[int_col@0 ASC NULLS LAST], file_type=parquet++query I rowsort+SELECT int_col FROM inferred_test_table;+----+1+2+3+4+5+6+7+8+9++# A partition filter can leave only the exact-empty file. Distribution-requiring+# operators must still receive a scan with a valid, non-zero partition count.+query II+SELECT int_col, count(*)+FROM inferred_test_table+WHERE partition_col = 'D'+GROUP BY int_col;+----+# Config reset# The SLT runner sets `target_partitions` to 4 instead of using the default, so