Skip to content

Apache DataFusion SQL Query Engine

fix: ignore exact empty files in ordering analysis (#24648)

xudong.w221523813a5f
diff --git a/datafusion/catalog-listing/src/table.rs b/datafusion/catalog-listing/src/table.rs
index eb5480dde6..6c294fe077 100644
--- a/datafusion/catalog-listing/src/table.rs
+++ b/datafusion/catalog-listing/src/table.rs
@@ -23,7 +23,7 @@ use crate::{ListingOptions, ListingTableConfig};
use arrow::datatypes::{Field, Schema, SchemaBuilder, SchemaRef};
use async_trait::async_trait;
use datafusion_catalog::{ScanArgs, ScanResult, Session, TableProvider};
-use datafusion_common::stats::Precision;
+use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_common::{
Constraints, DFSchema, SchemaExt, Statistics, internal_datafusion_err, plan_err,
project_schema,
@@ -400,6 +400,9 @@ fn derive_common_ordering_from_files(file_groups: &[FileGroup]) -> Option<LexOrd
// Collect file orderings and track counts
for group in file_groups {
for file in group.iter() {
+ if file.statistics.as_deref().is_some_and(is_known_empty) {
+ continue;
+ }
state = match (&state, &file.ordering) {
// If this is the first file with ordering, set it as current
(CurrentOrderingState::FirstFile, Some(ordering)) => {
@@ -1237,6 +1240,14 @@ mod tests {
PartitionedFile::new(name.to_string(), 1024).with_ordering(ordering)
}
+ /// Helper to create an exact zero-row file with optional ordering
+ fn create_empty_file(name: &str, ordering: Option<LexOrdering>) -> PartitionedFile {
+ create_file(name, ordering).with_statistics(Arc::new(Statistics {
+ num_rows: Precision::Exact(0),
+ ..Default::default()
+ }))
+ }
+
#[test]
fn test_derive_common_ordering_all_files_same_ordering() {
// All files have the same ordering -> returns that ordering
@@ -1308,6 +1319,19 @@ mod tests {
assert_eq!(result, None);
}
+ #[test]
+ fn test_derive_common_ordering_ignores_empty_files() {
+ let ordering = lex_ordering(vec![sort_expr("a", 0, false, true)]);
+
+ let file_groups = vec![FileGroup::new(vec![
+ create_empty_file("empty.parquet", None),
+ create_file("data.parquet", Some(ordering.clone())),
+ ])];
+
+ let result = derive_common_ordering_from_files(&file_groups);
+ assert_eq!(result, Some(ordering));
+ }
+
#[test]
fn test_derive_common_ordering_all_none() {
// No files have ordering -> returns None
diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rs
index 1a226c3698..1df71e15a2 100644
--- a/datafusion/common/src/stats.rs
+++ b/datafusion/common/src/stats.rs
@@ -382,6 +382,13 @@ pub struct Statistics {
pub column_statistics: Vec<ColumnStatistics>,
}
+/// Returns `true` when the statistics prove that the input contains no rows.
+///
+/// Inexact or absent row counts are not sufficient to treat an input as empty.
+pub fn is_known_empty(statistics: &Statistics) -> bool {
+ statistics.num_rows == Precision::Exact(0)
+}
+
/// Fallback to use when NDV overlap can not be estimated from column bounds.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum NdvFallback {
@@ -3376,4 +3383,17 @@ mod tests {
stats.calculate_total_byte_size(&non_primitive_schema);
assert_eq!(stats.total_byte_size, Precision::Inexact(999));
}
+
+ #[test]
+ fn test_is_known_empty() {
+ let statistics = |num_rows| Statistics {
+ num_rows,
+ ..Default::default()
+ };
+
+ assert!(is_known_empty(&statistics(Precision::Exact(0))));
+ assert!(!is_known_empty(&statistics(Precision::Inexact(0))));
+ assert!(!is_known_empty(&statistics(Precision::Exact(1))));
+ assert!(!is_known_empty(&statistics(Precision::Absent)));
+ }
}
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 73f8b8afa1..4e72b5e83b 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -48,7 +48,7 @@ use datafusion_execution::{
use datafusion_expr::Operator;
use crate::source::OpenArgs;
-use datafusion_common::stats::Precision;
+use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -1237,6 +1237,31 @@ impl DataSource for FileScanConfig {
}
}
+/// Separates files that require statistics reasoning from files proven empty.
+///
+/// The latter must remain in the scan even though they cannot affect ordering.
+fn partition_known_empty_files<'a>(
+ files: impl IntoIterator<Item = &'a PartitionedFile>,
+) -> (Vec<&'a PartitionedFile>, Vec<&'a PartitionedFile>) {
+ files
+ .into_iter()
+ .partition(|file| !file.statistics.as_deref().is_some_and(is_known_empty))
+}
+
+/// Appends empty files to the smallest groups without affecting their row order.
+fn append_known_empty_files(
+ file_groups: &mut [FileGroup],
+ empty_files: Vec<&PartitionedFile>,
+) {
+ for file in empty_files {
+ file_groups
+ .iter_mut()
+ .min_by_key(|group| group.len())
+ .expect("non-empty files must create at least one file group")
+ .push((*file).clone());
+ }
+}
+
impl FileScanConfig {
/// Returns only the output orderings that are validated against actual
/// file group statistics.
@@ -1417,11 +1442,16 @@ impl FileScanConfig {
return Ok(vec![]);
}
+ let (files, empty_files) = partition_known_empty_files(flattened_files);
+ if files.is_empty() {
+ return Ok(file_groups.to_vec());
+ }
+
let statistics = MinMaxStatistics::new_from_files(
sort_order,
table_schema,
None,
- flattened_files.iter().copied(),
+ files.iter().copied(),
)?;
let indices_sorted_by_min = statistics.min_values_sorted();
@@ -1452,17 +1482,20 @@ impl FileScanConfig {
file_groups_indices.retain(|group| !group.is_empty());
// Assemble indices back into groups of PartitionedFiles
- Ok(file_groups_indices
+ let mut file_groups = file_groups_indices
.into_iter()
.map(|file_group_indices| {
FileGroup::new(
file_group_indices
.into_iter()
- .map(|idx| flattened_files[idx].clone())
+ .map(|idx| files[idx].clone())
.collect(),
)
})
- .collect())
+ .collect::<Vec<_>>();
+
+ append_known_empty_files(&mut file_groups, empty_files);
+ Ok(file_groups)
}
/// Attempts to do a bin-packing on files into file groups, such that any two files
@@ -1492,11 +1525,16 @@ impl FileScanConfig {
return Ok(vec![]);
}
+ let (files, empty_files) = partition_known_empty_files(flattened_files);
+ if files.is_empty() {
+ return Ok(file_groups.to_vec());
+ }
+
let statistics = MinMaxStatistics::new_from_files(
sort_order,
table_schema,
None,
- flattened_files.iter().copied(),
+ files.iter().copied(),
)
.map_err(|e| {
e.context("construct min/max statistics for split_groups_by_statistics")
@@ -1522,15 +1560,18 @@ impl FileScanConfig {
}
// Assemble indices back into groups of PartitionedFiles
- Ok(file_groups_indices
+ let mut file_groups = file_groups_indices
.into_iter()
.map(|file_group_indices| {
file_group_indices
.into_iter()
- .map(|idx| flattened_files[idx].clone())
- .collect()
+ .map(|idx| files[idx].clone())
+ .collect::<FileGroup>()
})
- .collect())
+ .collect::<Vec<_>>();
+
+ append_known_empty_files(&mut file_groups, empty_files);
+ Ok(file_groups)
}
/// Write the data_type based on file_source
@@ -2703,6 +2744,78 @@ mod tests {
Ok(())
}
+ #[test]
+ fn split_groups_by_statistics_preserves_exact_empty_files() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Float64,
+ false,
+ )]));
+ let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(
+ Column::new("value", 0),
+ ))])
+ .unwrap();
+
+ let mixed_files = vec![FileGroup::new(vec![
+ make_file_with_stats("high", 20.0, 29.0),
+ make_exact_empty_file("empty"),
+ make_file_with_stats("low", 0.0, 9.0),
+ ])];
+ let split = FileScanConfig::split_groups_by_statistics(
+ &schema,
+ &mixed_files,
+ &sort_order,
+ )?;
+ let split_with_target =
+ FileScanConfig::split_groups_by_statistics_with_target_partitions(
+ &schema,
+ &mixed_files,
+ &sort_order,
+ 2,
+ )?;
+
+ for result in [split, split_with_target] {
+ assert!(verify_sort_integrity(&result));
+ let mut file_names = result
+ .iter()
+ .flat_map(FileGroup::iter)
+ .map(|file| file.object_meta.location.to_string())
+ .collect::<Vec<_>>();
+ file_names.sort();
+ assert_eq!(file_names, ["empty", "high", "low"]);
+ assert!(result.iter().all(|group| !group.is_empty()));
+ }
+
+ let empty_files = vec![FileGroup::new(vec![
+ make_exact_empty_file("empty1"),
+ make_exact_empty_file("empty2"),
+ ])];
+ let split = FileScanConfig::split_groups_by_statistics(
+ &schema,
+ &empty_files,
+ &sort_order,
+ )?;
+ let split_with_target =
+ FileScanConfig::split_groups_by_statistics_with_target_partitions(
+ &schema,
+ &empty_files,
+ &sort_order,
+ 2,
+ )?;
+
+ for result in [split, split_with_target] {
+ assert!(verify_sort_integrity(&result));
+ assert_eq!(result.len(), 1);
+ let file_names = result[0]
+ .iter()
+ .map(|file| file.object_meta.location.to_string())
+ .collect::<Vec<_>>();
+ assert_eq!(file_names, ["empty1", "empty2"]);
+ }
+
+ Ok(())
+ }
+
#[test]
fn test_partition_statistics_projection() {
// This test verifies that partition_statistics applies projection correctly.
@@ -3141,6 +3254,15 @@ mod tests {
))
}
+ fn make_exact_empty_file(name: &str) -> PartitionedFile {
+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(
+ Statistics {
+ num_rows: Precision::Exact(0),
+ ..Default::default()
+ },
+ ))
+ }
+
#[derive(Clone)]
struct ExactSortPushdownSource {
metrics: ExecutionPlanMetricsSet,
diff --git a/datafusion/datasource/src/file_scan_config/sort_pushdown.rs b/datafusion/datasource/src/file_scan_config/sort_pushdown.rs
index 3f5beed20f..bf2c3444fc 100644
--- a/datafusion/datasource/src/file_scan_config/sort_pushdown.rs
+++ b/datafusion/datasource/src/file_scan_config/sort_pushdown.rs
@@ -24,7 +24,7 @@
//! Extracted from `file_scan_config.rs` to keep that module focused on
//! core configuration and data-source plumbing.
-use super::FileScanConfig;
+use super::{FileScanConfig, partition_known_empty_files};
use crate::file::FileSource;
use crate::file_groups::FileGroup;
use crate::source::DataSource;
@@ -32,7 +32,7 @@ use crate::statistics::MinMaxStatistics;
use arrow::datatypes::SchemaRef;
use datafusion_common::Result;
-use datafusion_common::stats::Precision;
+use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_physical_expr::equivalence::project_orderings;
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::projection::ProjectionExprs;
@@ -328,7 +328,12 @@ pub(crate) fn sort_files_within_groups_by_statistics(
continue;
}
- let files: Vec<_> = group.iter().collect();
+ let (files, empty_files) = partition_known_empty_files(group.iter());
+ if files.len() <= 1 {
+ new_groups.push(group.clone());
+ confirmed_non_overlapping += 1;
+ continue;
+ }
let statistics = match MinMaxStatistics::new_from_files(
sort_order,
@@ -353,17 +358,22 @@ pub(crate) fn sort_files_within_groups_by_statistics(
.enumerate()
.all(|(pos, (idx, _))| pos == *idx);
+ let sorted_files = sorted_indices
+ .iter()
+ .map(|(idx, _)| files[*idx])
+ .collect::<Vec<_>>();
+
let sorted_group: FileGroup = if already_sorted {
group.clone()
} else {
any_reordered = true;
- sorted_indices
+ sorted_files
.iter()
- .map(|(idx, _)| files[*idx].clone())
+ .map(|file| (**file).clone())
+ .chain(empty_files.iter().map(|file| (**file).clone()))
.collect()
};
- let sorted_files: Vec<_> = sorted_group.iter().collect();
let is_non_overlapping = match MinMaxStatistics::new_from_files(
sort_order,
projected_schema,
@@ -406,6 +416,9 @@ pub(crate) fn any_file_has_nulls_in_sort_columns(
let Some(stats) = file.statistics.as_ref() else {
return true; // No stats, assume nulls exist
};
+ if is_known_empty(stats) {
+ continue;
+ }
for col in &sort_columns {
let stat_idx = projection_indices
.map(|p| p[col.index()])
@@ -468,11 +481,11 @@ pub(crate) fn is_ordering_valid_for_file_groups(
projection: Option<&[usize]>,
) -> bool {
file_groups.iter().all(|group| {
- if group.len() <= 1 {
+ let (files, _) = partition_known_empty_files(group.iter());
+ if files.len() <= 1 {
return true; // single-file groups are trivially sorted
}
- match MinMaxStatistics::new_from_files(ordering, schema, projection, group.iter())
- {
+ match MinMaxStatistics::new_from_files(ordering, schema, projection, files) {
Ok(stats) => stats.is_sorted(),
Err(_) => false, // can't prove sorted → reject
}
@@ -620,3 +633,86 @@ pub(crate) fn get_projected_output_ordering(
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::PartitionedFile;
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::{ColumnStatistics, ScalarValue, Statistics};
+
+ fn file_with_stats(name: &str, min: f64, max: f64) -> PartitionedFile {
+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(
+ Statistics {
+ num_rows: Precision::Exact(100),
+ total_byte_size: Precision::Exact(1024),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Exact(0),
+ min_value: Precision::Exact(ScalarValue::Float64(Some(min))),
+ max_value: Precision::Exact(ScalarValue::Float64(Some(max))),
+ ..Default::default()
+ }],
+ },
+ ))
+ }
+
+ fn exact_empty_file(name: &str) -> PartitionedFile {
+ PartitionedFile::new(name.to_string(), 1024).with_statistics(Arc::new(
+ Statistics {
+ num_rows: Precision::Exact(0),
+ ..Default::default()
+ },
+ ))
+ }
+
+ #[test]
+ fn exact_empty_files_are_ignored_and_preserved() {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "value",
+ DataType::Float64,
+ false,
+ )]));
+ let sort_order = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(
+ Column::new("value", 0),
+ ))])
+ .unwrap();
+ let file_groups = vec![
+ FileGroup::new(vec![
+ file_with_stats("high", 20.0, 29.0),
+ exact_empty_file("empty"),
+ file_with_stats("low", 0.0, 9.0),
+ ]),
+ FileGroup::new(vec![exact_empty_file("empty1"), exact_empty_file("empty2")]),
+ ];
+
+ let sorted = sort_files_within_groups_by_statistics(
+ &file_groups,
+ &sort_order,
+ &schema,
+ None,
+ );
+
+ assert!(sorted.any_reordered);
+ assert!(sorted.all_non_overlapping);
+ assert_eq!(sorted.file_groups[0].len(), 3);
+ assert_eq!(sorted.file_groups[1].len(), 2);
+ let first_group_names = sorted.file_groups[0]
+ .iter()
+ .map(|file| file.object_meta.location.to_string())
+ .collect::<Vec<_>>();
+ assert_eq!(first_group_names, ["low", "high", "empty"]);
+
+ assert!(!any_file_has_nulls_in_sort_columns(
+ &sorted.file_groups,
+ sort_order.as_ref(),
+ &schema,
+ None,
+ ));
+ assert!(is_ordering_valid_for_file_groups(
+ &sorted.file_groups,
+ &sort_order,
+ &schema,
+ None,
+ ));
+ }
+}
diff --git a/datafusion/datasource/src/mod.rs b/datafusion/datasource/src/mod.rs
index e415b3e48a..d0bf64d81c 100644
--- a/datafusion/datasource/src/mod.rs
+++ b/datafusion/datasource/src/mod.rs
@@ -61,7 +61,7 @@ pub use self::url::ListingTableUrl;
use crate::file_groups::FileGroup;
use arrow::datatypes::SchemaRef;
use chrono::TimeZone;
-use datafusion_common::stats::Precision;
+use datafusion_common::stats::{Precision, is_known_empty};
use datafusion_common::{ColumnStatistics, Result, TableReference};
use datafusion_common::{ScalarValue, Statistics};
use datafusion_physical_expr::LexOrdering;
@@ -504,7 +504,12 @@ pub fn generate_test_files(num_files: usize, overlap_factor: f64) -> Vec<FileGro
/// Used by tests and benchmarks
pub fn verify_sort_integrity(file_groups: &[FileGroup]) -> bool {
for group in file_groups {
- let files = group.iter().collect::<Vec<_>>();
+ // Known-empty files contribute no rows and may not have min/max
+ // statistics, so they cannot violate the ordering.
+ let files = group
+ .iter()
+ .filter(|file| !file.statistics.as_deref().is_some_and(is_known_empty))
+ .collect::<Vec<_>>();
for i in 1..files.len() {
let prev_file = files[i - 1];
let curr_file = files[i];
diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
index b1e76a555d..30688539d8 100644
--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt
@@ -276,6 +276,70 @@ physical_plan
01)SortPreservingMergeExec: [constant_col@0 ASC NULLS LAST]
02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet]]}, projection=[constant_col], output_ordering=[constant_col@0 ASC NULLS LAST], file_type=parquet
+# Add a zero-row file without ordering metadata or column min/max statistics.
+query I
+COPY (SELECT * FROM src_table WHERE FALSE)
+TO 'test_files/scratch/parquet_sorted_statistics/test_table/partition_col=D/empty.parquet'
+STORED AS PARQUET;
+----
+0
+
+# Verify ordering can be inferred from the non-empty files while the empty file
+# remains in the scan file groups.
+statement ok
+DROP TABLE test_table;
+
+statement ok
+CREATE EXTERNAL TABLE inferred_test_table (
+ partition_col TEXT NOT NULL,
+ int_col INT NOT NULL,
+ descending_col INT NOT NULL,
+ string_col TEXT NOT NULL,
+ bigint_col BIGINT NOT NULL,
+ date_col DATE NOT NULL,
+ overlapping_col INT NOT NULL,
+ constant_col INT NOT NULL,
+ nulls_first_col INT,
+ nulls_last_col INT
+)
+STORED AS PARQUET
+PARTITIONED BY (partition_col)
+LOCATION 'test_files/scratch/parquet_sorted_statistics/test_table';
+
+query TT
+EXPLAIN SELECT int_col
+FROM inferred_test_table
+ORDER BY int_col;
+----
+logical_plan
+01)Sort: inferred_test_table.int_col ASC NULLS LAST
+02)--TableScan: inferred_test_table projection=[int_col]
+physical_plan
+01)SortPreservingMergeExec: [int_col@0 ASC NULLS LAST]
+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=D/empty.parquet]]}, projection=[int_col], output_ordering=[int_col@0 ASC NULLS LAST], file_type=parquet
+
+query I rowsort
+SELECT int_col FROM inferred_test_table;
+----
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+# A partition filter can leave only the exact-empty file. Distribution-requiring
+# operators must still receive a scan with a valid, non-zero partition count.
+query II
+SELECT int_col, count(*)
+FROM inferred_test_table
+WHERE partition_col = 'D'
+GROUP BY int_col;
+----
+
# Config reset
# The SLT runner sets `target_partitions` to 4 instead of using the default, so