Skip to content

Apache DataFusion SQL Query Engine

fix: include nulls in file ordering bounds (#26280)

Patrick Rank04c02c58085e
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 3aa68826cb..07144d8191 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -2184,6 +2184,9 @@ mod tests {
.map(|stats| {
stats
.map(|(min, max)| ColumnStatistics {
+ null_count: Precision::Exact(usize::from(
+ min.is_none() || max.is_none(),
+ )),
min_value: Precision::Exact(ScalarValue::Float64(
min,
)),
@@ -3826,6 +3829,114 @@ mod tests {
))
}
+ #[test]
+ fn file_group_ordering_includes_nulls() -> Result<()> {
+ // Arrange
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
+ for (descending, nulls_first) in
+ [(false, false), (false, true), (true, false), (true, true)]
+ {
+ let sort = PhysicalSortExpr::new(
+ Arc::new(Column::new("a", 0)),
+ arrow::compute::SortOptions {
+ descending,
+ nulls_first,
+ },
+ );
+ for null_count in [
+ Precision::Exact(0),
+ Precision::Exact(1),
+ Precision::Absent,
+ Precision::Inexact(0),
+ ] {
+ let mut files = vec![
+ make_file_with_null_stats("low", 0.0, 9.0, 0),
+ make_file_with_null_stats("high", 10.0, 19.0, 0),
+ ];
+ for file in &mut files {
+ Arc::make_mut(file.statistics.as_mut().unwrap()).column_statistics
+ [0]
+ .null_count = null_count;
+ }
+ if descending {
+ files.reverse();
+ }
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::local_filesystem(),
+ Arc::new(MockSource::new(TableSchema::from(&schema))),
+ )
+ .with_file_groups(vec![FileGroup::new(files)])
+ .with_output_ordering(vec![[sort.clone()].into()])
+ .build();
+
+ // Act
+ let ordered = config.eq_properties().ordering_satisfy([sort.clone()])?;
+
+ // Assert
+ assert_eq!(
+ ordered,
+ null_count == Precision::Exact(0),
+ "descending={descending}, nulls_first={nulls_first}, null_count={null_count:?}"
+ );
+ }
+ }
+ Ok(())
+ }
+
+ #[test]
+ fn file_group_ordering_with_all_null_or_empty_file() -> Result<()> {
+ // Arrange
+ let schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));
+ for (descending, nulls_first) in
+ [(false, false), (false, true), (true, false), (true, true)]
+ {
+ let sort = PhysicalSortExpr::new(
+ Arc::new(Column::new("a", 0)),
+ arrow::compute::SortOptions {
+ descending,
+ nulls_first,
+ },
+ );
+ for (num_rows, file_first) in [(0, false), (0, true), (2, false), (2, true)] {
+ let file = PartitionedFile::new("nulls_or_empty", 1024).with_statistics(
+ Arc::new(Statistics {
+ num_rows: Precision::Exact(num_rows),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Exact(num_rows),
+ ..Default::default()
+ }],
+ ..Default::default()
+ }),
+ );
+ let mut files =
+ vec![file, make_file_with_null_stats("non_null", 1.0, 2.0, 0)];
+ if !file_first {
+ files.reverse();
+ }
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::local_filesystem(),
+ Arc::new(MockSource::new(TableSchema::from(&schema))),
+ )
+ .with_file_groups(vec![FileGroup::new(files)])
+ .with_output_ordering(vec![[sort.clone()].into()])
+ .build();
+
+ // Act
+ let ordered = config.eq_properties().ordering_satisfy([sort.clone()])?;
+
+ // Assert
+ assert_eq!(
+ ordered,
+ num_rows == 0 || file_first == nulls_first,
+ "descending={descending}, nulls_first={nulls_first}, num_rows={num_rows}, file_first={file_first}"
+ );
+ }
+ }
+ Ok(())
+ }
+
#[test]
fn sort_pushdown_unsupported_with_nulls_does_not_upgrade_to_exact() -> Result<()> {
// Files are non-overlapping but one has NULLs.
diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rs
index 48d180364f..11349b77e4 100644
--- a/datafusion/datasource/src/statistics.rs
+++ b/datafusion/datasource/src/statistics.rs
@@ -26,10 +26,10 @@ use crate::PartitionedFile;
use crate::file_groups::FileGroup;
use arrow::array::RecordBatch;
-use arrow::compute::SortColumn;
-use arrow::datatypes::SchemaRef;
+use arrow::compute::{SortColumn, SortOptions};
+use arrow::datatypes::{Field, SchemaRef};
use arrow::row::{Row, Rows};
-use datafusion_common::stats::NdvFallback;
+use datafusion_common::stats::{NdvFallback, Precision};
use datafusion_common::{
DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,
};
@@ -83,16 +83,46 @@ impl MinMaxStatistics {
};
// Helper function to get min/max statistics for a given column of projected_schema
- let get_min_max = |i: usize| -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
+ let get_min_max = |i: usize,
+ field: &Field,
+ options: SortOptions|
+ -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {
+ let null = ScalarValue::try_new_null(field.data_type())?;
Ok(statistics_and_partition_values
.iter()
.map(|(s, pv)| {
if i < s.column_statistics.len() {
- s.column_statistics[i]
- .min_value
- .get_value()
- .cloned()
- .zip(s.column_statistics[i].max_value.get_value().cloned())
+ let column_statistics = &s.column_statistics[i];
+ let all_null = matches!(
+ (s.num_rows, column_statistics.null_count),
+ (Precision::Exact(rows), Precision::Exact(nulls))
+ if rows > 0 && rows == nulls
+ );
+ let may_have_nulls = match column_statistics.null_count {
+ Precision::Exact(nulls) => nulls > 0,
+ _ => field.is_nullable(),
+ };
+ // Self::new swaps the bounds for descending sort keys.
+ let nulls_on_min = options.nulls_first != options.descending;
+ let min = if all_null || (may_have_nulls && nulls_on_min) {
+ Some(null.clone())
+ } else {
+ column_statistics
+ .min_value
+ .get_value()
+ .filter(|value| !value.is_null())
+ .cloned()
+ };
+ let max = if all_null || (may_have_nulls && !nulls_on_min) {
+ Some(null.clone())
+ } else {
+ column_statistics
+ .max_value
+ .get_value()
+ .filter(|value| !value.is_null())
+ .cloned()
+ };
+ min.zip(max)
.ok_or_else(|| plan_datafusion_err!("statistics not found"))
} else if let Some(partition_value) =
pv.get(i - s.column_statistics.len())
@@ -135,7 +165,8 @@ impl MinMaxStatistics {
let (min_values, max_values): (Vec<_>, Vec<_>) = sort_columns
.iter()
- .map(|c| {
+ .zip(projected_sort_order.iter())
+ .map(|(c, sort_expr)| {
// Reverse the projection to get the index of the column in the full statistics
// The file statistics contains _every_ column , but the sort column's index()
// refers to the index in projected_schema
@@ -143,9 +174,11 @@ impl MinMaxStatistics {
.map(|p| p[c.index()])
.unwrap_or_else(|| c.index());
- let (min, max) = get_min_max(i).map_err(|e| {
- e.context(format!("get min/max for column: '{}'", c.name()))
- })?;
+ let (min, max) =
+ get_min_max(i, projected_schema.field(c.index()), sort_expr.options)
+ .map_err(|e| {
+ e.context(format!("get min/max for column: '{}'", c.name()))
+ })?;
Ok((
ScalarValue::iter_to_array(min)?,
ScalarValue::iter_to_array(max)?,