Apache DataFusion SQL Query Engine
fix: include nulls in file ordering bounds (#26280)
Patrick Rank04c02c58085e
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 3aa68826cb..07144d8191 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -2184,6 +2184,9 @@ mod tests {.map(|stats| {stats.map(|(min, max)| ColumnStatistics {+ null_count: Precision::Exact(usize::from(+ min.is_none() || max.is_none(),+ )),min_value: Precision::Exact(ScalarValue::Float64(min,)),@@ -3826,6 +3829,114 @@ mod tests {))}+ #[test]+ fn file_group_ordering_includes_nulls() -> Result<()> {+ // Arrange+ let schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));+ for (descending, nulls_first) in+ [(false, false), (false, true), (true, false), (true, true)]+ {+ let sort = PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ arrow::compute::SortOptions {+ descending,+ nulls_first,+ },+ );+ for null_count in [+ Precision::Exact(0),+ Precision::Exact(1),+ Precision::Absent,+ Precision::Inexact(0),+ ] {+ let mut files = vec![+ make_file_with_null_stats("low", 0.0, 9.0, 0),+ make_file_with_null_stats("high", 10.0, 19.0, 0),+ ];+ for file in &mut files {+ Arc::make_mut(file.statistics.as_mut().unwrap()).column_statistics+ [0]+ .null_count = null_count;+ }+ if descending {+ files.reverse();+ }+ let config = FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(MockSource::new(TableSchema::from(&schema))),+ )+ .with_file_groups(vec![FileGroup::new(files)])+ .with_output_ordering(vec![[sort.clone()].into()])+ .build();++ // Act+ let ordered = config.eq_properties().ordering_satisfy([sort.clone()])?;++ // Assert+ assert_eq!(+ ordered,+ null_count == Precision::Exact(0),+ "descending={descending}, nulls_first={nulls_first}, null_count={null_count:?}"+ );+ }+ }+ Ok(())+ }++ #[test]+ fn file_group_ordering_with_all_null_or_empty_file() -> Result<()> {+ // Arrange+ let schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Float64, true)]));+ for (descending, nulls_first) in+ [(false, false), (false, true), (true, false), (true, true)]+ {+ let sort = PhysicalSortExpr::new(+ Arc::new(Column::new("a", 0)),+ arrow::compute::SortOptions {+ descending,+ nulls_first,+ },+ );+ for (num_rows, file_first) in [(0, false), (0, true), (2, false), (2, true)] {+ let file = PartitionedFile::new("nulls_or_empty", 1024).with_statistics(+ Arc::new(Statistics {+ num_rows: Precision::Exact(num_rows),+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Exact(num_rows),+ ..Default::default()+ }],+ ..Default::default()+ }),+ );+ let mut files =+ vec![file, make_file_with_null_stats("non_null", 1.0, 2.0, 0)];+ if !file_first {+ files.reverse();+ }+ let config = FileScanConfigBuilder::new(+ ObjectStoreUrl::local_filesystem(),+ Arc::new(MockSource::new(TableSchema::from(&schema))),+ )+ .with_file_groups(vec![FileGroup::new(files)])+ .with_output_ordering(vec![[sort.clone()].into()])+ .build();++ // Act+ let ordered = config.eq_properties().ordering_satisfy([sort.clone()])?;++ // Assert+ assert_eq!(+ ordered,+ num_rows == 0 || file_first == nulls_first,+ "descending={descending}, nulls_first={nulls_first}, num_rows={num_rows}, file_first={file_first}"+ );+ }+ }+ Ok(())+ }+#[test]fn sort_pushdown_unsupported_with_nulls_does_not_upgrade_to_exact() -> Result<()> {// Files are non-overlapping but one has NULLs.diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rsindex 48d180364f..11349b77e4 100644--- a/datafusion/datasource/src/statistics.rs+++ b/datafusion/datasource/src/statistics.rs@@ -26,10 +26,10 @@ use crate::PartitionedFile;use crate::file_groups::FileGroup;use arrow::array::RecordBatch;-use arrow::compute::SortColumn;-use arrow::datatypes::SchemaRef;+use arrow::compute::{SortColumn, SortOptions};+use arrow::datatypes::{Field, SchemaRef};use arrow::row::{Row, Rows};-use datafusion_common::stats::NdvFallback;+use datafusion_common::stats::{NdvFallback, Precision};use datafusion_common::{DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,};@@ -83,16 +83,46 @@ impl MinMaxStatistics {};// Helper function to get min/max statistics for a given column of projected_schema- let get_min_max = |i: usize| -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {+ let get_min_max = |i: usize,+ field: &Field,+ options: SortOptions|+ -> Result<(Vec<ScalarValue>, Vec<ScalarValue>)> {+ let null = ScalarValue::try_new_null(field.data_type())?;Ok(statistics_and_partition_values.iter().map(|(s, pv)| {if i < s.column_statistics.len() {- s.column_statistics[i]- .min_value- .get_value()- .cloned()- .zip(s.column_statistics[i].max_value.get_value().cloned())+ let column_statistics = &s.column_statistics[i];+ let all_null = matches!(+ (s.num_rows, column_statistics.null_count),+ (Precision::Exact(rows), Precision::Exact(nulls))+ if rows > 0 && rows == nulls+ );+ let may_have_nulls = match column_statistics.null_count {+ Precision::Exact(nulls) => nulls > 0,+ _ => field.is_nullable(),+ };+ // Self::new swaps the bounds for descending sort keys.+ let nulls_on_min = options.nulls_first != options.descending;+ let min = if all_null || (may_have_nulls && nulls_on_min) {+ Some(null.clone())+ } else {+ column_statistics+ .min_value+ .get_value()+ .filter(|value| !value.is_null())+ .cloned()+ };+ let max = if all_null || (may_have_nulls && !nulls_on_min) {+ Some(null.clone())+ } else {+ column_statistics+ .max_value+ .get_value()+ .filter(|value| !value.is_null())+ .cloned()+ };+ min.zip(max).ok_or_else(|| plan_datafusion_err!("statistics not found"))} else if let Some(partition_value) =pv.get(i - s.column_statistics.len())@@ -135,7 +165,8 @@ impl MinMaxStatistics {let (min_values, max_values): (Vec<_>, Vec<_>) = sort_columns.iter()- .map(|c| {+ .zip(projected_sort_order.iter())+ .map(|(c, sort_expr)| {// Reverse the projection to get the index of the column in the full statistics// The file statistics contains _every_ column , but the sort column's index()// refers to the index in projected_schema@@ -143,9 +174,11 @@ impl MinMaxStatistics {.map(|p| p[c.index()]).unwrap_or_else(|| c.index());- let (min, max) = get_min_max(i).map_err(|e| {- e.context(format!("get min/max for column: '{}'", c.name()))- })?;+ let (min, max) =+ get_min_max(i, projected_schema.field(c.index()), sort_expr.options)+ .map_err(|e| {+ e.context(format!("get min/max for column: '{}'", c.name()))+ })?;Ok((ScalarValue::iter_to_array(min)?,ScalarValue::iter_to_array(max)?,