Apache DataFusion SQL Query Engine
chore: remove get_statistics_with_limit (#24549)
Yongting You7538aefbe9ee
diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rsindex 4218823026..48d180364f 100644--- a/datafusion/datasource/src/statistics.rs+++ b/datafusion/datasource/src/statistics.rs@@ -29,15 +29,13 @@ use arrow::array::RecordBatch;use arrow::compute::SortColumn;use arrow::datatypes::SchemaRef;use arrow::row::{Row, Rows};-use datafusion_common::stats::{NdvFallback, Precision};+use datafusion_common::stats::NdvFallback;use datafusion_common::{DataFusionError, Result, ScalarValue, plan_datafusion_err, plan_err,};use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};-use datafusion_physical_plan::{ColumnStatistics, Statistics};--use futures::{Stream, StreamExt};+use datafusion_physical_plan::Statistics;/// A normalized representation of file min/max statistics that allows for efficient sorting & comparison./// The min/max values are ordered by [`Self::sort_order`].@@ -290,172 +288,6 @@ fn sort_columns_from_physical_sort_exprs(.collect()}-fn seed_summary_statistics(summary_statistics: &mut Statistics, file_stats: &Statistics) {- summary_statistics.num_rows = file_stats.num_rows;- summary_statistics.total_byte_size = file_stats.total_byte_size;-- for (summary_col_stats, file_col_stats) in summary_statistics- .column_statistics- .iter_mut()- .zip(file_stats.column_statistics.iter())- {- summary_col_stats.null_count = file_col_stats.null_count;- summary_col_stats.max_value = file_col_stats.max_value.clone();- summary_col_stats.min_value = file_col_stats.min_value.clone();- summary_col_stats.sum_value = file_col_stats.sum_value.cast_to_sum_type();- summary_col_stats.byte_size = file_col_stats.byte_size;- }-}--fn merge_summary_statistics(- summary_statistics: &mut Statistics,- file_stats: &Statistics,-) {- summary_statistics.num_rows = summary_statistics.num_rows.add(&file_stats.num_rows);- summary_statistics.total_byte_size = summary_statistics- .total_byte_size- .add(&file_stats.total_byte_size);-- for (summary_col_stats, file_col_stats) in summary_statistics- .column_statistics- .iter_mut()- .zip(file_stats.column_statistics.iter())- {- let ColumnStatistics {- null_count: file_nc,- max_value: file_max,- min_value: file_min,- sum_value: file_sum,- distinct_count: _,- byte_size: file_sbs,- } = file_col_stats;-- summary_col_stats.null_count = summary_col_stats.null_count.add(file_nc);- summary_col_stats.max_value = summary_col_stats.max_value.max(file_max);- summary_col_stats.min_value = summary_col_stats.min_value.min(file_min);- summary_col_stats.sum_value = summary_col_stats.sum_value.add_for_sum(file_sum);- summary_col_stats.byte_size = summary_col_stats.byte_size.add(file_sbs);- }-}--fn seed_first_file_statistics(- limit_num_rows: &mut Precision<usize>,- summary_statistics: &mut Statistics,- file_stats: &Statistics,- collect_stats: bool,-) {- *limit_num_rows = file_stats.num_rows;-- if collect_stats {- seed_summary_statistics(summary_statistics, file_stats);- }-}--fn merge_file_statistics(- limit_num_rows: &mut Precision<usize>,- summary_statistics: &mut Statistics,- file_stats: &Statistics,- collect_stats: bool,-) {- *limit_num_rows = limit_num_rows.add(&file_stats.num_rows);-- if collect_stats {- merge_summary_statistics(summary_statistics, file_stats);- }-}--/// Get all files as well as the file level summary statistics (no statistic for partition columns).-/// If the optional `limit` is provided, includes only sufficient files. Needed to read up to-/// `limit` number of rows. `collect_stats` is passed down from the configuration parameter on-/// `ListingTable`. If it is false we only construct bare statistics and skip a potentially expensive-/// call to `multiunzip` for constructing file level summary statistics.-#[deprecated(- since = "47.0.0",- note = "Please use `get_files_with_limit` and `compute_all_files_statistics` instead"-)]-#[cfg_attr(not(test), expect(unused))]-pub async fn get_statistics_with_limit(- all_files: impl Stream<Item = Result<(PartitionedFile, Arc<Statistics>)>>,- file_schema: SchemaRef,- limit: Option<usize>,- collect_stats: bool,-) -> Result<(FileGroup, Statistics)> {- let mut result_files = FileGroup::default();- // These statistics can be calculated as long as at least one file provides- // useful information. If none of the files provides any information, then- // they will end up having `Precision::Absent` values. Throughout calculations,- // missing values will be imputed as:- // - zero for summations, and- // - neutral element for extreme points.- let size = file_schema.fields().len();- let mut summary_statistics = Statistics {- num_rows: Precision::Absent,- total_byte_size: Precision::Absent,- column_statistics: vec![ColumnStatistics::default(); size],- };- // Keep limit pruning separate from the returned summary so `collect_stats=false`- // can still stop early using known file row counts.- let mut limit_num_rows = Precision::<usize>::Absent;-- // Fusing the stream allows us to call next safely even once it is finished.- let mut all_files = Box::pin(all_files.fuse());-- if let Some(first_file) = all_files.next().await {- let (mut file, file_stats) = first_file?;- file.statistics = Some(Arc::clone(&file_stats));- result_files.push(file);-- seed_first_file_statistics(- &mut limit_num_rows,- &mut summary_statistics,- &file_stats,- collect_stats,- );-- // If the number of rows exceeds the limit, we can stop processing- // files. This only applies when we know the number of rows. It also- // currently ignores tables that have no statistics regarding the- // number of rows.- let conservative_num_rows = match limit_num_rows {- Precision::Exact(nr) => nr,- _ => usize::MIN,- };- if conservative_num_rows <= limit.unwrap_or(usize::MAX) {- while let Some(current) = all_files.next().await {- let (mut file, file_stats) = current?;- file.statistics = Some(Arc::clone(&file_stats));- result_files.push(file);- merge_file_statistics(- &mut limit_num_rows,- &mut summary_statistics,- &file_stats,- collect_stats,- );-- // If the number of rows exceeds the limit, we can stop processing- // files. This only applies when we know the number of rows. It also- // currently ignores tables that have no statistics regarding the- // number of rows.- if limit_num_rows.get_value().unwrap_or(&usize::MIN)- > &limit.unwrap_or(usize::MAX)- {- break;- }- }- }- };-- let mut statistics = summary_statistics;- if all_files.next().await.is_some() {- // If we still have files in the stream, it means that the limit kicked- // in, and the statistic could have been different had we processed the- // files in a different order.- statistics = statistics.to_inexact()- }-- Ok((result_files, statistics))-}-/// Computes the summary statistics for a group of files(`FileGroup` level's statistics).////// This function combines statistics from all files in the file group to create@@ -553,56 +385,13 @@ mod tests {use crate::PartitionedFile;use crate::file_groups::FileGroup;use arrow::datatypes::{DataType, Field, Schema};- use futures::stream;-- fn file_stats(sum: u32) -> Statistics {- Statistics {- num_rows: Precision::Exact(1),- total_byte_size: Precision::Exact(4),- column_statistics: vec![ColumnStatistics {- null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),- min_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),- sum_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),- distinct_count: Precision::Exact(1),- byte_size: Precision::Exact(4),- }],- }- }+ use datafusion_common::stats::Precision;+ use datafusion_physical_plan::ColumnStatistics;fn test_schema() -> SchemaRef {Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, true)]))}- fn make_file_stats(- num_rows: usize,- total_byte_size: usize,- col_stats: ColumnStatistics,- ) -> Arc<Statistics> {- Arc::new(Statistics {- num_rows: Precision::Exact(num_rows),- total_byte_size: Precision::Exact(total_byte_size),- column_statistics: vec![col_stats],- })- }-- fn rich_col_stats(- null_count: usize,- min: i64,- max: i64,- sum: i64,- byte_size: usize,- ) -> ColumnStatistics {- ColumnStatistics {- null_count: Precision::Exact(null_count),- max_value: Precision::Exact(ScalarValue::Int64(Some(max))),- min_value: Precision::Exact(ScalarValue::Int64(Some(min))),- distinct_count: Precision::Absent,- sum_value: Precision::Exact(ScalarValue::Int64(Some(sum))),- byte_size: Precision::Exact(byte_size),- }- }-fn utf8_file_stats(ndv: usize, min: &str, max: &str) -> Statistics {Statistics {num_rows: Precision::Exact(1),@@ -621,204 +410,6 @@ mod tests {fn file_with_stats(path: &str, stats: Statistics) -> PartitionedFile {PartitionedFile::new(path, 1).with_statistics(Arc::new(stats))}- #[tokio::test]- #[expect(deprecated)]- async fn test_get_statistics_with_limit_casts_first_file_sum_to_sum_type()- -> Result<()> {- let schema =- Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));-- let files = stream::iter(vec![Ok((- PartitionedFile::new("f1.parquet", 1),- Arc::new(file_stats(100)),- ))]);-- let (_group, stats) =- get_statistics_with_limit(files, schema, None, true).await?;-- assert_eq!(- stats.column_statistics[0].sum_value,- Precision::Exact(ScalarValue::UInt64(Some(100)))- );-- Ok(())- }-- #[tokio::test]- #[expect(deprecated)]- async fn test_get_statistics_with_limit_merges_sum_with_unsigned_widening()- -> Result<()> {- let schema =- Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));-- let files = stream::iter(vec![- Ok((- PartitionedFile::new("f1.parquet", 1),- Arc::new(file_stats(100)),- )),- Ok((- PartitionedFile::new("f2.parquet", 1),- Arc::new(file_stats(200)),- )),- ]);-- let (_group, stats) =- get_statistics_with_limit(files, schema, None, true).await?;-- assert_eq!(- stats.column_statistics[0].sum_value,- Precision::Exact(ScalarValue::UInt64(Some(300)))- );-- Ok(())- }-- #[tokio::test]- #[expect(deprecated)]- async fn get_statistics_with_limit_collect_stats_false_returns_bare_statistics() {- let all_files = stream::iter(vec![- Ok((- PartitionedFile::new("first.parquet", 10),- make_file_stats(0, 0, rich_col_stats(1, 1, 9, 15, 64)),- )),- Ok((- PartitionedFile::new("second.parquet", 20),- make_file_stats(10, 100, rich_col_stats(2, 10, 99, 300, 128)),- )),- ]);-- let (_files, statistics) =- get_statistics_with_limit(all_files, test_schema(), None, false)- .await- .unwrap();-- assert_eq!(statistics.num_rows, Precision::Absent);- assert_eq!(statistics.total_byte_size, Precision::Absent);- assert_eq!(statistics.column_statistics.len(), 1);- assert_eq!(- statistics.column_statistics[0].null_count,- Precision::Absent- );- assert_eq!(statistics.column_statistics[0].max_value, Precision::Absent);- assert_eq!(statistics.column_statistics[0].min_value, Precision::Absent);- assert_eq!(statistics.column_statistics[0].sum_value, Precision::Absent);- assert_eq!(statistics.column_statistics[0].byte_size, Precision::Absent);- }-- #[tokio::test]- #[expect(deprecated)]- async fn get_statistics_with_limit_collect_stats_false_uses_row_counts_for_limit() {- let all_files = stream::iter(vec![- Ok((- PartitionedFile::new("first.parquet", 10),- make_file_stats(3, 30, rich_col_stats(1, 1, 9, 15, 64)),- )),- Ok((- PartitionedFile::new("second.parquet", 20),- make_file_stats(3, 30, rich_col_stats(2, 10, 99, 300, 128)),- )),- Ok((- PartitionedFile::new("third.parquet", 30),- make_file_stats(3, 30, rich_col_stats(0, 100, 199, 450, 256)),- )),- ]);-- let (files, statistics) =- get_statistics_with_limit(all_files, test_schema(), Some(4), false)- .await- .unwrap();-- assert_eq!(files.len(), 2);- assert_eq!(statistics.num_rows, Precision::Absent);- assert_eq!(statistics.total_byte_size, Precision::Absent);- }-- #[tokio::test]- #[expect(deprecated)]- async fn get_statistics_with_limit_collect_stats_true_aggregates_statistics() {- let all_files = stream::iter(vec![- Ok((- PartitionedFile::new("first.parquet", 10),- make_file_stats(5, 50, rich_col_stats(1, 1, 9, 15, 64)),- )),- Ok((- PartitionedFile::new("second.parquet", 20),- make_file_stats(10, 100, rich_col_stats(2, 10, 99, 300, 128)),- )),- ]);-- let (_files, statistics) =- get_statistics_with_limit(all_files, test_schema(), None, true)- .await- .unwrap();-- assert_eq!(statistics.num_rows, Precision::Exact(15));- assert_eq!(statistics.total_byte_size, Precision::Exact(150));- assert_eq!(- statistics.column_statistics[0].null_count,- Precision::Exact(3)- );- assert_eq!(- statistics.column_statistics[0].min_value,- Precision::Exact(ScalarValue::Int64(Some(1)))- );- assert_eq!(- statistics.column_statistics[0].max_value,- Precision::Exact(ScalarValue::Int64(Some(99)))- );- assert_eq!(- statistics.column_statistics[0].sum_value,- Precision::Exact(ScalarValue::Int64(Some(315)))- );- assert_eq!(- statistics.column_statistics[0].byte_size,- Precision::Exact(192)- );- }-- #[tokio::test]- #[expect(deprecated)]- async fn get_statistics_with_limit_collect_stats_true_limit_marks_inexact() {- let all_files = stream::iter(vec![- Ok((- PartitionedFile::new("first.parquet", 10),- make_file_stats(5, 50, rich_col_stats(0, 1, 5, 15, 64)),- )),- Ok((- PartitionedFile::new("second.parquet", 20),- make_file_stats(5, 50, rich_col_stats(1, 6, 10, 40, 64)),- )),- Ok((- PartitionedFile::new("third.parquet", 20),- make_file_stats(5, 50, rich_col_stats(2, 11, 15, 65, 64)),- )),- ]);-- let (files, statistics) =- get_statistics_with_limit(all_files, test_schema(), Some(8), true)- .await- .unwrap();-- assert_eq!(files.len(), 2);- assert_eq!(statistics.num_rows, Precision::Inexact(10));- assert_eq!(statistics.total_byte_size, Precision::Inexact(100));- assert_eq!(- statistics.column_statistics[0].min_value,- Precision::Inexact(ScalarValue::Int64(Some(1)))- );- assert_eq!(- statistics.column_statistics[0].max_value,- Precision::Inexact(ScalarValue::Int64(Some(10)))- );- assert_eq!(- statistics.column_statistics[0].sum_value,- Precision::Inexact(ScalarValue::Int64(Some(55)))- );- assert_eq!(- statistics.column_statistics[0].byte_size,- Precision::Inexact(128)- );- }#[test]fn test_compute_file_group_statistics_uses_max_ndv_fallback() -> Result<()> {