Apache DataFusion SQL Query Engine
fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)
Tobias Schwarzinger62cfc0ceae30
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 962df06302..d1dd3c11fc 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -39,6 +39,7 @@ use datafusion_execution::{use datafusion_expr::Operator;use crate::source::OpenArgs;+use datafusion_common::stats::Precision;use datafusion_physical_expr::expressions::{BinaryExpr, Column};use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};use datafusion_physical_expr::utils::reassign_expr_columns;@@ -1233,7 +1234,9 @@ impl FileScanConfig {/// we can't guarantee the statistics are exact because we don't know how many/// rows will be filtered out.pub fn statistics(&self) -> Statistics {- if self.file_source.filter().is_some() {+ let filter_may_change_row_count = self.file_source.filter().is_some()+ && self.statistics.num_rows != Precision::Exact(0);+ if filter_may_change_row_count {self.statistics.clone().to_inexact()} else {self.statistics.clone()@@ -2415,7 +2418,6 @@ mod tests {use crate::source::DataSourceExec;use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};- // Create a schema with 4 columnslet schema = Arc::new(Schema::new(vec![Field::new("col0", DataType::Int32, false),Field::new("col1", DataType::Int32, false),@@ -2499,6 +2501,45 @@ mod tests {assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));}+ #[test]+ fn test_statistics_with_filter() {+ assert_num_rows_with_filter(Precision::Absent, Precision::Absent);+ assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100));+ assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100));+ assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0));++ /// Creates a [`FileScanConfig`] with a filter and calls [`FileScanConfig::statistics`].+ /// Then the function checks the output num_rows stats, given the input num_rows stats.+ fn assert_num_rows_with_filter(+ input_num_rows: Precision<usize>,+ expected_num_rows: Precision<usize>,+ ) {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "col0",+ DataType::Int32,+ false,+ )]));++ let stats =+ Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows);+ let file_group =+ FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]);++ let table_schema = TableSchema::from(&schema);+ let config = FileScanConfigBuilder::new(+ ObjectStoreUrl::parse("test:///").unwrap(),+ Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new(+ Literal::new(ScalarValue::Boolean(Some(true))),+ ))),+ )+ .with_file_groups(vec![file_group])+ .with_statistics(stats)+ .build();++ assert_eq!(config.statistics().num_rows, expected_num_rows,);+ }+ }+/// Regression test for reusing a `DataSourceExec` after its execution-local/// shared work queue has been drained.///