Skip to content

Apache DataFusion SQL Query Engine

fix(datasource): avoid over-conservative transformation of num_rows statistics in file scan config (#23670)

Tobias Schwarzinger62cfc0ceae30
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 962df06302..d1dd3c11fc 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -39,6 +39,7 @@ use datafusion_execution::{
use datafusion_expr::Operator;
use crate::source::OpenArgs;
+use datafusion_common::stats::Precision;
use datafusion_physical_expr::expressions::{BinaryExpr, Column};
use datafusion_physical_expr::projection::{ProjectionExprs, ProjectionMapping};
use datafusion_physical_expr::utils::reassign_expr_columns;
@@ -1233,7 +1234,9 @@ impl FileScanConfig {
/// we can't guarantee the statistics are exact because we don't know how many
/// rows will be filtered out.
pub fn statistics(&self) -> Statistics {
- if self.file_source.filter().is_some() {
+ let filter_may_change_row_count = self.file_source.filter().is_some()
+ && self.statistics.num_rows != Precision::Exact(0);
+ if filter_may_change_row_count {
self.statistics.clone().to_inexact()
} else {
self.statistics.clone()
@@ -2415,7 +2418,6 @@ mod tests {
use crate::source::DataSourceExec;
use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
- // Create a schema with 4 columns
let schema = Arc::new(Schema::new(vec![
Field::new("col0", DataType::Int32, false),
Field::new("col1", DataType::Int32, false),
@@ -2499,6 +2501,45 @@ mod tests {
assert_eq!(partition_stats.total_byte_size, Precision::Exact(800));
}
+ #[test]
+ fn test_statistics_with_filter() {
+ assert_num_rows_with_filter(Precision::Absent, Precision::Absent);
+ assert_num_rows_with_filter(Precision::Exact(100), Precision::Inexact(100));
+ assert_num_rows_with_filter(Precision::Inexact(100), Precision::Inexact(100));
+ assert_num_rows_with_filter(Precision::Exact(0), Precision::Exact(0));
+
+ /// Creates a [`FileScanConfig`] with a filter and calls [`FileScanConfig::statistics`].
+ /// Then the function checks the output num_rows stats, given the input num_rows stats.
+ fn assert_num_rows_with_filter(
+ input_num_rows: Precision<usize>,
+ expected_num_rows: Precision<usize>,
+ ) {
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "col0",
+ DataType::Int32,
+ false,
+ )]));
+
+ let stats =
+ Statistics::new_unknown(schema.as_ref()).with_num_rows(input_num_rows);
+ let file_group =
+ FileGroup::new(vec![PartitionedFile::new("test.parquet", 1024)]);
+
+ let table_schema = TableSchema::from(&schema);
+ let config = FileScanConfigBuilder::new(
+ ObjectStoreUrl::parse("test:///").unwrap(),
+ Arc::new(MockSource::new(table_schema.clone()).with_filter(Arc::new(
+ Literal::new(ScalarValue::Boolean(Some(true))),
+ ))),
+ )
+ .with_file_groups(vec![file_group])
+ .with_statistics(stats)
+ .build();
+
+ assert_eq!(config.statistics().num_rows, expected_num_rows,);
+ }
+ }
+
/// Regression test for reusing a `DataSourceExec` after its execution-local
/// shared work queue has been drained.
///