Apache DataFusion SQL Query Engine
fix: apply pushed-down fetch to filter statistics (#25862)
Kumar Ujjawal8487794436c2
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex dcaae294b6..bc1c1ce5d1 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -64,7 +64,7 @@ use datafusion_execution::TaskContext;use datafusion_expr::Operator;use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::expressions::{- BinaryExpr, Column, InListExpr, IsNotNullExpr, Literal, lit,+ BinaryExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr, Literal, lit,};use datafusion_physical_expr::intervals::utils::check_support;use datafusion_physical_expr::utils::collect_columns;@@ -380,17 +380,8 @@ impl FilterExec {let match_limit = unique_match_limit(predicate, &input_stats);let (selectivity, num_rows, column_statistics) = if is_infeasible {- // Contradictory predicate: no rows survive. Row-bounded counts are- // zero; value statistics are undefined on an empty column.- let mut cs = input_stats.to_inexact().column_statistics;- for col_stat in &mut cs {- col_stat.distinct_count = Precision::Exact(0);- col_stat.null_count = Precision::Exact(0);- col_stat.min_value = Precision::Absent;- col_stat.max_value = Precision::Absent;- col_stat.sum_value = Precision::Absent;- col_stat.byte_size = Precision::Exact(0);- }+ // Contradictory predicate: no rows survive.+ let cs = vec![empty_column_statistics(); input_stats.column_statistics.len()];(0.0, Precision::Exact(0), cs)} else {let null_rejecting_columns = collect_null_rejecting_columns(predicate);@@ -456,6 +447,87 @@ impl FilterExec {})}+ /// Applies the filter's `fetch` to its output statistics for `partition`,+ /// or for all partitions when `partition` is `None`. The fetch stops each+ /// partition separately.+ #[inline]+ pub(crate) fn statistics_with_fetch(+ &self,+ stats: Statistics,+ partition: Option<usize>,+ ) -> Result<Statistics> {+ match self.fetch {+ Some(fetch) => self.statistics_under_fetch(stats, fetch, partition),+ None => Ok(stats),+ }+ }++ fn statistics_under_fetch(+ &self,+ stats: Statistics,+ fetch: usize,+ partition: Option<usize>,+ ) -> Result<Statistics> {+ if stats+ .num_rows+ .get_value()+ .is_some_and(|rows| *rows <= fetch)+ {+ // No partition reaches the fetch, so no rows are dropped.+ return Ok(stats);+ }+ let partitions = self.properties().partitioning.partition_count();+ let single = partition.is_some() || partitions <= 1;+ let bound = if single {+ fetch+ } else {+ fetch.saturating_mul(partitions)+ };+ let input_columns = stats.column_statistics.clone();+ let stats = if single {+ stats+ } else {+ // Any partition can hold more than `fetch` rows, so the total+ // under the fetch is only an estimate.+ stats.to_inexact()+ };++ let mut stats = stats.with_fetch(Some(bound), 0, 1)?;+ if stats.num_rows == Precision::Exact(0) {+ stats.total_byte_size = Precision::Exact(0);+ stats.column_statistics.fill(empty_column_statistics());+ return Ok(stats);+ }+ // A fetch only drops rows. A count that is exactly zero stays exact, and+ // so does a column with one value in every row while rows remain.+ for (column_stats, input) in stats.column_statistics.iter_mut().zip(input_columns)+ {+ if input.null_count == Precision::Exact(0) {+ if fetch > 0 && input.is_singleton() {+ column_stats.min_value = input.min_value;+ column_stats.max_value = input.max_value;+ if input.distinct_count == Precision::Exact(1) {+ column_stats.distinct_count = input.distinct_count;+ }+ }+ column_stats.null_count = input.null_count;+ } else {+ column_stats.null_count =+ cap_at_rows(column_stats.null_count, stats.num_rows);+ }+ if input.distinct_count == Precision::Exact(0) {+ column_stats.distinct_count = input.distinct_count;+ }+ }+ if let Some((column, true)) = null_check_column(self.predicate())+ && let Some(column_stats) = stats.column_statistics.get_mut(column)+ {+ // Every surviving row is still null after the fetch.+ column_stats.null_count = stats.num_rows;+ }+ Ok(stats)+ }+/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.fn compute_properties(input: &Arc<dyn ExecutionPlan>,@@ -673,7 +745,7 @@ impl ExecutionPlan for FilterExec {fn statistics_from_inputs(&self,input_stats: &[Arc<Statistics>],- _args: &StatisticsArgs,+ args: &StatisticsArgs,) -> Result<Arc<Statistics>> {let input_stats = input_stats[0].as_ref().clone();let stats = Self::statistics_helper(@@ -682,6 +754,7 @@ impl ExecutionPlan for FilterExec {self.predicate(),self.default_selectivity,)?;+ let stats = self.statistics_with_fetch(stats, args.partition())?;Ok(Arc::new(stats.project(self.projection.as_ref())))}@@ -1207,6 +1280,31 @@ fn collect_null_rejecting_columns(predicate: &Arc<dyn PhysicalExpr>) -> HashSet<columns}+/// Returns the checked column index, and whether the check is `IS NULL`, when+/// `predicate` is a bare `IS NULL` or `IS NOT NULL` check on a column.+pub(crate) fn null_check_column(+ predicate: &Arc<dyn PhysicalExpr>,+) -> Option<(usize, bool)> {+ let (arg, is_null) = if let Some(expr) = predicate.downcast_ref::<IsNullExpr>() {+ (expr.arg(), true)+ } else {+ let expr = predicate.downcast_ref::<IsNotNullExpr>()?;+ (expr.arg(), false)+ };+ Some((arg.downcast_ref::<Column>()?.index(), is_null))+}++/// Column statistics of an exactly empty output: no nulls, distinct values or+/// bytes, and no value bounds.+fn empty_column_statistics() -> ColumnStatistics {+ ColumnStatistics {+ null_count: Precision::Exact(0),+ distinct_count: Precision::Exact(0),+ byte_size: Precision::Exact(0),+ ..ColumnStatistics::new_unknown()+ }+}+/// Converts an interval bound to a [`Precision`] value. NULL bounds (which/// represent "unbounded" in the interval type) map to [`Precision::Absent`].fn interval_bound_to_precision(@@ -4174,6 +4272,125 @@ mod tests {Ok(())}+ #[test]+ fn test_filter_statistics_fetch_preserves_singleton() -> Result<()> {+ use Precision::{Absent, Exact, Inexact};++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let input = Arc::new(StatisticsExec::new(+ Statistics::new_unknown(&schema),+ schema,+ ));+ let filter = FilterExecBuilder::new(lit(true), input)+ .with_fetch(Some(3))+ .build()?;+ // A positive fetch keeps the singleton that its input reports, for+ // both overall and partition stats.+ for num_rows in [Exact(100), Inexact(100), Absent] {+ for partition in [None, Some(0)] {+ let stats = filter.statistics_with_fetch(+ Statistics {+ num_rows,+ total_byte_size: Absent,+ column_statistics: vec![ColumnStatistics {+ null_count: Exact(0),+ distinct_count: Exact(1),+ min_value: Exact(ScalarValue::Int32(Some(5))),+ max_value: Exact(ScalarValue::Int32(Some(5))),+ ..Default::default()+ }],+ },+ partition,+ )?;+ let column = &stats.column_statistics[0];+ assert_eq!(column.null_count, Exact(0));+ assert!(column.is_singleton());+ assert_eq!(column.distinct_count, Exact(1));+ }+ }+ Ok(())+ }++ #[tokio::test]+ async fn test_filter_fetch_statistics_match_execution() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from_iter_values(0..100))],+ )?;+ for partitions in [1, 2] {+ let input: Arc<dyn ExecutionPlan> = test::TestMemoryExec::try_new_exec(+ &vec![vec![batch.clone()]; partitions],+ Arc::clone(&schema),+ None,+ )?;+ for fetch in [0, 3] {+ let filter: Arc<dyn ExecutionPlan> = Arc::new(+ FilterExecBuilder::new(lit(true), Arc::clone(&input))+ .with_fetch(Some(fetch))+ .build()?,+ );+ let stats = StatisticsContext::new()+ .compute(filter.as_ref(), &StatisticsArgs::new())?;+ let partition_stats = StatisticsContext::new().compute(+ filter.as_ref(),+ &StatisticsArgs::new().with_partition(Some(0)),+ )?;+ assert_eq!(stats.num_rows.get_value(), Some(&(fetch * partitions)));+ assert_eq!(partition_stats.num_rows.get_value(), Some(&fetch));+ let batches = crate::execution_plan::collect(+ filter,+ Arc::new(TaskContext::default()),+ )+ .await?;+ assert_eq!(+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),+ fetch * partitions+ );+ }+ }+ Ok(())+ }++ #[test]+ fn test_filter_fetch_null_column_statistics() -> Result<()> {+ use Precision::{Exact, Inexact};++ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let stats = Statistics {+ num_rows: Exact(100),+ total_byte_size: Exact(400),+ column_statistics: vec![ColumnStatistics {+ null_count: Exact(100),+ distinct_count: Exact(0),+ byte_size: Exact(400),+ ..Default::default()+ }],+ };+ let input: Arc<dyn ExecutionPlan> = Arc::new(+ StatisticsExec::new(stats.clone(), schema.clone())+ .with_partition_statistics(vec![stats.clone()]),+ );+ for fetch in [0, 3, 200] {+ let filter =+ FilterExecBuilder::new(is_null(col("a", &schema)?)?, Arc::clone(&input))+ .with_fetch(Some(fetch))+ .build()?;+ let output = filter.statistics_with_fetch(stats.clone(), None)?;+ assert_eq!(output.num_rows, Exact(fetch.min(100)));+ assert_eq!(output.column_statistics[0].null_count, output.num_rows);+ assert_eq!(output.column_statistics[0].distinct_count, Exact(0));+ let expected_bytes = match fetch {+ 0 => Exact(0),+ 3 => Inexact(12),+ _ => Exact(400),+ };+ assert_eq!(output.total_byte_size, expected_bytes);+ assert_eq!(output.column_statistics[0].byte_size, expected_bytes);+ }+ Ok(())+ }+#[tokio::test]async fn test_filter_statistics_singleton_precision() -> Result<()> {use Precision::{Exact, Inexact};diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rsindex 07a0fa49d6..4274b3b011 100644--- a/datafusion/physical-plan/src/operator_statistics/mod.rs+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs@@ -637,6 +637,7 @@ impl StatisticsProvider for FilterStatisticsProvider {}}+ let stats = filter.statistics_with_fetch(stats, None)?;let stats = stats.project(filter.projection().as_ref());Ok(StatisticsResult::Computed(ExtendedStatistics::new(stats)))}@@ -1794,6 +1795,21 @@ mod tests {Ok(())}+ #[test]+ fn test_filter_provider_applies_fetch() -> Result<()> {+ use crate::filter::FilterExecBuilder;++ let registry =+ StatisticsRegistry::with_providers(vec![Arc::new(FilterStatisticsProvider)]);+ let source = make_source(1000);+ let filter = FilterExecBuilder::new(lit(true), source)+ .with_fetch(Some(3))+ .build()?;+ let stats = compute(®istry, &filter)?;+ assert_eq!(stats.base.num_rows, Precision::Inexact(3));+ Ok(())+ }+#[test]fn test_filter_adjusts_ndv_by_selectivity() -> Result<()> {use datafusion_common::ScalarValue;