Apache DataFusion SQL Query Engine
fix(physical-plan): preserve Exact(0) in FilterExec for null_count, distinct_count and total_byte_size upon empty input (#24000)
Alessandro Solimando9eb31bfd1621
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 511be0bbdd..50c8246b37 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -320,7 +320,8 @@ impl FilterExec {/// The estimated output row count is used to keep the per-column statistics/// consistent with it:/// - null and distinct counts are capped at the estimated row count;- /// - byte sizes (per column and total) are scaled by the selectivity;+ /// - byte sizes (per column and total) are scaled by the selectivity, and+ /// are an exact zero when the row count is an exact zero;/// - a column constrained to a single value (`col = literal`, or an/// interval that collapses to one point) gets a distinct count of 1;/// - a column in a null-rejecting conjunct gets a null count of 0.@@ -384,8 +385,11 @@ impl FilterExec {input_num_rows.with_estimated_selectivity(selectivity);let mut cs = input_stats.to_inexact().column_statistics;for (idx, col_stat) in cs.iter_mut().enumerate() {- col_stat.byte_size =- col_stat.byte_size.with_estimated_selectivity(selectivity);+ col_stat.byte_size = scale_byte_size_at_rows(+ col_stat.byte_size,+ selectivity,+ filtered_num_rows,+ );col_stat.null_count = if null_rejecting_columns.contains(&idx) {Precision::Exact(0)} else {@@ -402,7 +406,7 @@ impl FilterExec {};let total_byte_size =- input_total_byte_size.with_estimated_selectivity(selectivity);+ scale_byte_size_at_rows(input_total_byte_size, selectivity, num_rows);Ok(Statistics {num_rows,@@ -1032,19 +1036,35 @@ fn interval_bound_to_precision(}/// Caps a row-bounded column statistic (a null count or distinct count) at the-/// filtered row estimate, since a column cannot have more nulls or distinct-/// values than it has rows. Known counts are demoted to inexact because the-/// filtered row count is itself an estimate.+/// filtered row count, since a column cannot have more nulls or distinct values+/// than it has rows. Known counts are demoted to inexact because a+/// filter-derived row bound is normally an estimate, the exception being an+/// exact zero, which proves the column is empty.fn cap_at_rows(value: Precision<usize>,filtered_num_rows: Precision<usize>,) -> Precision<usize> {match filtered_num_rows {Precision::Absent => value.to_inexact(),+ Precision::Exact(0) => Precision::Exact(0),rows => value.to_inexact().min(&rows),}}+/// Scales a byte size by the filter selectivity. An exact zero row count means+/// the output is exactly empty, so the byte size is an exact zero too.+fn scale_byte_size_at_rows(+ byte_size: Precision<usize>,+ selectivity: f64,+ filtered_num_rows: Precision<usize>,+) -> Precision<usize> {+ if filtered_num_rows == Precision::Exact(0) {+ Precision::Exact(0)+ } else {+ byte_size.with_estimated_selectivity(selectivity)+ }+}+/// Returns the NDV for a column constrained to one non-null value (e.g./// `column = literal` or a singleton interval), derived from the filtered row/// estimate: zero rows means zero distinct values, a known positive row count@@ -1124,9 +1144,11 @@ fn collect_new_statistics(} else {cap_at_rows(input_column_stats[idx].null_count, filtered_num_rows)};- let byte_size = input_column_stats[idx]- .byte_size- .with_estimated_selectivity(selectivity);+ let byte_size = scale_byte_size_at_rows(+ input_column_stats[idx].byte_size,+ selectivity,+ filtered_num_rows,+ );ColumnStatistics {null_count: capped_null_count,max_value,@@ -2904,16 +2926,29 @@ mod tests {#[tokio::test]async fn test_filter_statistics_preserves_exactly_empty_input() -> Result<()> {// A satisfiable predicate over an exactly empty input: the filter cannot- // produce rows, so the whole estimate stays exact.- let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ // produce rows, so the whole estimate stays exact. Column `b` is not+ // mentioned by the predicate, so its null and distinct counts go through+ // the generic row cap.+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, true),+ ]);let input_stats = Statistics {num_rows: Precision::Exact(0),total_byte_size: Precision::Exact(0),- column_statistics: vec![ColumnStatistics {- null_count: Precision::Exact(0),- byte_size: Precision::Exact(0),- ..Default::default()- }],+ column_statistics: vec![+ ColumnStatistics {+ null_count: Precision::Exact(0),+ byte_size: Precision::Exact(0),+ ..Default::default()+ },+ ColumnStatistics {+ null_count: Precision::Exact(3),+ distinct_count: Precision::Exact(7),+ byte_size: Precision::Exact(0),+ ..Default::default()+ },+ ],};let predicate = Arc::new(BinaryExpr::new(Arc::new(Column::new("a", 0)),@@ -2921,7 +2956,7 @@ mod tests {Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),));- let input = Arc::new(StatisticsExec::new(input_stats, schema));+ let input = Arc::new(StatisticsExec::new(input_stats, schema.clone()));let filter: Arc<dyn ExecutionPlan> =Arc::new(FilterExec::try_new(predicate, input)?);let statistics =@@ -2933,6 +2968,94 @@ mod tests {statistics.column_statistics[0].byte_size,Precision::Exact(0));+ assert_eq!(+ statistics.column_statistics[1].null_count,+ Precision::Exact(0)+ );+ assert_eq!(+ statistics.column_statistics[1].distinct_count,+ Precision::Exact(0)+ );++ // A contradictory predicate (`a = 1 AND a = 2`) discards all rows, the+ // output is empty independently of the input.+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Inexact(1000),+ total_byte_size: Precision::Inexact(8000),+ column_statistics: vec![ColumnStatistics::new_unknown(); 2],+ },+ schema,+ ));+ let contradiction = Arc::new(BinaryExpr::new(+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Eq,+ Arc::new(Literal::new(ScalarValue::Int32(Some(1)))),+ )),+ Operator::And,+ Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Eq,+ Arc::new(Literal::new(ScalarValue::Int32(Some(2)))),+ )),+ ));+ let filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(contradiction, input)?);+ let statistics =+ StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;++ assert_eq!(statistics.num_rows, Precision::Exact(0));+ assert_eq!(statistics.total_byte_size, Precision::Exact(0));++ Ok(())+ }++ #[tokio::test]+ async fn test_filter_statistics_exact_empty_input_zeroes_byte_size() -> Result<()> {+ let cases = [+ ("absent", Precision::Absent, Precision::Absent),+ ("inexact", Precision::Inexact(8000), Precision::Inexact(400)),+ ];++ for (desc, input_total_byte_size, input_byte_size) in cases {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);+ let input_stats = Statistics {+ num_rows: Precision::Exact(0),+ total_byte_size: input_total_byte_size,+ column_statistics: vec![ColumnStatistics {+ byte_size: input_byte_size,+ ..Default::default()+ }],+ };+ let predicate = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),+ ));++ let input = Arc::new(StatisticsExec::new(input_stats, schema));+ let filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(predicate, input)?);+ let statistics = StatisticsContext::new()+ .compute(filter.as_ref(), &StatisticsArgs::new())?;++ assert_eq!(+ statistics.num_rows,+ Precision::Exact(0),+ "case '{desc}': num_rows mismatch"+ );+ assert_eq!(+ statistics.total_byte_size,+ Precision::Exact(0),+ "case '{desc}': total_byte_size mismatch"+ );+ assert_eq!(+ statistics.column_statistics[0].byte_size,+ Precision::Exact(0),+ "case '{desc}': byte_size mismatch"+ );+ }Ok(())}