Skip to content

Apache DataFusion SQL Query Engine

fix: Improve consistency of per-column stats on `FilterExec` output (#22718)

Neil Conwaye71bd56ebf34
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index b3b107dc58..11d36192f3 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -60,7 +60,9 @@ use datafusion_common::{
use datafusion_execution::TaskContext;
use datafusion_expr::Operator;
use datafusion_physical_expr::equivalence::ProjectionMapping;
-use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal, lit};
+use datafusion_physical_expr::expressions::{
+ BinaryExpr, Column, IsNotNullExpr, Literal, lit,
+};
use datafusion_physical_expr::intervals::utils::check_support;
use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};
use datafusion_physical_expr::{
@@ -310,11 +312,23 @@ impl FilterExec {
&self.projection
}
- /// Calculates `Statistics` for `FilterExec`, by applying selectivity
- /// (either default, or estimated) to input statistics.
+ /// Calculates `Statistics` for `FilterExec` by applying the filter's
+ /// selectivity (default, or estimated from interval analysis) to the input
+ /// statistics.
+ ///
+ /// The estimated output row count is used to keep the per-column statistics
+ /// consistent with it:
+ /// - null and distinct counts are capped at the estimated row count;
+ /// - byte sizes (per column and total) are scaled by the selectivity;
+ /// - a column constrained to a single value (`col = literal`, or an
+ /// interval that collapses to one point) gets a distinct count of 1;
+ /// - a column in a null-rejecting conjunct gets a null count of 0.
+ ///
+ /// When interval analysis applies, min/max are also tightened to the
+ /// surviving value range.
///
- /// Equality predicates (`col = literal`) set NDV to `Exact(1)`, or
- /// `Exact(0)` when the predicate is contradictory (e.g. `a = 1 AND a = 2`).
+ /// A contradictory predicate (e.g. `a = 1 AND a = 2`) yields zero rows and
+ /// empty-column statistics.
pub(crate) fn statistics_helper(
schema: &SchemaRef,
input_stats: Statistics,
@@ -327,8 +341,8 @@ impl FilterExec {
let input_total_byte_size = input_stats.total_byte_size;
let (selectivity, num_rows, column_statistics) = if is_infeasible {
- // Contradictory predicate: zero rows, and null/min/max are
- // undefined on an empty column.
+ // Contradictory predicate: no rows survive. Row-bounded counts are
+ // zero; value statistics are undefined on an empty column.
let mut cs = input_stats.to_inexact().column_statistics;
for col_stat in &mut cs {
col_stat.distinct_count = Precision::Exact(0);
@@ -339,43 +353,50 @@ impl FilterExec {
col_stat.byte_size = Precision::Exact(0);
}
(0.0, Precision::Exact(0), cs)
- } else if !check_support(predicate, schema) {
- // Interval analysis is not applicable; fall back to the default
- // selectivity but still pin NDV=1 for every `col = literal` column.
- let selectivity = default_selectivity as f64 / 100.0;
- let mut cs = input_stats.to_inexact().column_statistics;
- for &idx in &eq_columns {
- if idx < cs.len() && cs[idx].distinct_count != Precision::Exact(0) {
- cs[idx].distinct_count = Precision::Exact(1);
+ } else {
+ let null_rejecting_columns = collect_null_rejecting_columns(predicate);
+
+ if check_support(predicate, schema) {
+ let input_analysis_ctx = AnalysisContext::try_from_statistics(
+ schema,
+ &input_stats.column_statistics,
+ )?;
+ let analysis_ctx = analyze(predicate, input_analysis_ctx, schema)?;
+ let selectivity = analysis_ctx.selectivity.unwrap_or(1.0);
+ let filtered_num_rows =
+ input_num_rows.with_estimated_selectivity(selectivity);
+ let cs = collect_new_statistics(
+ schema,
+ &input_stats.column_statistics,
+ analysis_ctx.boundaries,
+ selectivity,
+ &null_rejecting_columns,
+ filtered_num_rows,
+ );
+ (selectivity, filtered_num_rows, cs)
+ } else {
+ // Without interval boundaries, use the default selectivity and
+ // apply the row-count constraints that still follow from the
+ // filter predicate.
+ let selectivity = default_selectivity as f64 / 100.0;
+ let filtered_num_rows =
+ input_num_rows.with_estimated_selectivity(selectivity);
+ let mut cs = input_stats.to_inexact().column_statistics;
+ for (idx, col_stat) in cs.iter_mut().enumerate() {
+ col_stat.byte_size = scale_byte_size(col_stat.byte_size, selectivity);
+ col_stat.null_count = if null_rejecting_columns.contains(&idx) {
+ Precision::Exact(0)
+ } else {
+ cap_at_rows(col_stat.null_count, filtered_num_rows)
+ };
+ col_stat.distinct_count = if eq_columns.contains(&idx) {
+ distinct_count_for_singleton_domain(filtered_num_rows)
+ } else {
+ cap_at_rows(col_stat.distinct_count, filtered_num_rows)
+ };
}
+ (selectivity, filtered_num_rows, cs)
}
- (
- selectivity,
- input_num_rows.with_estimated_selectivity(selectivity),
- cs,
- )
- } else {
- // Interval-analysis path. `collect_new_statistics` already sets
- // distinct_count = Exact(1) when an interval collapses to a single
- // value, so no post-fix is needed here.
- let input_analysis_ctx = AnalysisContext::try_from_statistics(
- schema,
- &input_stats.column_statistics,
- )?;
- let analysis_ctx = analyze(predicate, input_analysis_ctx, schema)?;
- let selectivity = analysis_ctx.selectivity.unwrap_or(1.0);
- let filtered_num_rows =
- input_num_rows.with_estimated_selectivity(selectivity);
- let cs = collect_new_statistics(
- schema,
- &input_stats.column_statistics,
- analysis_ctx.boundaries,
- match &filtered_num_rows {
- Precision::Absent => None,
- p => Some(*p),
- },
- );
- (selectivity, filtered_num_rows, cs)
};
let total_byte_size =
@@ -846,6 +867,48 @@ fn collect_equality_columns(predicate: &Arc<dyn PhysicalExpr>) -> (HashSet<usize
(eq_values.into_keys().collect(), infeasible)
}
+/// Collects columns that cannot be NULL in any surviving row.
+///
+/// A filter keeps only rows where the predicate is TRUE, so a column is
+/// null-rejecting if some top-level AND conjunct evaluates to NULL or FALSE
+/// whenever that column is NULL. Two such conjuncts are recognized:
+///
+/// - a binary operator that returns NULL on NULL input, applied directly to the
+/// column (e.g. `a = 10`, `a < b`);
+/// - an `IS NOT NULL` check on the column (e.g. `a IS NOT NULL`).
+///
+/// This analysis is conservative; for example, OR clauses are not considered
+/// null-rejecting, and neither are indirect operands like `a + 1 < 10`.
+fn collect_null_rejecting_columns(predicate: &Arc<dyn PhysicalExpr>) -> HashSet<usize> {
+ let mut columns = HashSet::new();
+
+ for expr in split_conjunction(predicate) {
+ // `col IS NOT NULL` keeps only rows where `col` is non-null.
+ if let Some(is_not_null) = expr.downcast_ref::<IsNotNullExpr>() {
+ if let Some(col) = is_not_null.arg().downcast_ref::<Column>() {
+ columns.insert(col.index());
+ }
+ continue;
+ }
+
+ // A binary operator that returns NULL on NULL input rejects rows where
+ // a direct column operand is NULL.
+ if let Some(binary) = expr.downcast_ref::<BinaryExpr>() {
+ if !binary.op().returns_null_on_null() {
+ continue;
+ }
+ if let Some(col) = binary.left().downcast_ref::<Column>() {
+ columns.insert(col.index());
+ }
+ if let Some(col) = binary.right().downcast_ref::<Column>() {
+ columns.insert(col.index());
+ }
+ }
+ }
+
+ columns
+}
+
/// Converts an interval bound to a [`Precision`] value. NULL bounds (which
/// represent "unbounded" in the interval type) map to [`Precision::Absent`].
fn interval_bound_to_precision(
@@ -861,15 +924,61 @@ fn interval_bound_to_precision(
}
}
-/// This function ensures that all bounds in the `ExprBoundaries` vector are
-/// converted to closed bounds. If a lower/upper bound is initially open, it
-/// is adjusted by using the next/previous value for its data type to convert
-/// it into a closed bound.
+/// Scales a column's `byte_size` by the estimated filter `selectivity`. An
+/// exact zero is preserved: an empty column stays exactly empty after
+/// filtering.
+fn scale_byte_size(byte_size: Precision<usize>, selectivity: f64) -> Precision<usize> {
+ match byte_size {
+ Precision::Exact(0) => Precision::Exact(0),
+ byte_size => byte_size.with_estimated_selectivity(selectivity),
+ }
+}
+
+/// Caps a row-bounded column statistic (a null count or distinct count) at the
+/// filtered row estimate, since a column cannot have more nulls or distinct
+/// values than it has rows. Known counts are demoted to inexact because the
+/// filtered row count is itself an estimate.
+fn cap_at_rows(
+ value: Precision<usize>,
+ filtered_num_rows: Precision<usize>,
+) -> Precision<usize> {
+ match filtered_num_rows {
+ Precision::Absent => value.to_inexact(),
+ rows => value.to_inexact().min(&rows),
+ }
+}
+
+/// Returns the NDV for a column constrained to one non-null value (e.g.
+/// `column = literal` or a singleton interval), derived from the filtered row
+/// estimate: zero rows means zero distinct values, a known positive row count
+/// means exactly one, and an unknown row count means an inexact one (the column
+/// could still be empty).
+///
+/// The caller is responsible for proving the singleton domain.
+fn distinct_count_for_singleton_domain(
+ filtered_num_rows: Precision<usize>,
+) -> Precision<usize> {
+ match filtered_num_rows {
+ Precision::Exact(0) | Precision::Inexact(0) => filtered_num_rows,
+ // The row count is unknown, so the column could still be empty (zero
+ // distinct values); report an inexact one rather than overstating it.
+ Precision::Absent => Precision::Inexact(1),
+ _ => Precision::Exact(1),
+ }
+}
+
+/// Builds output column statistics from interval-analysis boundaries.
+///
+/// The interval bounds become min/max values, singleton intervals become
+/// singleton NDV, and row-bounded counts are kept consistent with the filtered
+/// row estimate.
fn collect_new_statistics(
schema: &SchemaRef,
input_column_stats: &[ColumnStatistics],
analysis_boundaries: Vec<ExprBoundaries>,
- filtered_num_rows: Option<Precision<usize>>,
+ selectivity: f64,
+ null_rejecting_columns: &HashSet<usize>,
+ filtered_num_rows: Precision<usize>,
) -> Vec<ColumnStatistics> {
analysis_boundaries
.into_iter()
@@ -904,24 +1013,29 @@ fn collect_new_statistics(
!lower.is_null() && !upper.is_null() && lower == upper;
let min_value = interval_bound_to_precision(lower, is_single_value);
let max_value = interval_bound_to_precision(upper, is_single_value);
- // When the interval collapses to a single value (equality
- // predicate), the column has exactly 1 distinct value.
- // Otherwise, cap NDV at the filtered row count.
+
+ // Distinct and null counts cannot exceed the number of rows
+ // that survive the filter. Singleton intervals and
+ // null-rejecting predicates provide tighter bounds.
let capped_distinct_count = if is_single_value {
- Precision::Exact(1)
+ distinct_count_for_singleton_domain(filtered_num_rows)
} else {
- match filtered_num_rows {
- Some(rows) => distinct_count.to_inexact().min(&rows),
- None => distinct_count.to_inexact(),
- }
+ cap_at_rows(distinct_count, filtered_num_rows)
+ };
+ let capped_null_count = if null_rejecting_columns.contains(&idx) {
+ Precision::Exact(0)
+ } else {
+ cap_at_rows(input_column_stats[idx].null_count, filtered_num_rows)
};
+ let byte_size =
+ scale_byte_size(input_column_stats[idx].byte_size, selectivity);
ColumnStatistics {
- null_count: input_column_stats[idx].null_count.to_inexact(),
+ null_count: capped_null_count,
max_value,
min_value,
sum_value: Precision::Absent,
distinct_count: capped_distinct_count,
- byte_size: input_column_stats[idx].byte_size,
+ byte_size,
}
},
)
@@ -1237,6 +1351,8 @@ mod tests {
assert_eq!(
statistics.column_statistics,
vec![ColumnStatistics {
+ // `a <= 25` rejects nulls, so the column has no surviving nulls.
+ null_count: Precision::Exact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(25))),
..Default::default()
@@ -1283,6 +1399,8 @@ mod tests {
assert_eq!(
statistics.column_statistics,
vec![ColumnStatistics {
+ // `a <= 25 AND a >= 10` rejects nulls in `a`.
+ null_count: Precision::Exact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(10))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(25))),
..Default::default()
@@ -1350,11 +1468,16 @@ mod tests {
statistics.column_statistics,
vec![
ColumnStatistics {
+ // `a <= 25 AND a >= 10` rejects nulls in `a`.
+ null_count: Precision::Exact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(10))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(25))),
..Default::default()
},
ColumnStatistics {
+ // `b > 45` in the upstream filter zeroes b's nulls; the outer
+ // filter then caps the (already zero) count, demoting to inexact.
+ null_count: Precision::Inexact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(46))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(50))),
..Default::default()
@@ -1551,8 +1674,13 @@ mod tests {
Arc::new(Column::new("b", 1)),
)),
));
- // Since filter predicate passes all entries, statistics after filter shouldn't change.
- let expected = input.partition_statistics(None)?.column_statistics.clone();
+ // The filter predicate passes all (non-null) entries, so min/max/NDV
+ // are unchanged. `a < 200` and `1 <= b` are null-rejecting, though, so
+ // both columns lose any nulls regardless of selectivity.
+ let mut expected = input.partition_statistics(None)?.column_statistics.clone();
+ for col in &mut expected {
+ col.null_count = Precision::Exact(0);
+ }
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
let statistics = filter.partition_statistics(None)?;
@@ -1742,10 +1870,14 @@ mod tests {
statistics.column_statistics,
vec![
ColumnStatistics {
+ // `a < 50` rejects nulls in `a`.
+ null_count: Precision::Exact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(49))),
..Default::default()
},
+ // `b` is not referenced by the predicate, so its stats are
+ // unchanged (null count stays unknown).
ColumnStatistics {
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(100))),
@@ -1790,7 +1922,9 @@ mod tests {
num_rows: Precision::Absent,
total_byte_size: Precision::Absent,
column_statistics: vec![ColumnStatistics {
- null_count: Precision::Absent,
+ // `a <= 10` rejects nulls, so `a` has no surviving nulls even
+ // though the input statistics are entirely unknown.
+ null_count: Precision::Exact(0),
min_value: Precision::Inexact(ScalarValue::Int32(Some(5))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(10))),
sum_value: Precision::Absent,
@@ -2425,7 +2559,7 @@ mod tests {
vec![Precision::Exact(1)],
),
(
- "OR preserves original NDV",
+ "OR is not collapsed to NDV=1, but NDV is capped at filtered rows",
vec![Field::new("name", DataType::Utf8, false)],
vec![ColumnStatistics {
distinct_count: Precision::Inexact(50),
@@ -2444,7 +2578,9 @@ mod tests {
Arc::new(Literal::new(ScalarValue::Utf8(Some("b".to_string())))),
)),
)),
- vec![Precision::Inexact(50)],
+ // Input NDV is 50, but the 20% default selectivity on 100 rows
+ // estimates 20 output rows, so NDV is capped at 20.
+ vec![Precision::Inexact(20)],
),
(
"AND with mixed types (Utf8 + Int32)",
@@ -2643,11 +2779,73 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn test_filter_statistics_empty_input_equality_ndv_zero() -> Result<()> {
+ let cases: Vec<(&str, Schema, Statistics, Arc<dyn PhysicalExpr>)> = vec![
+ (
+ "fallback string equality",
+ Schema::new(vec![Field::new("name", DataType::Utf8, true)]),
+ Statistics {
+ num_rows: Precision::Exact(0),
+ total_byte_size: Precision::Exact(0),
+ column_statistics: vec![ColumnStatistics {
+ distinct_count: Precision::Exact(0),
+ null_count: Precision::Exact(0),
+ byte_size: Precision::Exact(0),
+ ..Default::default()
+ }],
+ },
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("name", 0)),
+ Operator::Eq,
+ Arc::new(Literal::new(ScalarValue::Utf8(Some("x".to_string())))),
+ )),
+ ),
+ (
+ "interval numeric equality",
+ Schema::new(vec![Field::new("a", DataType::Int32, true)]),
+ Statistics {
+ num_rows: Precision::Exact(0),
+ total_byte_size: Precision::Exact(0),
+ column_statistics: vec![ColumnStatistics {
+ min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
+ max_value: Precision::Inexact(ScalarValue::Int32(Some(10))),
+ distinct_count: Precision::Exact(0),
+ null_count: Precision::Exact(0),
+ byte_size: Precision::Exact(0),
+ ..Default::default()
+ }],
+ },
+ Arc::new(BinaryExpr::new(
+ Arc::new(Column::new("a", 0)),
+ Operator::Eq,
+ Arc::new(Literal::new(ScalarValue::Int32(Some(5)))),
+ )),
+ ),
+ ];
+
+ for (desc, schema, input_stats, predicate) in cases {
+ let input = Arc::new(StatisticsExec::new(input_stats, schema));
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+ let statistics = filter.partition_statistics(None)?;
+
+ assert_eq!(
+ statistics.num_rows,
+ Precision::Inexact(0),
+ "case '{desc}': row count mismatch"
+ );
+ assert_eq!(
+ statistics.column_statistics[0].distinct_count,
+ Precision::Inexact(0),
+ "case '{desc}': NDV should be capped at zero rows"
+ );
+ }
+ Ok(())
+ }
+
#[tokio::test]
async fn test_filter_statistics_and_equality_ndv() -> Result<()> {
- // a: min=1, max=100, ndv=80
- // b: min=1, max=50, ndv=40
- // c: min=1, max=200, ndv=150
let schema = Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
@@ -2661,6 +2859,7 @@ mod tests {
ColumnStatistics {
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(100))),
+ null_count: Precision::Inexact(80),
distinct_count: Precision::Inexact(80),
..Default::default()
},
@@ -2673,6 +2872,7 @@ mod tests {
ColumnStatistics {
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(200))),
+ null_count: Precision::Inexact(90),
distinct_count: Precision::Inexact(150),
..Default::default()
},
@@ -2706,11 +2906,15 @@ mod tests {
let filter: Arc<dyn ExecutionPlan> =
Arc::new(FilterExec::try_new(predicate, input)?);
let statistics = filter.partition_statistics(None)?;
- // a = 42 collapses to single value
+ // Equality predicates collapse NDV and reject nulls for their columns.
assert_eq!(
statistics.column_statistics[0].distinct_count,
Precision::Exact(1)
);
+ assert_eq!(
+ statistics.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
// b > 10 narrows to [11, 50] but doesn't collapse to a single value.
// The combined selectivity of a=42 (1/80) and c=7 (1/150) on 100 rows
// computes num_rows = 1, so NDV is capped at the row count: min(40, 1) = 1.
@@ -2718,11 +2922,14 @@ mod tests {
statistics.column_statistics[1].distinct_count,
Precision::Inexact(1)
);
- // c = 7 collapses to single value
assert_eq!(
statistics.column_statistics[2].distinct_count,
Precision::Exact(1)
);
+ assert_eq!(
+ statistics.column_statistics[2].null_count,
+ Precision::Exact(0)
+ );
Ok(())
}
@@ -2742,8 +2949,8 @@ mod tests {
schema.clone(),
));
- // a = 42: even without known bounds, interval analysis resolves
- // the equality to [42, 42], so NDV is correctly set to Exact(1)
+ // Even without input bounds, interval analysis can derive singleton
+ // bounds from the equality itself.
let predicate = Arc::new(BinaryExpr::new(
Arc::new(Column::new("a", 0)),
Operator::Eq,
@@ -3208,16 +3415,17 @@ mod tests {
#[tokio::test]
async fn test_filter_statistics_ndv_capped_at_row_count() -> Result<()> {
- // Table: a: min=1, max=100, distinct_count=80, 100 rows
- let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
let input = Arc::new(StatisticsExec::new(
Statistics {
num_rows: Precision::Inexact(100),
- total_byte_size: Precision::Inexact(400),
+ total_byte_size: Precision::Inexact(1000),
column_statistics: vec![ColumnStatistics {
min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),
max_value: Precision::Inexact(ScalarValue::Int32(Some(100))),
+ null_count: Precision::Inexact(80),
distinct_count: Precision::Inexact(80),
+ byte_size: Precision::Exact(1000),
..Default::default()
}],
},
@@ -3232,14 +3440,148 @@ mod tests {
Arc::new(FilterExec::try_new(predicate, input)?);
let statistics = filter.partition_statistics(None)?;
- // Filter estimates ~10 rows (selectivity = 10/100)
assert_eq!(statistics.num_rows, Precision::Inexact(10));
- // NDV should be capped at the filtered row count (10), not the original 80
let ndv = &statistics.column_statistics[0].distinct_count;
assert!(
ndv.get_value().copied() <= Some(10),
"Expected NDV <= 10 (filtered row count), got {ndv:?}"
);
+ // `a <= 10` rejects nulls, so the 80 input nulls drop to exactly zero.
+ assert_eq!(
+ statistics.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
+ // byte_size follows the same 10% selectivity estimate.
+ assert_eq!(
+ statistics.column_statistics[0].byte_size,
+ Precision::Inexact(100)
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_filter_statistics_default_selectivity_column_stats() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
+ let input = Arc::new(StatisticsExec::new(
+ Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1000),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Inexact(80),
+ distinct_count: Precision::Inexact(60),
+ byte_size: Precision::Exact(1000),
+ ..Default::default()
+ }],
+ },
+ schema.clone(),
+ ));
+
+ // Utf8 interval analysis is unsupported, so this exercises the default
+ // selectivity path. The predicate rejects nulls but does not constrain
+ // the column to one value.
+ let predicate: Arc<dyn PhysicalExpr> =
+ binary(col("name", &schema)?, Operator::Gt, lit("m"), &schema)?;
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+
+ let statistics = filter.partition_statistics(None)?;
+ assert_eq!(statistics.num_rows, Precision::Inexact(20));
+ assert_eq!(
+ statistics.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].byte_size,
+ Precision::Inexact(200)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].distinct_count,
+ Precision::Inexact(20)
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_filter_statistics_or_does_not_reject_nulls() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
+ let input = Arc::new(StatisticsExec::new(
+ Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1000),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Inexact(80),
+ distinct_count: Precision::Inexact(60),
+ byte_size: Precision::Exact(1000),
+ ..Default::default()
+ }],
+ },
+ schema.clone(),
+ ));
+
+ let predicate: Arc<dyn PhysicalExpr> = binary(
+ binary(col("name", &schema)?, Operator::Gt, lit("m"), &schema)?,
+ Operator::Or,
+ is_null(col("name", &schema)?)?,
+ &schema,
+ )?;
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+
+ let statistics = filter.partition_statistics(None)?;
+ assert_eq!(statistics.num_rows, Precision::Inexact(20));
+ assert_eq!(
+ statistics.column_statistics[0].null_count,
+ Precision::Inexact(20)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].byte_size,
+ Precision::Inexact(200)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].distinct_count,
+ Precision::Inexact(20)
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn test_filter_statistics_is_not_null_rejects_nulls() -> Result<()> {
+ let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);
+ let input = Arc::new(StatisticsExec::new(
+ Statistics {
+ num_rows: Precision::Inexact(100),
+ total_byte_size: Precision::Inexact(1000),
+ column_statistics: vec![ColumnStatistics {
+ null_count: Precision::Inexact(80),
+ distinct_count: Precision::Inexact(60),
+ byte_size: Precision::Exact(1000),
+ ..Default::default()
+ }],
+ },
+ schema.clone(),
+ ));
+
+ // `name IS NOT NULL` keeps only non-null rows, so the surviving null
+ // count is exactly zero. Utf8 interval analysis is unsupported, so this
+ // also exercises the default-selectivity path.
+ let predicate: Arc<dyn PhysicalExpr> = is_not_null(col("name", &schema)?)?;
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExec::try_new(predicate, input)?);
+
+ let statistics = filter.partition_statistics(None)?;
+ assert_eq!(statistics.num_rows, Precision::Inexact(20));
+ assert_eq!(
+ statistics.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].byte_size,
+ Precision::Inexact(200)
+ );
+ assert_eq!(
+ statistics.column_statistics[0].distinct_count,
+ Precision::Inexact(20)
+ );
Ok(())
}
}
diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt
index 1073f60a0f..9cf6b1e038 100644
--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt
@@ -59,7 +59,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(40))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -84,7 +84,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(40))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -109,7 +109,7 @@ query TT
EXPLAIN SELECT * FROM test_table WHERE column1 = 1;
----
physical_plan
-01)FilterExec: column1@0 = 1, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Distinct=Exact(1))]]
+01)FilterExec: column1@0 = 1, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]]
@@ -152,7 +152,7 @@ query TT
EXPLAIN SELECT i8 FROM typed_table WHERE i8 = 2;
----
physical_plan
-01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(5))]]
+01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i8], file_type=parquet, predicate=i8@0 = 2, pruning_predicate=i8_null_count@2 != row_count@3 AND i8_min@0 <= 2 AND 2 <= i8_max@1, required_guarantees=[i8 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]
@@ -161,7 +161,7 @@ query TT
EXPLAIN SELECT i64 FROM typed_table WHERE i64 = 2;
----
physical_plan
-01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(40))]]
+01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(8))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i64], file_type=parquet, predicate=i64@1 = 2, pruning_predicate=i64_null_count@2 != row_count@3 AND i64_min@0 <= 2 AND 2 <= i64_max@1, required_guarantees=[i64 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
@@ -170,7 +170,7 @@ query TT
EXPLAIN SELECT f32 FROM typed_table WHERE f32 = 2.5;
----
physical_plan
-01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(20))]]
+01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f32], file_type=parquet, predicate=CAST(f32@2 AS Float64) = 2.5, pruning_predicate=f32_null_count@2 != row_count@3 AND CAST(f32_min@0 AS Float64) <= 2.5 AND 2.5 <= CAST(f32_max@1 AS Float64), required_guarantees=[], statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]
@@ -179,7 +179,7 @@ query TT
EXPLAIN SELECT f64 FROM typed_table WHERE 2.5 = f64;
----
physical_plan
-01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(40))]]
+01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]
02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]
03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f64], file_type=parquet, predicate=f64@3 = 2.5, pruning_predicate=f64_null_count@2 != row_count@3 AND f64_min@0 <= 2.5 AND 2.5 <= f64_max@1, required_guarantees=[f64 in (2.5)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]