Apache DataFusion SQL Query Engine
fix: interval analysis error when have two filterexec that inner filter proves zero selectivity (#20743)
Huaijindaa8f52c1684
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex e4b1f1b261..12ce141b47 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -390,17 +390,17 @@ mod test {column_statistics: vec![ColumnStatistics {null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::Null),- min_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ max_value: Precision::Exact(ScalarValue::Int32(None)),+ min_value: Precision::Exact(ScalarValue::Int32(None)),+ sum_value: Precision::Exact(ScalarValue::Int32(None)),distinct_count: Precision::Exact(0),byte_size: Precision::Exact(16),},ColumnStatistics {null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::Null),- min_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ max_value: Precision::Exact(ScalarValue::Date32(None)),+ min_value: Precision::Exact(ScalarValue::Date32(None)),+ sum_value: Precision::Exact(ScalarValue::Date32(None)),distinct_count: Precision::Exact(0),byte_size: Precision::Exact(16), // 4 rows * 4 bytes (Date32)},@@ -419,17 +419,17 @@ mod test {column_statistics: vec![ColumnStatistics {null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::Null),- min_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ max_value: Precision::Exact(ScalarValue::Int32(None)),+ min_value: Precision::Exact(ScalarValue::Int32(None)),+ sum_value: Precision::Exact(ScalarValue::Int32(None)),distinct_count: Precision::Exact(0),byte_size: Precision::Exact(8),},ColumnStatistics {null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::Null),- min_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ max_value: Precision::Exact(ScalarValue::Date32(None)),+ min_value: Precision::Exact(ScalarValue::Date32(None)),+ sum_value: Precision::Exact(ScalarValue::Date32(None)),distinct_count: Precision::Exact(0),byte_size: Precision::Exact(8), // 2 rows * 4 bytes (Date32)},diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 1e6b4e3193..c8b37a247c 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -338,6 +338,7 @@ impl FilterExec {let total_byte_size = total_byte_size.with_estimated_selectivity(selectivity);let column_statistics = collect_new_statistics(+ schema,&input_stats.column_statistics,analysis_ctx.boundaries,);@@ -785,6 +786,7 @@ fn interval_bound_to_precision(/// is adjusted by using the next/previous value for its data type to convert/// it into a closed bound.fn collect_new_statistics(+ schema: &SchemaRef,input_column_stats: &[ColumnStatistics],analysis_boundaries: Vec<ExprBoundaries>,) -> Vec<ColumnStatistics> {@@ -801,12 +803,17 @@ fn collect_new_statistics(},)| {let Some(interval) = interval else {- // If the interval is `None`, we can say that there are no rows:+ // If the interval is `None`, we can say that there are no rows.+ // Use a typed null to preserve the column's data type, so that+ // downstream interval analysis can still intersect intervals+ // of the same type.+ let typed_null = ScalarValue::try_from(schema.field(idx).data_type())+ .unwrap_or(ScalarValue::Null);return ColumnStatistics {null_count: Precision::Exact(0),- max_value: Precision::Exact(ScalarValue::Null),- min_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ max_value: Precision::Exact(typed_null.clone()),+ min_value: Precision::Exact(typed_null.clone()),+ sum_value: Precision::Exact(typed_null),distinct_count: Precision::Exact(0),byte_size: input_column_stats[idx].byte_size,};@@ -1497,17 +1504,17 @@ mod tests {statistics.column_statistics,vec![ColumnStatistics {- min_value: Precision::Exact(ScalarValue::Null),- max_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ min_value: Precision::Exact(ScalarValue::Int32(None)),+ max_value: Precision::Exact(ScalarValue::Int32(None)),+ sum_value: Precision::Exact(ScalarValue::Int32(None)),distinct_count: Precision::Exact(0),null_count: Precision::Exact(0),byte_size: Precision::Absent,},ColumnStatistics {- min_value: Precision::Exact(ScalarValue::Null),- max_value: Precision::Exact(ScalarValue::Null),- sum_value: Precision::Exact(ScalarValue::Null),+ min_value: Precision::Exact(ScalarValue::Int32(None)),+ max_value: Precision::Exact(ScalarValue::Int32(None)),+ sum_value: Precision::Exact(ScalarValue::Int32(None)),distinct_count: Precision::Exact(0),null_count: Precision::Exact(0),byte_size: Precision::Absent,@@ -1518,6 +1525,70 @@ mod tests {Ok(())}+ /// Regression test: stacking two FilterExecs where the inner filter+ /// proves zero selectivity should not panic with a type mismatch+ /// during interval intersection.+ ///+ /// Previously, when a filter proved no rows could match, the column+ /// statistics used untyped `ScalarValue::Null` (data type `Null`).+ /// If an outer FilterExec then tried to analyze its own predicate+ /// against those statistics, `Interval::intersect` would fail with:+ /// "Only intervals with the same data type are intersectable, lhs:Null, rhs:Int32"+ #[tokio::test]+ async fn test_nested_filter_with_zero_selectivity_inner() -> Result<()> {+ // Inner table: a: [1, 100], b: [1, 3]+ let schema = Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]);+ let input = Arc::new(StatisticsExec::new(+ Statistics {+ num_rows: Precision::Inexact(1000),+ total_byte_size: Precision::Inexact(4000),+ column_statistics: vec![+ ColumnStatistics {+ min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),+ max_value: Precision::Inexact(ScalarValue::Int32(Some(100))),+ ..Default::default()+ },+ ColumnStatistics {+ min_value: Precision::Inexact(ScalarValue::Int32(Some(1))),+ max_value: Precision::Inexact(ScalarValue::Int32(Some(3))),+ ..Default::default()+ },+ ],+ },+ schema,+ ));++ // Inner filter: a > 200 (impossible given a max=100 → zero selectivity)+ let inner_predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Gt,+ Arc::new(Literal::new(ScalarValue::Int32(Some(200)))),+ ));+ let inner_filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(inner_predicate, input)?);++ // Outer filter: a = 50+ // Before the fix, this would panic because the inner filter's+ // zero-selectivity statistics produced Null-typed intervals for+ // column `a`, which couldn't intersect with the Int32 literal.+ let outer_predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(+ Arc::new(Column::new("a", 0)),+ Operator::Eq,+ Arc::new(Literal::new(ScalarValue::Int32(Some(50)))),+ ));+ let outer_filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExec::try_new(outer_predicate, inner_filter)?);++ // Should succeed without error+ let statistics = outer_filter.partition_statistics(None)?;+ assert_eq!(statistics.num_rows, Precision::Inexact(0));++ Ok(())+ }+#[tokio::test]async fn test_filter_statistics_more_inputs() -> Result<()> {let schema = Schema::new(vec![