Apache DataFusion SQL Query Engine
fix: preserve uncertainty in singleton filter statistics (#25863)
Kumar Ujjawalb9298572aeee
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex f71d67d04e..dcaae294b6 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -1254,9 +1254,9 @@ fn scale_byte_size_at_rows(/// Returns the NDV for a column constrained to one non-null value (e.g./// `column = literal` or a singleton interval), derived from the filtered row-/// estimate: zero rows means zero distinct values, a known positive row count-/// means exactly one, and an unknown row count means an inexact one (the column-/// could still be empty).+/// estimate: zero rows means zero distinct values, an exact positive row count+/// means exactly one, and an estimated or unknown row count means an inexact+/// one (the column could still be empty).////// The caller is responsible for proving the singleton domain.fn distinct_count_for_singleton_domain(@@ -1264,10 +1264,10 @@ fn distinct_count_for_singleton_domain() -> Precision<usize> {match filtered_num_rows {Precision::Exact(0) | Precision::Inexact(0) => filtered_num_rows,- // The row count is unknown, so the column could still be empty (zero- // distinct values); report an inexact one rather than overstating it.- Precision::Absent => Precision::Inexact(1),- _ => Precision::Exact(1),+ Precision::Exact(_) => Precision::Exact(1),+ // The row count is not known exactly, so the column could still be+ // empty (zero distinct values); report an inexact one.+ Precision::Inexact(_) | Precision::Absent => Precision::Inexact(1),}}@@ -2990,7 +2990,7 @@ mod tests {Operator::Eq,Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("utf8view equality",@@ -3006,7 +3006,7 @@ mod tests {"hello".to_string(),)))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("largeutf8 equality",@@ -3022,7 +3022,7 @@ mod tests {"hello".to_string(),)))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("utf8 reversed (literal = column)",@@ -3036,7 +3036,7 @@ mod tests {Operator::Eq,Arc::new(Column::new("name", 0)),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("OR is not collapsed to NDV=1, but NDV is capped at filtered rows",@@ -3093,7 +3093,7 @@ mod tests {Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),)),)),- vec![Precision::Exact(1), Precision::Exact(1)],+ vec![Precision::Inexact(1), Precision::Inexact(1)],),("numeric equality with min/max bounds (interval analysis path)",@@ -3109,7 +3109,7 @@ mod tests {Operator::Eq,Arc::new(Literal::new(ScalarValue::Int32(Some(42)))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("timestamp equality",@@ -3130,7 +3130,7 @@ mod tests {None,))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("contradictory numeric equality (infeasible)",@@ -3166,7 +3166,7 @@ mod tests {Operator::Eq,Arc::new(Literal::new(ScalarValue::Utf8(Some("hello".to_string())))),)),- vec![Precision::Exact(1)],+ vec![Precision::Inexact(1)],),("contradictory utf8 equality (infeasible)",@@ -3231,7 +3231,7 @@ mod tests {Arc::new(Literal::new(ScalarValue::Int32(Some(2)))),)),)),- vec![Precision::Exact(1), Precision::Exact(1)],+ vec![Precision::Inexact(1), Precision::Inexact(1)],),];@@ -3529,7 +3529,7 @@ mod tests {// Equality predicates collapse NDV and reject nulls for their columns.assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));assert_eq!(statistics.column_statistics[0].null_count,@@ -3544,7 +3544,7 @@ mod tests {);assert_eq!(statistics.column_statistics[2].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));assert_eq!(statistics.column_statistics[2].null_count,@@ -3582,7 +3582,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -3616,7 +3616,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -3650,7 +3650,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -3684,7 +3684,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -3719,7 +3719,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -3766,7 +3766,7 @@ mod tests {StatisticsContext::new().compute(filter.as_ref(), &StatisticsArgs::new())?;assert_eq!(statistics.column_statistics[0].distinct_count,- Precision::Exact(1)+ Precision::Inexact(1));Ok(())}@@ -4174,6 +4174,39 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_filter_statistics_singleton_precision() -> Result<()> {+ use Precision::{Exact, Inexact};++ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));+ // The bounds include 5, but no row contains it.+ let values = Int32Array::from_iter_values(+ (0..1000).map(|i| if i % 2 == 0 { 1 } else { 100 }),+ );+ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(values)])?;+ let input = test::TestMemoryExec::try_new_exec(+ &[vec![batch]],+ Arc::clone(&schema),+ None,+ )?;+ let mut input_stats = StatisticsContext::new()+ .compute(input.as_ref(), &StatisticsArgs::new())?+ .as_ref()+ .clone();+ // Supply the bounds a file scan could report for these rows.+ input_stats.column_statistics[0].min_value = Exact(ScalarValue::Int32(Some(1)));+ input_stats.column_statistics[0].max_value = Exact(ScalarValue::Int32(Some(100)));+ let predicate = binary(col("a", &schema)?, Operator::Eq, lit(5i32), &schema)?;+ let filter = FilterExecBuilder::new(predicate, input).build()?;+ let stats = filter+ .statistics_from_inputs(&[Arc::new(input_stats)], &StatisticsArgs::new())?;+ let batches =+ collect(filter.execute(0, Arc::new(TaskContext::default()))?).await?;+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 0);+ assert_eq!(stats.column_statistics[0].distinct_count, Inexact(1));+ Ok(())+ }+#[tokio::test]async fn test_filter_statistics_is_not_null_rejects_nulls() -> Result<()> {let schema = Schema::new(vec![Field::new("name", DataType::Utf8, true)]);diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex 71de383370..2a8d174751 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -59,7 +59,7 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]@@ -84,7 +84,7 @@ query TTEXPLAIN SELECT * FROM test_table WHERE column1 = 1;----physical_plan-01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(10))]]+01)FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(10), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(10))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0) ScanBytes=Inexact(40))]]@@ -152,7 +152,7 @@ query TTEXPLAIN SELECT i8 FROM typed_table WHERE i8 = 2;----physical_plan-01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]+01)FilterExec: i8@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Int8(2)) Max=Exact(Int8(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i8], file_type=parquet, predicate=i8@0 = 2, pruning_predicate=i8_null_count@2 != row_count@3 AND i8_min@0 <= 2 AND 2 <= i8_max@1, required_guarantees=[i8 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(5), [(Col[0]: Min=Inexact(Int8(1)) Max=Inexact(Int8(5)) Null=Inexact(0) ScanBytes=Inexact(5))]]@@ -161,7 +161,7 @@ query TTEXPLAIN SELECT i64 FROM typed_table WHERE i64 = 2;----physical_plan-01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(8))]]+01)FilterExec: i64@0 = 2, statistics=[Rows=Inexact(1), Bytes=Inexact(8), [(Col[0]: Min=Exact(Int64(2)) Max=Exact(Int64(2)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(8))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[i64], file_type=parquet, predicate=i64@1 = 2, pruning_predicate=i64_null_count@2 != row_count@3 AND i64_min@0 <= 2 AND 2 <= i64_max@1, required_guarantees=[i64 in (2)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(5)) Null=Inexact(0) ScanBytes=Inexact(40))]]@@ -170,7 +170,7 @@ query TTEXPLAIN SELECT f32 FROM typed_table WHERE f32 = 2.5;----physical_plan-01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]+01)FilterExec: CAST(f32@0 AS Float64) = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float32(2.5)) Max=Exact(Float32(2.5)) Null=Inexact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f32], file_type=parquet, predicate=CAST(f32@2 AS Float64) = 2.5, pruning_predicate=f32_null_count@2 != row_count@3 AND CAST(f32_min@0 AS Float64) <= 2.5 AND 2.5 <= CAST(f32_max@1 AS Float64), required_guarantees=[], statistics=[Rows=Inexact(5), Bytes=Inexact(20), [(Col[0]: Min=Inexact(Float32(1.5)) Max=Inexact(Float32(5.5)) Null=Inexact(0) ScanBytes=Inexact(20))]]@@ -179,7 +179,7 @@ query TTEXPLAIN SELECT f64 FROM typed_table WHERE 2.5 = f64;----physical_plan-01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Exact(1) ScanBytes=Inexact(1))]]+01)FilterExec: f64@0 = 2.5, statistics=[Rows=Inexact(1), Bytes=Inexact(1), [(Col[0]: Min=Exact(Float64(2.5)) Max=Exact(Float64(2.5)) Null=Exact(0) Distinct=Inexact(1) ScanBytes=Inexact(1))]]02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/typed_table.parquet]]}, projection=[f64], file_type=parquet, predicate=f64@3 = 2.5, pruning_predicate=f64_null_count@2 != row_count@3 AND f64_min@0 <= 2.5 AND 2.5 <= f64_max@1, required_guarantees=[f64 in (2.5)], statistics=[Rows=Inexact(5), Bytes=Inexact(40), [(Col[0]: Min=Inexact(Float64(1.5)) Max=Inexact(Float64(5.5)) Null=Inexact(0) ScanBytes=Inexact(40))]]