Apache DataFusion SQL Query Engine
Fix literal projection distinct count statistics (#26385)
Masonca23256b4ab0
diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex fefb7eaf09..742d66ac42 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -753,14 +753,20 @@ impl ProjectionExprs {ColumnStatistics {min_value: Precision::Exact(literal.value().clone()),max_value: Precision::Exact(literal.value().clone()),- distinct_count: Precision::Exact(1),+ distinct_count: Precision::Exact(0),null_count,sum_value: Precision::Exact(literal.value().clone()),byte_size: Precision::Exact(0),}} else {let value = literal.value();- let distinct_count = Precision::Exact(1);+ let distinct_count = match stats.num_rows {+ Precision::Exact(0) | Precision::Inexact(0) => stats.num_rows,+ Precision::Exact(_) => Precision::Exact(1),+ Precision::Inexact(_) | Precision::Absent => {+ Precision::Inexact(1)+ }+ };let null_count = Precision::Exact(0);let byte_size = if let Some(byte_width) = data_type.primitive_width()@@ -3689,7 +3695,7 @@ pub(crate) mod tests {);assert_eq!(output_stats.column_statistics[0].distinct_count,- Precision::Exact(1) // All NULLs are considered the same+ Precision::Exact(0) // NULLs are not counted as distinct values);assert_eq!(output_stats.column_statistics[0].null_count,@@ -3717,6 +3723,53 @@ pub(crate) mod tests {Ok(())}+ // A literal's distinct count depends on whether any row is present: an+ // estimated row count cannot prove the value appears. NULLs are not+ // counted as distinct values, matching `COUNT(DISTINCT)`.+ #[test]+ fn test_project_statistics_literal_distinct_count_by_row_count() -> Result<()> {+ let input_schema = get_schema();+ let projection = ProjectionExprs::new(vec![+ ProjectionExpr {+ expr: Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),+ alias: "constant".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Literal::new(ScalarValue::Int64(None))),+ alias: "null_col".to_string(),+ },+ ]);+ let output_schema = projection.project_schema(&input_schema)?;++ let cases = [+ (Precision::Exact(0), Precision::Exact(0)),+ (Precision::Exact(5), Precision::Exact(1)),+ (Precision::Inexact(0), Precision::Inexact(0)),+ (Precision::Inexact(5), Precision::Inexact(1)),+ (Precision::Absent, Precision::Inexact(1)),+ ];+ for (num_rows, expected) in cases {+ let input_stats = Statistics {+ num_rows,+ total_byte_size: Precision::Absent,+ column_statistics: Statistics::unknown_column(&input_schema),+ };+ let output_stats =+ projection.project_statistics(input_stats, &output_schema)?;+ assert_eq!(+ output_stats.column_statistics[0].distinct_count, expected,+ "non-null literal, num_rows={num_rows:?}"+ );+ assert_eq!(+ output_stats.column_statistics[1].distinct_count,+ Precision::Exact(0),+ "NULL literal, num_rows={num_rows:?}"+ );+ }++ Ok(())+ }+// Test statistics calculation for complex type literal (e.g., Utf8 string)#[test]fn test_project_statistics_with_complex_type_literal() -> Result<()> {diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.sltindex e3fcee842e..35929ac5af 100644--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt@@ -337,6 +337,67 @@ physical_planstatement okDROP TABLE cast_stats_ints_overflow;+######+# Aggregate statistics through projected literals+######++statement ok+COPY (SELECT * FROM (VALUES (1, 10), (3, 30)) AS t(a, b))+TO 'test_files/scratch/parquet_statistics/literal_ndv.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE literal_ndv STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/literal_ndv.parquet';++# No row has a = 2, but the filter only estimates its row count.+query II+SELECT count(DISTINCT x), count(DISTINCT y)+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a = 2);+----+0 0++# The filter estimates zero rows, but not exactly.+query II+SELECT count(DISTINCT x), count(DISTINCT y)+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a > 100);+----+0 0++# An exact positive row count proves each literal is present once.+query II+SELECT count(DISTINCT x), count(DISTINCT y)+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv);+----+1 1++# COUNT(DISTINCT) does not count NULLs.+query II+SELECT count(DISTINCT x), count(DISTINCT y)+FROM (SELECT NULL AS x, 2 AS y FROM literal_ndv);+----+0 1++statement ok+DROP TABLE literal_ndv;++# An exactly empty file has no distinct values.+statement ok+COPY (SELECT * FROM (VALUES (1, 10)) AS t(a, b) WHERE a > 100)+TO 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet' STORED AS PARQUET;++statement ok+CREATE EXTERNAL TABLE literal_ndv_empty STORED AS PARQUET+LOCATION 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet';++query II+SELECT count(DISTINCT x), count(DISTINCT y)+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);+----+0 0++statement ok+DROP TABLE literal_ndv_empty;+statement okSET datafusion.execution.target_partitions = 4;