Skip to content

Apache DataFusion SQL Query Engine

Fix literal projection distinct count statistics (#26385)

Masonca23256b4ab0
diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rs
index fefb7eaf09..742d66ac42 100644
--- a/datafusion/physical-expr/src/projection.rs
+++ b/datafusion/physical-expr/src/projection.rs
@@ -753,14 +753,20 @@ impl ProjectionExprs {
ColumnStatistics {
min_value: Precision::Exact(literal.value().clone()),
max_value: Precision::Exact(literal.value().clone()),
- distinct_count: Precision::Exact(1),
+ distinct_count: Precision::Exact(0),
null_count,
sum_value: Precision::Exact(literal.value().clone()),
byte_size: Precision::Exact(0),
}
} else {
let value = literal.value();
- let distinct_count = Precision::Exact(1);
+ let distinct_count = match stats.num_rows {
+ Precision::Exact(0) | Precision::Inexact(0) => stats.num_rows,
+ Precision::Exact(_) => Precision::Exact(1),
+ Precision::Inexact(_) | Precision::Absent => {
+ Precision::Inexact(1)
+ }
+ };
let null_count = Precision::Exact(0);
let byte_size = if let Some(byte_width) = data_type.primitive_width()
@@ -3689,7 +3695,7 @@ pub(crate) mod tests {
);
assert_eq!(
output_stats.column_statistics[0].distinct_count,
- Precision::Exact(1) // All NULLs are considered the same
+ Precision::Exact(0) // NULLs are not counted as distinct values
);
assert_eq!(
output_stats.column_statistics[0].null_count,
@@ -3717,6 +3723,53 @@ pub(crate) mod tests {
Ok(())
}
+ // A literal's distinct count depends on whether any row is present: an
+ // estimated row count cannot prove the value appears. NULLs are not
+ // counted as distinct values, matching `COUNT(DISTINCT)`.
+ #[test]
+ fn test_project_statistics_literal_distinct_count_by_row_count() -> Result<()> {
+ let input_schema = get_schema();
+ let projection = ProjectionExprs::new(vec![
+ ProjectionExpr {
+ expr: Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),
+ alias: "constant".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(Literal::new(ScalarValue::Int64(None))),
+ alias: "null_col".to_string(),
+ },
+ ]);
+ let output_schema = projection.project_schema(&input_schema)?;
+
+ let cases = [
+ (Precision::Exact(0), Precision::Exact(0)),
+ (Precision::Exact(5), Precision::Exact(1)),
+ (Precision::Inexact(0), Precision::Inexact(0)),
+ (Precision::Inexact(5), Precision::Inexact(1)),
+ (Precision::Absent, Precision::Inexact(1)),
+ ];
+ for (num_rows, expected) in cases {
+ let input_stats = Statistics {
+ num_rows,
+ total_byte_size: Precision::Absent,
+ column_statistics: Statistics::unknown_column(&input_schema),
+ };
+ let output_stats =
+ projection.project_statistics(input_stats, &output_schema)?;
+ assert_eq!(
+ output_stats.column_statistics[0].distinct_count, expected,
+ "non-null literal, num_rows={num_rows:?}"
+ );
+ assert_eq!(
+ output_stats.column_statistics[1].distinct_count,
+ Precision::Exact(0),
+ "NULL literal, num_rows={num_rows:?}"
+ );
+ }
+
+ Ok(())
+ }
+
// Test statistics calculation for complex type literal (e.g., Utf8 string)
#[test]
fn test_project_statistics_with_complex_type_literal() -> Result<()> {
diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt
index e3fcee842e..35929ac5af 100644
--- a/datafusion/sqllogictest/test_files/parquet_statistics.slt
+++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt
@@ -337,6 +337,67 @@ physical_plan
statement ok
DROP TABLE cast_stats_ints_overflow;
+######
+# Aggregate statistics through projected literals
+######
+
+statement ok
+COPY (SELECT * FROM (VALUES (1, 10), (3, 30)) AS t(a, b))
+TO 'test_files/scratch/parquet_statistics/literal_ndv.parquet' STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE literal_ndv STORED AS PARQUET
+LOCATION 'test_files/scratch/parquet_statistics/literal_ndv.parquet';
+
+# No row has a = 2, but the filter only estimates its row count.
+query II
+SELECT count(DISTINCT x), count(DISTINCT y)
+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a = 2);
+----
+0 0
+
+# The filter estimates zero rows, but not exactly.
+query II
+SELECT count(DISTINCT x), count(DISTINCT y)
+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv WHERE a > 100);
+----
+0 0
+
+# An exact positive row count proves each literal is present once.
+query II
+SELECT count(DISTINCT x), count(DISTINCT y)
+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv);
+----
+1 1
+
+# COUNT(DISTINCT) does not count NULLs.
+query II
+SELECT count(DISTINCT x), count(DISTINCT y)
+FROM (SELECT NULL AS x, 2 AS y FROM literal_ndv);
+----
+0 1
+
+statement ok
+DROP TABLE literal_ndv;
+
+# An exactly empty file has no distinct values.
+statement ok
+COPY (SELECT * FROM (VALUES (1, 10)) AS t(a, b) WHERE a > 100)
+TO 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet' STORED AS PARQUET;
+
+statement ok
+CREATE EXTERNAL TABLE literal_ndv_empty STORED AS PARQUET
+LOCATION 'test_files/scratch/parquet_statistics/literal_ndv_empty.parquet';
+
+query II
+SELECT count(DISTINCT x), count(DISTINCT y)
+FROM (SELECT 1 AS x, 2 AS y FROM literal_ndv_empty);
+----
+0 0
+
+statement ok
+DROP TABLE literal_ndv_empty;
+
statement ok
SET datafusion.execution.target_partitions = 4;