Skip to content

Apache DataFusion SQL Query Engine

feat: output statistics for constant columns in projections (#19419)

Shashidhar B M8959b3d11e8c
diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rs
index 8d4afb5d19..540fd620c9 100644
--- a/datafusion/physical-expr/src/projection.rs
+++ b/datafusion/physical-expr/src/projection.rs
@@ -21,15 +21,15 @@ use std::ops::Deref;
use std::sync::Arc;
use crate::PhysicalExpr;
-use crate::expressions::Column;
+use crate::expressions::{Column, Literal};
use crate::utils::collect_columns;
use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::datatypes::{Field, Schema, SchemaRef};
-use datafusion_common::stats::ColumnStatistics;
+use datafusion_common::stats::{ColumnStatistics, Precision};
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{
- Result, assert_or_internal_err, internal_datafusion_err, plan_err,
+ Result, ScalarValue, assert_or_internal_err, internal_datafusion_err, plan_err,
};
use datafusion_physical_expr_common::metrics::ExecutionPlanMetricsSet;
@@ -611,6 +611,54 @@ impl ProjectionExprs {
let expr = &proj_expr.expr;
let col_stats = if let Some(col) = expr.as_any().downcast_ref::<Column>() {
std::mem::take(&mut stats.column_statistics[col.index()])
+ } else if let Some(literal) = expr.as_any().downcast_ref::<Literal>() {
+ // Handle literal expressions (constants) by calculating proper statistics
+ let data_type = expr.data_type(output_schema)?;
+
+ if literal.value().is_null() {
+ let null_count = match stats.num_rows {
+ Precision::Exact(num_rows) => Precision::Exact(num_rows),
+ _ => Precision::Absent,
+ };
+
+ ColumnStatistics {
+ min_value: Precision::Exact(literal.value().clone()),
+ max_value: Precision::Exact(literal.value().clone()),
+ distinct_count: Precision::Exact(1),
+ null_count,
+ sum_value: Precision::Exact(literal.value().clone()),
+ byte_size: Precision::Exact(0),
+ }
+ } else {
+ let value = literal.value();
+ let distinct_count = Precision::Exact(1);
+ let null_count = Precision::Exact(0);
+
+ let byte_size = if let Some(byte_width) = data_type.primitive_width()
+ {
+ stats.num_rows.multiply(&Precision::Exact(byte_width))
+ } else {
+ // Complex types depend on array encoding, so set to Absent
+ Precision::Absent
+ };
+
+ let sum_value = Precision::<ScalarValue>::from(stats.num_rows)
+ .cast_to(&value.data_type())
+ .ok()
+ .map(|row_count| {
+ Precision::Exact(value.clone()).multiply(&row_count)
+ })
+ .unwrap_or(Precision::Absent);
+
+ ColumnStatistics {
+ min_value: Precision::Exact(value.clone()),
+ max_value: Precision::Exact(value.clone()),
+ distinct_count,
+ null_count,
+ sum_value,
+ byte_size,
+ }
+ }
} else {
// TODO stats: estimate more statistics from expressions
// (expressions should compute their statistics themselves)
@@ -2639,4 +2687,217 @@ pub(crate) mod tests {
Ok(())
}
+
+ // Test statistics calculation for non-null literal (numeric constant)
+ #[test]
+ fn test_project_statistics_with_literal() -> Result<()> {
+ let input_stats = get_stats();
+ let input_schema = get_schema();
+
+ // Projection with literal: SELECT 42 AS constant, col0 AS num
+ let projection = ProjectionExprs::new(vec![
+ ProjectionExpr {
+ expr: Arc::new(Literal::new(ScalarValue::Int64(Some(42)))),
+ alias: "constant".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(Column::new("col0", 0)),
+ alias: "num".to_string(),
+ },
+ ]);
+
+ let output_stats = projection.project_statistics(
+ input_stats,
+ &projection.project_schema(&input_schema)?,
+ )?;
+
+ // Row count should be preserved
+ assert_eq!(output_stats.num_rows, Precision::Exact(5));
+
+ // Should have 2 column statistics
+ assert_eq!(output_stats.column_statistics.len(), 2);
+
+ // First column (literal 42) should have proper constant statistics
+ assert_eq!(
+ output_stats.column_statistics[0].min_value,
+ Precision::Exact(ScalarValue::Int64(Some(42)))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].max_value,
+ Precision::Exact(ScalarValue::Int64(Some(42)))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].distinct_count,
+ Precision::Exact(1)
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
+ // Int64 is 8 bytes, 5 rows = 40 bytes
+ assert_eq!(
+ output_stats.column_statistics[0].byte_size,
+ Precision::Exact(40)
+ );
+ // For a constant column, sum_value = value * num_rows = 42 * 5 = 210
+ assert_eq!(
+ output_stats.column_statistics[0].sum_value,
+ Precision::Exact(ScalarValue::Int64(Some(210)))
+ );
+
+ // Second column (col0) should preserve statistics
+ assert_eq!(
+ output_stats.column_statistics[1].distinct_count,
+ Precision::Exact(5)
+ );
+ assert_eq!(
+ output_stats.column_statistics[1].max_value,
+ Precision::Exact(ScalarValue::Int64(Some(21)))
+ );
+
+ Ok(())
+ }
+
+ // Test statistics calculation for NULL literal (constant NULL column)
+ #[test]
+ fn test_project_statistics_with_null_literal() -> Result<()> {
+ let input_stats = get_stats();
+ let input_schema = get_schema();
+
+ // Projection with NULL literal: SELECT NULL AS null_col, col0 AS num
+ let projection = ProjectionExprs::new(vec![
+ ProjectionExpr {
+ expr: Arc::new(Literal::new(ScalarValue::Int64(None))),
+ alias: "null_col".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(Column::new("col0", 0)),
+ alias: "num".to_string(),
+ },
+ ]);
+
+ let output_stats = projection.project_statistics(
+ input_stats,
+ &projection.project_schema(&input_schema)?,
+ )?;
+
+ // Row count should be preserved
+ assert_eq!(output_stats.num_rows, Precision::Exact(5));
+
+ // Should have 2 column statistics
+ assert_eq!(output_stats.column_statistics.len(), 2);
+
+ // First column (NULL literal) should have proper constant NULL statistics
+ assert_eq!(
+ output_stats.column_statistics[0].min_value,
+ Precision::Exact(ScalarValue::Int64(None))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].max_value,
+ Precision::Exact(ScalarValue::Int64(None))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].distinct_count,
+ Precision::Exact(1) // All NULLs are considered the same
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].null_count,
+ Precision::Exact(5) // All rows are NULL
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].byte_size,
+ Precision::Exact(0)
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].sum_value,
+ Precision::Exact(ScalarValue::Int64(None))
+ );
+
+ // Second column (col0) should preserve statistics
+ assert_eq!(
+ output_stats.column_statistics[1].distinct_count,
+ Precision::Exact(5)
+ );
+ assert_eq!(
+ output_stats.column_statistics[1].max_value,
+ Precision::Exact(ScalarValue::Int64(Some(21)))
+ );
+
+ Ok(())
+ }
+
+ // Test statistics calculation for complex type literal (e.g., Utf8 string)
+ #[test]
+ fn test_project_statistics_with_complex_type_literal() -> Result<()> {
+ let input_stats = get_stats();
+ let input_schema = get_schema();
+
+ // Projection with Utf8 literal (complex type): SELECT 'hello' AS text, col0 AS num
+ let projection = ProjectionExprs::new(vec![
+ ProjectionExpr {
+ expr: Arc::new(Literal::new(ScalarValue::Utf8(Some(
+ "hello".to_string(),
+ )))),
+ alias: "text".to_string(),
+ },
+ ProjectionExpr {
+ expr: Arc::new(Column::new("col0", 0)),
+ alias: "num".to_string(),
+ },
+ ]);
+
+ let output_stats = projection.project_statistics(
+ input_stats,
+ &projection.project_schema(&input_schema)?,
+ )?;
+
+ // Row count should be preserved
+ assert_eq!(output_stats.num_rows, Precision::Exact(5));
+
+ // Should have 2 column statistics
+ assert_eq!(output_stats.column_statistics.len(), 2);
+
+ // First column (Utf8 literal 'hello') should have proper constant statistics
+ // but byte_size should be Absent for complex types
+ assert_eq!(
+ output_stats.column_statistics[0].min_value,
+ Precision::Exact(ScalarValue::Utf8(Some("hello".to_string())))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].max_value,
+ Precision::Exact(ScalarValue::Utf8(Some("hello".to_string())))
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].distinct_count,
+ Precision::Exact(1)
+ );
+ assert_eq!(
+ output_stats.column_statistics[0].null_count,
+ Precision::Exact(0)
+ );
+ // Complex types (Utf8, List, etc.) should have byte_size = Absent
+ // because we can't calculate exact size without knowing the actual data
+ assert_eq!(
+ output_stats.column_statistics[0].byte_size,
+ Precision::Absent
+ );
+ // Non-numeric types (Utf8) should have sum_value = Absent
+ // because sum is only meaningful for numeric types
+ assert_eq!(
+ output_stats.column_statistics[0].sum_value,
+ Precision::Absent
+ );
+
+ // Second column (col0) should preserve statistics
+ assert_eq!(
+ output_stats.column_statistics[1].distinct_count,
+ Precision::Exact(5)
+ );
+ assert_eq!(
+ output_stats.column_statistics[1].max_value,
+ Precision::Exact(ScalarValue::Int64(Some(21)))
+ );
+
+ Ok(())
+ }
}