Apache DataFusion SQL Query Engine
fix(stats): widen sum_value integer arithmetic to SUM-compatible types (#20865)
Kumar Ujjawal6e0dde0890ef
diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rsindex 7d3d511ca7..4cf5cc3661 100644--- a/datafusion/common/src/stats.rs+++ b/datafusion/common/src/stats.rs@@ -203,6 +203,24 @@ impl Precision<usize> {}impl Precision<ScalarValue> {+ fn sum_data_type(data_type: &DataType) -> DataType {+ match data_type {+ DataType::Int8 | DataType::Int16 | DataType::Int32 => DataType::Int64,+ DataType::UInt8 | DataType::UInt16 | DataType::UInt32 => DataType::UInt64,+ _ => data_type.clone(),+ }+ }++ fn cast_scalar_to_sum_type(value: &ScalarValue) -> Result<ScalarValue> {+ let source_type = value.data_type();+ let target_type = Self::sum_data_type(&source_type);+ if source_type == target_type {+ Ok(value.clone())+ } else {+ value.cast_to(&target_type)+ }+ }+/// Calculates the sum of two (possibly inexact) [`ScalarValue`] values,/// conservatively propagating exactness information. If one of the input/// values is [`Precision::Absent`], the result is `Absent` too.@@ -228,6 +246,31 @@ impl Precision<ScalarValue> {}}+ /// Casts integer values to the wider SQL `SUM` return type.+ ///+ /// This narrows overflow risk when `sum_value` statistics are merged:+ /// `Int8/Int16/Int32 -> Int64` and `UInt8/UInt16/UInt32 -> UInt64`.+ pub fn cast_to_sum_type(&self) -> Precision<ScalarValue> {+ match (self.is_exact(), self.get_value()) {+ (Some(true), Some(value)) => Self::cast_scalar_to_sum_type(value)+ .map(Precision::Exact)+ .unwrap_or(Precision::Absent),+ (Some(false), Some(value)) => Self::cast_scalar_to_sum_type(value)+ .map(Precision::Inexact)+ .unwrap_or(Precision::Absent),+ (_, _) => Precision::Absent,+ }+ }++ /// SUM-style addition with integer widening to match SQL `SUM` return+ /// types for smaller integral inputs.+ pub fn add_for_sum(&self, other: &Precision<ScalarValue>) -> Precision<ScalarValue> {+ let mut lhs = self.cast_to_sum_type();+ let rhs = other.cast_to_sum_type();+ precision_add(&mut lhs, &rhs);+ lhs+ }+/// Calculates the difference of two (possibly inexact) [`ScalarValue`] values,/// conservatively propagating exactness information. If one of the input/// values is [`Precision::Absent`], the result is `Absent` too.@@ -620,7 +663,7 @@ impl Statistics {/// assert_eq!(merged.column_statistics[0].max_value,/// Precision::Exact(ScalarValue::from(200)));/// assert_eq!(merged.column_statistics[0].sum_value,- /// Precision::Exact(ScalarValue::from(1500)));+ /// Precision::Exact(ScalarValue::Int64(Some(1500))));/// ```pub fn try_merge_iter<'a, I>(items: I, schema: &Schema) -> Result<Statistics>where@@ -664,7 +707,7 @@ impl Statistics {null_count: cs.null_count,max_value: cs.max_value.clone(),min_value: cs.min_value.clone(),- sum_value: cs.sum_value.clone(),+ sum_value: cs.sum_value.cast_to_sum_type(),distinct_count: cs.distinct_count,byte_size: cs.byte_size,})@@ -693,7 +736,8 @@ impl Statistics {};col_stats.min_value = col_stats.min_value.min(&item_cs.min_value);col_stats.max_value = col_stats.max_value.max(&item_cs.max_value);- precision_add(&mut col_stats.sum_value, &item_cs.sum_value);+ let item_sum_value = item_cs.sum_value.cast_to_sum_type();+ precision_add(&mut col_stats.sum_value, &item_sum_value);col_stats.byte_size = col_stats.byte_size.add(&item_cs.byte_size);}}@@ -877,7 +921,15 @@ pub struct ColumnStatistics {pub max_value: Precision<ScalarValue>,/// Minimum value of columnpub min_value: Precision<ScalarValue>,- /// Sum value of a column+ /// Sum value of a column.+ ///+ /// For integral columns, values should be kept in SUM-compatible widened+ /// types (`Int8/Int16/Int32 -> Int64`, `UInt8/UInt16/UInt32 -> UInt64`) to+ /// reduce overflow risk during statistics propagation.+ ///+ /// Callers should prefer [`ColumnStatistics::with_sum_value`] for setting+ /// this field and [`Precision<ScalarValue>::add_for_sum`] /+ /// [`Precision<ScalarValue>::cast_to_sum_type`] for sum arithmetic.pub sum_value: Precision<ScalarValue>,/// Number of distinct valuespub distinct_count: Precision<usize>,@@ -942,7 +994,19 @@ impl ColumnStatistics {/// Set the sum valuepub fn with_sum_value(mut self, sum_value: Precision<ScalarValue>) -> Self {- self.sum_value = sum_value;+ self.sum_value = match sum_value {+ Precision::Exact(value) => {+ Precision::<ScalarValue>::cast_scalar_to_sum_type(&value)+ .map(Precision::Exact)+ .unwrap_or(Precision::Absent)+ }+ Precision::Inexact(value) => {+ Precision::<ScalarValue>::cast_scalar_to_sum_type(&value)+ .map(Precision::Inexact)+ .unwrap_or(Precision::Absent)+ }+ Precision::Absent => Precision::Absent,+ };self}@@ -1095,6 +1159,45 @@ mod tests {assert_eq!(precision.add(&Precision::Absent), Precision::Absent);}+ #[test]+ fn test_add_for_sum_scalar_integer_widening() {+ let precision = Precision::Exact(ScalarValue::Int32(Some(42)));++ assert_eq!(+ precision.add_for_sum(&Precision::Exact(ScalarValue::Int32(Some(23)))),+ Precision::Exact(ScalarValue::Int64(Some(65))),+ );+ assert_eq!(+ precision.add_for_sum(&Precision::Inexact(ScalarValue::Int32(Some(23)))),+ Precision::Inexact(ScalarValue::Int64(Some(65))),+ );+ }++ #[test]+ fn test_add_for_sum_prevents_int32_overflow() {+ let lhs = Precision::Exact(ScalarValue::Int32(Some(i32::MAX)));+ let rhs = Precision::Exact(ScalarValue::Int32(Some(1)));++ assert_eq!(+ lhs.add_for_sum(&rhs),+ Precision::Exact(ScalarValue::Int64(Some(i64::from(i32::MAX) + 1))),+ );+ }++ #[test]+ fn test_add_for_sum_scalar_unsigned_integer_widening() {+ let precision = Precision::Exact(ScalarValue::UInt32(Some(42)));++ assert_eq!(+ precision.add_for_sum(&Precision::Exact(ScalarValue::UInt32(Some(23)))),+ Precision::Exact(ScalarValue::UInt64(Some(65))),+ );+ assert_eq!(+ precision.add_for_sum(&Precision::Inexact(ScalarValue::UInt32(Some(23)))),+ Precision::Inexact(ScalarValue::UInt64(Some(65))),+ );+ }+#[test]fn test_sub() {let precision1 = Precision::Exact(42);@@ -1340,7 +1443,7 @@ mod tests {);assert_eq!(col1_stats.sum_value,- Precision::Exact(ScalarValue::Int32(Some(1100)))+ Precision::Exact(ScalarValue::Int64(Some(1100)))); // 500 + 600let col2_stats = &summary_stats.column_statistics[1];@@ -1355,7 +1458,7 @@ mod tests {);assert_eq!(col2_stats.sum_value,- Precision::Exact(ScalarValue::Int32(Some(2200)))+ Precision::Exact(ScalarValue::Int64(Some(2200)))); // 1000 + 1200}@@ -1997,6 +2100,16 @@ mod tests {assert_eq!(col_stats.byte_size, Precision::Exact(8192));}+ #[test]+ fn test_with_sum_value_builder_widens_small_integers() {+ let col_stats = ColumnStatistics::new_unknown()+ .with_sum_value(Precision::Exact(ScalarValue::UInt32(Some(123))));+ assert_eq!(+ col_stats.sum_value,+ Precision::Exact(ScalarValue::UInt64(Some(123)))+ );+ }+#[test]fn test_with_fetch_scales_byte_size() {// Test that byte_size is scaled by the row ratio in with_fetch@@ -2144,7 +2257,7 @@ mod tests {);assert_eq!(col1_stats.sum_value,- Precision::Exact(ScalarValue::Int32(Some(1100)))+ Precision::Exact(ScalarValue::Int64(Some(1100))));let col2_stats = &summary_stats.column_statistics[1];@@ -2159,7 +2272,7 @@ mod tests {);assert_eq!(col2_stats.sum_value,- Precision::Exact(ScalarValue::Int32(Some(2200)))+ Precision::Exact(ScalarValue::Int64(Some(2200))));}@@ -2508,7 +2621,7 @@ mod tests {);assert_eq!(col_stats.sum_value,- Precision::Inexact(ScalarValue::Int32(Some(1500)))+ Precision::Inexact(ScalarValue::Int64(Some(1500))));}}diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rsindex b1a56e096c..e5a1e4613b 100644--- a/datafusion/datasource/src/statistics.rs+++ b/datafusion/datasource/src/statistics.rs@@ -293,7 +293,7 @@ fn sort_columns_from_physical_sort_exprs(since = "47.0.0",note = "Please use `get_files_with_limit` and `compute_all_files_statistics` instead")]-#[expect(unused)]+#[cfg_attr(not(test), expect(unused))]pub async fn get_statistics_with_limit(all_files: impl Stream<Item = Result<(PartitionedFile, Arc<Statistics>)>>,file_schema: SchemaRef,@@ -329,7 +329,7 @@ pub async fn get_statistics_with_limit(col_stats_set[index].null_count = file_column.null_count;col_stats_set[index].max_value = file_column.max_value;col_stats_set[index].min_value = file_column.min_value;- col_stats_set[index].sum_value = file_column.sum_value;+ col_stats_set[index].sum_value = file_column.sum_value.cast_to_sum_type();}// If the number of rows exceeds the limit, we can stop processing@@ -374,7 +374,7 @@ pub async fn get_statistics_with_limit(col_stats.null_count = col_stats.null_count.add(file_nc);col_stats.max_value = col_stats.max_value.max(file_max);col_stats.min_value = col_stats.min_value.min(file_min);- col_stats.sum_value = col_stats.sum_value.add(file_sum);+ col_stats.sum_value = col_stats.sum_value.add_for_sum(file_sum);col_stats.byte_size = col_stats.byte_size.add(file_sbs);}@@ -497,3 +497,78 @@ pub fn add_row_stats() -> Precision<usize> {file_num_rows.add(&num_rows)}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::PartitionedFile;+ use arrow::datatypes::{DataType, Field, Schema};+ use futures::stream;++ fn file_stats(sum: u32) -> Statistics {+ Statistics {+ num_rows: Precision::Exact(1),+ total_byte_size: Precision::Exact(4),+ column_statistics: vec![ColumnStatistics {+ null_count: Precision::Exact(0),+ max_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),+ min_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),+ sum_value: Precision::Exact(ScalarValue::UInt32(Some(sum))),+ distinct_count: Precision::Exact(1),+ byte_size: Precision::Exact(4),+ }],+ }+ }++ #[tokio::test]+ #[expect(deprecated)]+ async fn test_get_statistics_with_limit_casts_first_file_sum_to_sum_type()+ -> Result<()> {+ let schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));++ let files = stream::iter(vec![Ok((+ PartitionedFile::new("f1.parquet", 1),+ Arc::new(file_stats(100)),+ ))]);++ let (_group, stats) =+ get_statistics_with_limit(files, schema, None, false).await?;++ assert_eq!(+ stats.column_statistics[0].sum_value,+ Precision::Exact(ScalarValue::UInt64(Some(100)))+ );++ Ok(())+ }++ #[tokio::test]+ #[expect(deprecated)]+ async fn test_get_statistics_with_limit_merges_sum_with_unsigned_widening()+ -> Result<()> {+ let schema =+ Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt32, true)]));++ let files = stream::iter(vec![+ Ok((+ PartitionedFile::new("f1.parquet", 1),+ Arc::new(file_stats(100)),+ )),+ Ok((+ PartitionedFile::new("f2.parquet", 1),+ Arc::new(file_stats(200)),+ )),+ ]);++ let (_group, stats) =+ get_statistics_with_limit(files, schema, None, true).await?;++ assert_eq!(+ stats.column_statistics[0].sum_value,+ Precision::Exact(ScalarValue::UInt64(Some(300)))+ );++ Ok(())+ }+}diff --git a/datafusion/physical-expr/src/projection.rs b/datafusion/physical-expr/src/projection.rsindex b9f98c03da..e133e5a849 100644--- a/datafusion/physical-expr/src/projection.rs+++ b/datafusion/physical-expr/src/projection.rs@@ -693,12 +693,15 @@ impl ProjectionExprs {Precision::Absent};- let sum_value = Precision::<ScalarValue>::from(stats.num_rows)- .cast_to(&value.data_type())- .ok()- .map(|row_count| {- Precision::Exact(value.clone()).multiply(&row_count)+ let widened_sum = Precision::Exact(value.clone()).cast_to_sum_type();+ let sum_value = widened_sum+ .get_value()+ .and_then(|sum| {+ Precision::<ScalarValue>::from(stats.num_rows)+ .cast_to(&sum.data_type())+ .ok()})+ .map(|row_count| widened_sum.multiply(&row_count)).unwrap_or(Precision::Absent);ColumnStatistics {@@ -2864,6 +2867,35 @@ pub(crate) mod tests {Ok(())}+ #[test]+ fn test_project_statistics_with_i32_literal_sum_widens_to_i64() -> Result<()> {+ let input_stats = get_stats();+ let input_schema = get_schema();++ let projection = ProjectionExprs::new(vec![+ ProjectionExpr {+ expr: Arc::new(Literal::new(ScalarValue::Int32(Some(10)))),+ alias: "constant".to_string(),+ },+ ProjectionExpr {+ expr: Arc::new(Column::new("col0", 0)),+ alias: "num".to_string(),+ },+ ]);++ let output_stats = projection.project_statistics(+ input_stats,+ &projection.project_schema(&input_schema)?,+ )?;++ assert_eq!(+ output_stats.column_statistics[0].sum_value,+ Precision::Exact(ScalarValue::Int64(Some(50)))+ );++ Ok(())+ }+// Test statistics calculation for NULL literal (constant NULL column)#[test]fn test_project_statistics_with_null_literal() -> Result<()> {diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex a895f69dc5..b64de91d95 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -458,32 +458,34 @@ fn stats_cartesian_product(// Min, max and distinct_count on the other hand are invariants.let cross_join_stats = left_col_stats.into_iter()- .map(|s| ColumnStatistics {- null_count: s.null_count.multiply(&right_row_count),- distinct_count: s.distinct_count,- min_value: s.min_value,- max_value: s.max_value,- sum_value: s- .sum_value- .get_value()- // Cast the row count into the same type as any existing sum value- .and_then(|v| {- Precision::<ScalarValue>::from(right_row_count)- .cast_to(&v.data_type())- .ok()- })- .map(|row_count| s.sum_value.multiply(&row_count))- .unwrap_or(Precision::Absent),- byte_size: Precision::Absent,+ .map(|s| {+ let widened_sum = s.sum_value.cast_to_sum_type();+ ColumnStatistics {+ null_count: s.null_count.multiply(&right_row_count),+ distinct_count: s.distinct_count,+ min_value: s.min_value,+ max_value: s.max_value,+ sum_value: widened_sum+ .get_value()+ // Cast the row count into the same type as any existing sum value+ .and_then(|v| {+ Precision::<ScalarValue>::from(right_row_count)+ .cast_to(&v.data_type())+ .ok()+ })+ .map(|row_count| widened_sum.multiply(&row_count))+ .unwrap_or(Precision::Absent),+ byte_size: Precision::Absent,+ }}).chain(right_col_stats.into_iter().map(|s| {+ let widened_sum = s.sum_value.cast_to_sum_type();ColumnStatistics {null_count: s.null_count.multiply(&left_row_count),distinct_count: s.distinct_count,min_value: s.min_value,max_value: s.max_value,- sum_value: s- .sum_value+ sum_value: widened_sum.get_value()// Cast the row count into the same type as any existing sum value.and_then(|v| {@@ -491,7 +493,7 @@ fn stats_cartesian_product(.cast_to(&v.data_type()).ok()})- .map(|row_count| s.sum_value.multiply(&row_count))+ .map(|row_count| widened_sum.multiply(&row_count)).unwrap_or(Precision::Absent),byte_size: Precision::Absent,}@@ -875,6 +877,49 @@ mod tests {assert_eq!(result, expected);}+ #[tokio::test]+ async fn test_stats_cartesian_product_unsigned_sum_widens_to_u64() {+ let left_row_count = 2;+ let right_row_count = 3;++ let left = Statistics {+ num_rows: Precision::Exact(left_row_count),+ total_byte_size: Precision::Exact(10),+ column_statistics: vec![ColumnStatistics {+ distinct_count: Precision::Exact(2),+ max_value: Precision::Exact(ScalarValue::UInt32(Some(10))),+ min_value: Precision::Exact(ScalarValue::UInt32(Some(1))),+ sum_value: Precision::Exact(ScalarValue::UInt32(Some(7))),+ null_count: Precision::Exact(0),+ byte_size: Precision::Absent,+ }],+ };++ let right = Statistics {+ num_rows: Precision::Exact(right_row_count),+ total_byte_size: Precision::Exact(10),+ column_statistics: vec![ColumnStatistics {+ distinct_count: Precision::Exact(3),+ max_value: Precision::Exact(ScalarValue::UInt32(Some(12))),+ min_value: Precision::Exact(ScalarValue::UInt32(Some(0))),+ sum_value: Precision::Exact(ScalarValue::UInt32(Some(11))),+ null_count: Precision::Exact(0),+ byte_size: Precision::Absent,+ }],+ };++ let result = stats_cartesian_product(left, right);++ assert_eq!(+ result.column_statistics[0].sum_value,+ Precision::Exact(ScalarValue::UInt64(Some(21)))+ );+ assert_eq!(+ result.column_statistics[1].sum_value,+ Precision::Exact(ScalarValue::UInt64(Some(22)))+ );+ }+#[tokio::test]async fn test_join() -> Result<()> {let task_ctx = Arc::new(TaskContext::default());diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 218eb50015..eb16375a2d 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -857,7 +857,7 @@ fn col_stats_union(left.distinct_count = union_distinct_count(&left, right);left.min_value = left.min_value.min(&right.min_value);left.max_value = left.max_value.max(&right.max_value);- left.sum_value = left.sum_value.add(&right.sum_value);+ left.sum_value = left.sum_value.add_for_sum(&right.sum_value);left.null_count = left.null_count.add(&right.null_count);left