Apache DataFusion SQL Query Engine
perf: implement convert_to_state for SparkAvg (#21548)
Alex Zhangca1d39dfca53
diff --git a/Cargo.lock b/Cargo.lockindex ed3c4825bd..9d60540d9c 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2632,6 +2632,7 @@ dependencies = ["datafusion-expr","datafusion-functions","datafusion-functions-aggregate",+ "datafusion-functions-aggregate-common","datafusion-functions-nested","log","num-traits",diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.tomlindex 0651fe5651..72bca58b7a 100644--- a/datafusion/spark/Cargo.toml+++ b/datafusion/spark/Cargo.toml@@ -55,6 +55,7 @@ datafusion-execution = { workspace = true }datafusion-expr = { workspace = true }datafusion-functions = { workspace = true, features = ["crypto_expressions"] }datafusion-functions-aggregate = { workspace = true }+datafusion-functions-aggregate-common = { workspace = true }datafusion-functions-nested = { workspace = true }log = { workspace = true }num-traits = { workspace = true }diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rsindex c504f5387f..5f4d2c253a 100644--- a/datafusion/spark/src/function/aggregate/avg.rs+++ b/datafusion/spark/src/function/aggregate/avg.rs@@ -16,7 +16,8 @@// under the License.use arrow::array::{- Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, Int64Array, PrimitiveArray,+ Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, BooleanArray, Int64Array,+ PrimitiveArray,builder::PrimitiveBuilder,cast::AsArray,types::{Float64Type, Int64Type},@@ -31,6 +32,9 @@ use datafusion_expr::{Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,Signature, TypeSignatureClass, Volatility,};+use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::{+ filtered_null_mask, set_nulls,+};use std::sync::Arc;/// AVG aggregate expression@@ -248,7 +252,7 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- _opt_filter: Option<&arrow::array::BooleanArray>,+ _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "single argument to update_batch");@@ -285,26 +289,26 @@ where&mut self,values: &[ArrayRef],group_indices: &[usize],- _opt_filter: Option<&arrow::array::BooleanArray>,+ _opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 2, "two arguments to merge_batch");// first batch is partial sums, second is countslet partial_sums = values[0].as_primitive::<T>();let partial_counts = values[1].as_primitive::<Int64Type>();- // update counts with partial counts- self.counts.resize(total_num_groups, 0);- let iter1 = group_indices.iter().zip(partial_counts.values().iter());- for (&group_index, &partial_count) in iter1 {- self.counts[group_index] += partial_count;- }- // update sums+ self.counts.resize(total_num_groups, 0);self.sums.resize(total_num_groups, T::default_value());- let iter2 = group_indices.iter().zip(partial_sums.values().iter());- for (&group_index, &new_value) in iter2 {++ for (idx, &group_index) in group_indices.iter().enumerate() {+ // Skip null state entries emitted by convert_to_state for+ // filtered / null input rows.+ if partial_counts.is_null(idx) || partial_sums.is_null(idx) {+ continue;+ }+ self.counts[group_index] += partial_counts.value(idx);let sum = &mut self.sums[group_index];- *sum = sum.add_wrapping(new_value);+ *sum = sum.add_wrapping(partial_sums.value(idx));}Ok(())@@ -343,7 +347,149 @@ where])}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ let sums = values[0]+ .as_primitive::<T>()+ .clone()+ .with_data_type(self.return_data_type.clone());+ let counts = Int64Array::from_value(1, sums.len());++ let nulls = filtered_null_mask(opt_filter, &sums);+ let counts = set_nulls(counts, nulls.clone());+ let sums = set_nulls(sums, nulls);++ // [sum, count] - must match state() and merge_batch()+ Ok(vec![+ Arc::new(sums) as ArrayRef,+ Arc::new(counts) as ArrayRef,+ ])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {self.counts.capacity() * size_of::<i64>() + self.sums.capacity() * size_of::<T>()}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::Float64Array;++ fn make_acc() -> AvgGroupsAccumulator<Float64Type, impl Fn(f64, i64) -> Result<f64>> {+ AvgGroupsAccumulator::<Float64Type, _>::new(&DataType::Float64, |sum, count| {+ Ok(sum / count as f64)+ })+ }++ #[test]+ fn supports_convert_to_state() {+ assert!(make_acc().supports_convert_to_state());+ }++ #[test]+ fn convert_to_state_basic() {+ let acc = make_acc();+ let values: Vec<ArrayRef> =+ vec![Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))];+ let state = acc.convert_to_state(&values, None).unwrap();++ assert_eq!(state.len(), 2);+ let sums = state[0].as_primitive::<Float64Type>();+ let counts = state[1].as_primitive::<Int64Type>();++ assert_eq!(sums.values().as_ref(), &[1.0, 2.0, 3.0]);+ assert_eq!(counts.values().as_ref(), &[1, 1, 1]);+ assert_eq!(sums.null_count(), 0);+ assert_eq!(counts.null_count(), 0);+ }++ #[test]+ fn convert_to_state_with_nulls() {+ let acc = make_acc();+ let values: Vec<ArrayRef> = vec![Arc::new(Float64Array::from(vec![+ Some(1.0),+ None,+ Some(3.0),+ ]))];+ let state = acc.convert_to_state(&values, None).unwrap();++ let sums = state[0].as_primitive::<Float64Type>();+ let counts = state[1].as_primitive::<Int64Type>();++ assert!(!sums.is_null(0));+ assert!(sums.is_null(1));+ assert!(!sums.is_null(2));++ assert_eq!(counts.value(0), 1);+ assert!(counts.is_null(1));+ assert_eq!(counts.value(2), 1);+ }++ #[test]+ fn convert_to_state_with_filter() {+ let acc = make_acc();+ let values: Vec<ArrayRef> =+ vec![Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))];+ let filter = BooleanArray::from(vec![true, false, true]);+ let state = acc.convert_to_state(&values, Some(&filter)).unwrap();++ let sums = state[0].as_primitive::<Float64Type>();+ let counts = state[1].as_primitive::<Int64Type>();++ assert!(!sums.is_null(0));+ assert!(sums.is_null(1));+ assert!(!sums.is_null(2));++ assert_eq!(counts.value(0), 1);+ assert!(counts.is_null(1));+ assert_eq!(counts.value(2), 1);+ }++ #[test]+ fn convert_to_state_roundtrips_through_merge() {+ let mut acc = make_acc();+ let input: Vec<ArrayRef> =+ vec![Arc::new(Float64Array::from(vec![10.0, 20.0, 30.0]))];+ let state = acc.convert_to_state(&input, None).unwrap();++ // feed the converted state back through merge_batch+ acc.merge_batch(+ &state,+ &[0, 0, 0],+ None,+ 1, // single group+ )+ .unwrap();++ let result = acc.evaluate(EmitTo::All).unwrap();+ let result = result.as_primitive::<Float64Type>();+ assert_eq!(result.value(0), 20.0); // (10+20+30)/3+ }++ #[test]+ fn convert_to_state_null_merge_matches_direct() {+ // avg([1.0, NULL, 3.0]) must be 2.0 after a convert_to_state → merge_batch+ // round-trip. Before the merge-path null fix this leaked the backing+ // buffer value at the null slot and produced the wrong average.+ let mut acc = make_acc();+ let input: Vec<ArrayRef> = vec![Arc::new(Float64Array::from(vec![+ Some(1.0),+ None,+ Some(3.0),+ ]))];+ let state = acc.convert_to_state(&input, None).unwrap();+ acc.merge_batch(&state, &[0, 0, 0], None, 1).unwrap();++ let result = acc.evaluate(EmitTo::All).unwrap();+ let result = result.as_primitive::<Float64Type>();+ assert_eq!(result.value(0), 2.0);+ }+}