Apache DataFusion SQL Query Engine
feat: Implement state conversion for remaining group accumulators (#23275)
linfeng3e058f09c91a
diff --git a/datafusion-examples/examples/udf/advanced_udaf.rs b/datafusion-examples/examples/udf/advanced_udaf.rsindex b990740159..096753d2b5 100644--- a/datafusion-examples/examples/udf/advanced_udaf.rs+++ b/datafusion-examples/examples/udf/advanced_udaf.rs@@ -23,8 +23,10 @@ use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility};use std::sync::Arc;use arrow::array::{- ArrayRef, AsArray, Float32Array, PrimitiveArray, PrimitiveBuilder, UInt32Array,+ Array, ArrayRef, AsArray, BooleanArray, Float32Array, PrimitiveArray,+ PrimitiveBuilder, UInt32Array,};+use arrow::buffer::NullBuffer;use arrow::datatypes::{ArrowNativeTypeOp, ArrowPrimitiveType, Float64Type, UInt32Type};use arrow::record_batch::RecordBatch;use arrow_schema::FieldRef;@@ -237,7 +239,7 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&arrow::array::BooleanArray>,+ opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {assert_eq!(values.len(), 1, "single argument to update_batch");@@ -359,6 +361,43 @@ impl GroupsAccumulator for GeometricMeanGroupsAccumulator {])}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ assert_eq!(values.len(), 1, "single argument to convert_to_state");++ let prods = values[0]+ .as_primitive::<Float64Type>()+ .clone()+ .with_data_type(self.prod_data_type.clone());+ let counts = UInt32Array::from_value(1, prods.len());++ let filter_nulls = opt_filter.map(|filter| {+ let validity = match filter.nulls() {+ Some(nulls) => filter.values() & nulls.inner(),+ None => filter.values().clone(),+ };+ NullBuffer::new(validity)+ });+ let nulls = NullBuffer::union(filter_nulls.as_ref(), prods.nulls());++ let prods =+ PrimitiveArray::<Float64Type>::new(prods.values().clone(), nulls.clone())+ .with_data_type(self.prod_data_type.clone());+ let counts = UInt32Array::new(counts.values().clone(), nulls);++ Ok(vec![+ Arc::new(prods) as ArrayRef,+ Arc::new(counts) as ArrayRef,+ ])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {self.counts.capacity() * size_of::<u32>()+ self.prods.capacity() * size_of::<Float64Type>()diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rsindex b895cb9c7c..1d4b222301 100644--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs@@ -877,6 +877,22 @@ impl GroupsAccumulator for TestGroupsAccumulator {Ok(())}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ _opt_filter: Option<&arrow::array::BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ let len = values.first().map_or(0, |value| value.len());+ Ok(vec![+ Arc::new(PrimitiveArray::<UInt64Type>::from_value(self.result, len))+ as ArrayRef,+ ])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {size_of::<u64>()}diff --git a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rsindex 60fe0388c4..986d4ec0d7 100644--- a/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/count_distinct/groups.rs@@ -16,7 +16,8 @@// under the License.use arrow::array::{- ArrayRef, AsArray, BooleanArray, Int64Array, ListArray, PrimitiveArray,+ Array, ArrayRef, AsArray, BooleanArray, Int64Array, ListArray, ListBuilder,+ PrimitiveArray, PrimitiveBuilder,};use arrow::buffer::{OffsetBuffer, ScalarBuffer};use arrow::datatypes::{ArrowPrimitiveType, Field};@@ -182,9 +183,127 @@ whereOk(())}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> datafusion_common::Result<Vec<ArrayRef>> {+ debug_assert_eq!(values.len(), 1);+ let arr = values[0].as_primitive::<T>();++ let values_builder = PrimitiveBuilder::<T>::with_capacity(arr.len());+ let mut builder = ListBuilder::new(values_builder)+ .with_field(Arc::new(Field::new_list_field(T::DATA_TYPE, true)));++ for row in 0..arr.len() {+ let included = arr.is_valid(row)+ && opt_filter+ .is_none_or(|filter| filter.is_valid(row) && filter.value(row));+ if included {+ builder.values().append_value(arr.value(row));+ }+ builder.append(true);+ }++ Ok(vec![Arc::new(builder.finish())])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {size_of::<Self>()+ self.seen.capacity() * (size_of::<(usize, T::Native)>() + size_of::<u64>())+ self.counts.capacity() * size_of::<i64>()}}++#[cfg(test)]+mod tests {+ use super::*;+ use arrow::array::Int32Array;+ use arrow::datatypes::Int32Type;+ use datafusion_common::Result;++ #[test]+ fn convert_to_state_roundtrips_through_merge() -> Result<()> {+ let values = Arc::new(Int32Array::from(vec![+ Some(1),+ Some(2),+ Some(2),+ None,+ Some(3),+ Some(4),+ Some(5),+ Some(5),+ ])) as ArrayRef;+ let filter = BooleanArray::from(vec![+ Some(true),+ Some(true),+ Some(true),+ Some(true),+ None,+ Some(true),+ Some(true),+ Some(true),+ ]);+ let group_indices = vec![0usize, 1, 0, 1, 0, 0, 0, 0];++ let mut direct = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();+ direct.update_batch(+ std::slice::from_ref(&values),+ &group_indices,+ Some(&filter),+ 2,+ )?;+ let direct = direct.evaluate(EmitTo::All)?;++ let converter = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();+ let state =+ converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;+ assert_eq!(state[0].null_count(), 0);+ let mut merged = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();+ merged.merge_batch(&state, &group_indices, 2)?;+ let merged = merged.evaluate(EmitTo::All)?;++ assert_eq!(+ direct.as_any().downcast_ref::<Int64Array>().unwrap(),+ merged.as_any().downcast_ref::<Int64Array>().unwrap()+ );+ Ok(())+ }++ #[test]+ fn convert_to_state_preserves_empty_and_filtered_rows() -> Result<()> {+ let converter = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();+ let empty_values =+ Arc::new(Int32Array::from(Vec::<Option<i32>>::new())) as ArrayRef;+ let state =+ converter.convert_to_state(std::slice::from_ref(&empty_values), None)?;+ assert_eq!(state[0].len(), 0);+ assert_eq!(state[0].null_count(), 0);++ let values = Arc::new(Int32Array::from(vec![Some(1), Some(2), None])) as ArrayRef;+ let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);+ let group_indices = vec![0usize, 1, 0];++ let state =+ converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;+ assert_eq!(state[0].len(), values.len());+ assert_eq!(state[0].null_count(), 0);+ let list_state = state[0].as_list::<i32>();+ for row in 0..list_state.len() {+ assert_eq!(list_state.value_length(row), 0);+ }++ let mut merged = PrimitiveDistinctCountGroupsAccumulator::<Int32Type>::new();+ merged.merge_batch(&state, &group_indices, 2)?;+ let result = merged.evaluate(EmitTo::All)?;+ assert_eq!(+ result.as_any().downcast_ref::<Int64Array>().unwrap(),+ &Int64Array::from(vec![0, 0])+ );+ Ok(())+ }+}diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex 0e35b47d64..74bc9ad6cb 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -583,6 +583,43 @@ impl GroupsAccumulator for HllGroupsAccumulator {Ok(vec![Arc::new(builder.finish())])}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ assert_eq!(values.len(), 1, "single argument to convert_to_state");+ let array = values[0].as_ref();+ let mut hashes = vec![0; array.len()];+ create_hashes([array], &HLL_HASH_STATE, &mut hashes)?;++ let filter_nulls = opt_filter.map(filter_to_nulls);+ let value_nulls = array.logical_nulls();+ let combined_nulls =+ NullBuffer::union(filter_nulls.as_ref(), value_nulls.as_ref());++ let mut builder = BinaryBuilder::new();+ let mut scratch = Vec::new();+ for (row, hash) in hashes.into_iter().enumerate() {+ if combined_nulls+ .as_ref()+ .is_none_or(|nulls| nulls.is_valid(row))+ {+ scratch.clear();+ scratch.extend_from_slice(&hash.to_le_bytes());+ builder.append_value(&scratch);+ } else {+ builder.append_value([]);+ }+ }++ Ok(vec![Arc::new(builder.finish())])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {self.groups.capacity() * size_of::<GroupHll>()+ self.allocated_bytes@@ -1081,6 +1118,98 @@ mod tests {assert_eq!(counts.value(0), expected);}+ #[test]+ fn groups_convert_to_state_roundtrips_through_merge() {+ let values: ArrayRef = Arc::new(Int64Array::from(vec![+ Some(1),+ Some(2),+ Some(2),+ None,+ Some(3),+ ]));+ let filter = BooleanArray::from(vec![+ Some(true),+ Some(true),+ Some(true),+ Some(true),+ None,+ ]);+ let group_indices = vec![0usize, 1, 0, 1, 0];++ let mut direct = HllGroupsAccumulator::new();+ direct+ .update_batch(+ std::slice::from_ref(&values),+ &group_indices,+ Some(&filter),+ 2,+ )+ .unwrap();+ let direct = direct+ .evaluate(EmitTo::All)+ .unwrap()+ .as_any()+ .downcast_ref::<UInt64Array>()+ .unwrap()+ .clone();++ let converter = HllGroupsAccumulator::new();+ let state = converter+ .convert_to_state(std::slice::from_ref(&values), Some(&filter))+ .unwrap();+ assert_eq!(state[0].null_count(), 0);+ let mut merged = HllGroupsAccumulator::new();+ merged.merge_batch(&state, &group_indices, 2).unwrap();+ let merged = merged+ .evaluate(EmitTo::All)+ .unwrap()+ .as_any()+ .downcast_ref::<UInt64Array>()+ .unwrap()+ .clone();++ assert_eq!(direct, merged);+ }++ #[test]+ fn groups_convert_to_state_preserves_empty_and_filtered_rows() {+ let converter = HllGroupsAccumulator::new();+ let empty_values: ArrayRef =+ Arc::new(Int64Array::from(Vec::<Option<i64>>::new()));+ let state = converter+ .convert_to_state(std::slice::from_ref(&empty_values), None)+ .unwrap();+ assert_eq!(state[0].len(), 0);+ assert_eq!(state[0].null_count(), 0);++ let values: ArrayRef =+ Arc::new(Int64Array::from(vec![Some(1), Some(2), None]));+ let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);+ let group_indices = vec![0usize, 1, 0];+ let state = converter+ .convert_to_state(std::slice::from_ref(&values), Some(&filter))+ .unwrap();+ assert_eq!(state[0].len(), values.len());+ assert_eq!(state[0].null_count(), 0);+ let state = state[0].as_any().downcast_ref::<BinaryArray>().unwrap();+ for row in 0..state.len() {+ assert_eq!(state.value(row), b"");+ }++ let mut merged = HllGroupsAccumulator::new();+ merged+ .merge_batch(&[Arc::new(state.clone())], &group_indices, 2)+ .unwrap();+ let result = merged+ .evaluate(EmitTo::All)+ .unwrap()+ .as_any()+ .downcast_ref::<UInt64Array>()+ .unwrap()+ .clone();+ assert_eq!(result, UInt64Array::from(vec![0, 0]));+ }+/// Regression: a short (≤ 12-byte) Utf8View string must hash identically/// in an all-inline batch and in a mixed batch that also contains a long/// string (which forces a data buffer).diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rsindex 7fcf4bb61f..2e90cac6d9 100644--- a/datafusion/functions-aggregate/src/correlation.rs+++ b/datafusion/functions-aggregate/src/correlation.rs@@ -489,6 +489,61 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {])}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ assert_eq!(values.len(), 2, "two arguments to convert_to_state");+ let array_x = downcast_array::<Float64Array>(&values[0]);+ let array_y = downcast_array::<Float64Array>(&values[1]);++ let len = array_x.len();+ let mut counts = Vec::with_capacity(len);+ let mut sum_x = Vec::with_capacity(len);+ let mut sum_y = Vec::with_capacity(len);+ let mut sum_xy = Vec::with_capacity(len);+ let mut sum_xx = Vec::with_capacity(len);+ let mut sum_yy = Vec::with_capacity(len);++ for row in 0..len {+ let included = array_x.is_valid(row)+ && array_y.is_valid(row)+ && opt_filter+ .is_none_or(|filter| filter.is_valid(row) && filter.value(row));+ if included {+ let x = array_x.value(row);+ let y = array_y.value(row);+ counts.push(1);+ sum_x.push(x);+ sum_y.push(y);+ sum_xy.push(x * y);+ sum_xx.push(x * x);+ sum_yy.push(y * y);+ } else {+ counts.push(0);+ sum_x.push(0.0);+ sum_y.push(0.0);+ sum_xy.push(0.0);+ sum_xx.push(0.0);+ sum_yy.push(0.0);+ }+ }++ Ok(vec![+ Arc::new(UInt64Array::from(counts)),+ Arc::new(Float64Array::from(sum_x)),+ Arc::new(Float64Array::from(sum_y)),+ Arc::new(Float64Array::from(sum_xy)),+ Arc::new(Float64Array::from(sum_xx)),+ Arc::new(Float64Array::from(sum_yy)),+ ])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn merge_batch(&mut self,values: &[ArrayRef],@@ -593,4 +648,90 @@ mod tests {});assert!(result.is_err());}++ #[test]+ fn convert_to_state_roundtrips_through_merge() -> Result<()> {+ let x = Arc::new(Float64Array::from(vec![+ Some(1.0),+ Some(2.0),+ None,+ Some(4.0),+ Some(8.0),+ Some(16.0),+ Some(32.0),+ ])) as ArrayRef;+ let y = Arc::new(Float64Array::from(vec![+ Some(2.0),+ Some(4.0),+ Some(6.0),+ None,+ Some(16.0),+ Some(32.0),+ Some(64.0),+ ])) as ArrayRef;+ let filter = BooleanArray::from(vec![+ Some(true),+ Some(false),+ Some(true),+ Some(true),+ None,+ Some(true),+ Some(true),+ ]);+ let values = vec![x, y];+ let group_indices = vec![0, 1, 0, 1, 0, 0, 0];++ let mut direct = CorrelationGroupsAccumulator::new();+ direct.update_batch(&values, &group_indices, Some(&filter), 2)?;+ let direct = direct.evaluate(EmitTo::All)?;++ let converter = CorrelationGroupsAccumulator::new();+ let state = converter.convert_to_state(&values, Some(&filter))?;+ let mut merged = CorrelationGroupsAccumulator::new();+ merged.merge_batch(&state, &group_indices, 2)?;+ let merged = merged.evaluate(EmitTo::All)?;++ assert_eq!(+ direct.as_any().downcast_ref::<Float64Array>().unwrap(),+ merged.as_any().downcast_ref::<Float64Array>().unwrap()+ );+ Ok(())+ }++ #[test]+ fn convert_to_state_preserves_empty_and_filtered_rows() -> Result<()> {+ let converter = CorrelationGroupsAccumulator::new();+ let empty_values = vec![+ Arc::new(Float64Array::from(Vec::<Option<f64>>::new())) as ArrayRef,+ Arc::new(Float64Array::from(Vec::<Option<f64>>::new())) as ArrayRef,+ ];+ let state = converter.convert_to_state(&empty_values, None)?;+ for state_array in &state {+ assert_eq!(state_array.len(), 0);+ assert_eq!(state_array.null_count(), 0);+ }++ let values = vec![+ Arc::new(Float64Array::from(vec![Some(1.0), Some(2.0), None])) as ArrayRef,+ Arc::new(Float64Array::from(vec![Some(2.0), None, Some(4.0)])) as ArrayRef,+ ];+ let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);+ let group_indices = vec![0, 1, 0];+ let state = converter.convert_to_state(&values, Some(&filter))?;+ for state_array in &state {+ assert_eq!(state_array.len(), values[0].len());+ assert_eq!(state_array.null_count(), 0);+ }++ let counts = state[0].as_any().downcast_ref::<UInt64Array>().unwrap();+ assert_eq!(counts, &UInt64Array::from(vec![0, 0, 0]));++ let mut merged = CorrelationGroupsAccumulator::new();+ merged.merge_batch(&state, &group_indices, 2)?;+ let result = merged.evaluate(EmitTo::All)?;+ let result = result.as_any().downcast_ref::<Float64Array>().unwrap();+ assert_eq!(result.len(), 2);+ assert_eq!(result.null_count(), 2);+ Ok(())+ }}diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rsindex f0482b23d1..a31517b93e 100644--- a/datafusion/functions-aggregate/src/stddev.rs+++ b/datafusion/functions-aggregate/src/stddev.rs@@ -22,7 +22,7 @@ use std::hash::Hash;use std::mem::align_of_val;use std::sync::Arc;-use arrow::array::Float64Array;+use arrow::array::{BooleanArray, Float64Array};use arrow::datatypes::FieldRef;use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field};use datafusion_common::ScalarValue;@@ -318,7 +318,7 @@ impl GroupsAccumulator for StddevGroupsAccumulator {&mut self,values: &[ArrayRef],group_indices: &[usize],- opt_filter: Option<&arrow::array::BooleanArray>,+ opt_filter: Option<&BooleanArray>,total_num_groups: usize,) -> Result<()> {self.variance@@ -345,6 +345,18 @@ impl GroupsAccumulator for StddevGroupsAccumulator {self.variance.state(emit_to)}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ self.variance.convert_to_state(values, opt_filter)+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {self.variance.size()}diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rsindex 551fcfe120..0278ce2c23 100644--- a/datafusion/functions-aggregate/src/variance.rs+++ b/datafusion/functions-aggregate/src/variance.rs@@ -580,6 +580,44 @@ impl GroupsAccumulator for VarianceGroupsAccumulator {])}+ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ assert_eq!(values.len(), 1, "single argument to convert_to_state");+ let values = as_float64_array(&values[0])?;++ let len = values.len();+ let mut counts = Vec::with_capacity(len);+ let mut means = Vec::with_capacity(len);+ let mut m2s = Vec::with_capacity(len);++ for row in 0..len {+ if values.is_valid(row)+ && opt_filter+ .is_none_or(|filter| filter.is_valid(row) && filter.value(row))+ {+ counts.push(1);+ means.push(values.value(row));+ } else {+ counts.push(0);+ means.push(0.0);+ }+ m2s.push(0.0);+ }++ Ok(vec![+ Arc::new(UInt64Array::new(counts.into(), None)),+ Arc::new(Float64Array::new(means.into(), None)),+ Arc::new(Float64Array::new(m2s.into(), None)),+ ])+ }++ fn supports_convert_to_state(&self) -> bool {+ true+ }+fn size(&self) -> usize {self.m2s.capacity() * size_of::<f64>()+ self.means.capacity() * size_of::<f64>()@@ -679,4 +717,89 @@ mod tests {assert_eq!(result.value(0), 1.0);Ok(())}++ #[test]+ fn convert_to_state_roundtrips_through_merge() -> Result<()> {+ let values = Arc::new(Float64Array::from(vec![+ Some(1.0),+ Some(2.0),+ None,+ Some(4.0),+ Some(8.0),+ Some(16.0),+ Some(32.0),+ ])) as ArrayRef;+ let filter = BooleanArray::from(vec![+ Some(true),+ Some(false),+ Some(true),+ None,+ Some(true),+ Some(true),+ Some(true),+ ]);+ let group_indices = vec![0, 1, 0, 1, 0, 0, 0];++ let mut direct = VarianceGroupsAccumulator::new(StatsType::Sample);+ direct.update_batch(+ std::slice::from_ref(&values),+ &group_indices,+ Some(&filter),+ 2,+ )?;+ let direct = direct.evaluate(EmitTo::All)?;++ let converter = VarianceGroupsAccumulator::new(StatsType::Sample);+ let state =+ converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;+ let mut merged = VarianceGroupsAccumulator::new(StatsType::Sample);+ merged.merge_batch(&state, &group_indices, 2)?;+ let merged = merged.evaluate(EmitTo::All)?;++ let direct = direct.as_any().downcast_ref::<Float64Array>().unwrap();+ let merged = merged.as_any().downcast_ref::<Float64Array>().unwrap();+ assert_eq!(direct.len(), merged.len());+ for row in 0..direct.len() {+ assert_eq!(direct.is_null(row), merged.is_null(row));+ if direct.is_valid(row) {+ assert!((direct.value(row) - merged.value(row)).abs() < 1e-12);+ }+ }+ Ok(())+ }++ #[test]+ fn convert_to_state_preserves_empty_and_filtered_rows() -> Result<()> {+ let converter = VarianceGroupsAccumulator::new(StatsType::Sample);+ let empty_values =+ Arc::new(Float64Array::from(Vec::<Option<f64>>::new())) as ArrayRef;+ let state =+ converter.convert_to_state(std::slice::from_ref(&empty_values), None)?;+ for state_array in &state {+ assert_eq!(state_array.len(), 0);+ assert_eq!(state_array.null_count(), 0);+ }++ let values =+ Arc::new(Float64Array::from(vec![Some(1.0), Some(2.0), None])) as ArrayRef;+ let filter = BooleanArray::from(vec![Some(false), None, Some(false)]);+ let group_indices = vec![0, 1, 0];+ let state =+ converter.convert_to_state(std::slice::from_ref(&values), Some(&filter))?;+ for state_array in &state {+ assert_eq!(state_array.len(), values.len());+ assert_eq!(state_array.null_count(), 0);+ }++ let counts = state[0].as_any().downcast_ref::<UInt64Array>().unwrap();+ assert_eq!(counts, &UInt64Array::from(vec![0, 0, 0]));++ let mut merged = VarianceGroupsAccumulator::new(StatsType::Sample);+ merged.merge_batch(&state, &group_indices, 2)?;+ let result = merged.evaluate(EmitTo::All)?;+ let result = result.as_any().downcast_ref::<Float64Array>().unwrap();+ assert_eq!(result.len(), 2);+ assert_eq!(result.null_count(), 2);+ Ok(())+ }}diff --git a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt b/datafusion/sqllogictest/test_files/aggregate_skip_partial.sltindex 195441a119..2ed4c9921f 100644--- a/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt+++ b/datafusion/sqllogictest/test_files/aggregate_skip_partial.slt@@ -636,6 +636,44 @@ FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;4 35 6+# Test variance and stddev with nullable fields and filters+query IRRR+SELECT c2,+ var_samp(c11) FILTER (WHERE c3 > 0),+ stddev_samp(c11) FILTER (WHERE c3 > 0),+ stddev_pop(c11) FILTER (WHERE c3 > 0)+FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;+----+1 0.085786074994 0.29289259976 0.276141791266+2 0.070769227104 0.266024861816 0.24884347232+3 0.087515365779 0.295829960922 0.270054571305+4 0.038697229167 0.196716113134 0.182123731489+5 0.081817232141 0.286037116719 0.23354832782++# Test corr with nullable fields and filters+query IR+SELECT c2,+ corr(c3, c11) FILTER (WHERE c5 > 0)+FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;+----+1 -0.38515658251+2 0.414489249329+3 -0.796447131429+4 -0.938568248748+5 -0.051058146743++# Test count distinct with nullable fields and filters+query II+SELECT c2,+ count(distinct c3) FILTER (WHERE c11 > 0.5)+FROM aggregate_test_100_null GROUP BY c2 ORDER BY c2;+----+1 10+2 6+3 3+4 3+5 6+# Test median with nullable fields and filterquery IRRSELECT c2,