Skip to content

Apache DataFusion SQL Query Engine

chore: refactor `VarianceAccumulator`, add tests and benchmark (#23977)

Neil Conway1955d5af75c2
diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml
index 3ac882e9c6..5abea16e2c 100644
--- a/datafusion/functions-aggregate/Cargo.toml
+++ b/datafusion/functions-aggregate/Cargo.toml
@@ -100,5 +100,9 @@ harness = false
name = "sliding_max"
harness = false
+[[bench]]
+name = "variance"
+harness = false
+
[features]
force_hash_collisions = ["datafusion-common/force_hash_collisions"]
diff --git a/datafusion/functions-aggregate/benches/variance.rs b/datafusion/functions-aggregate/benches/variance.rs
new file mode 100644
index 0000000000..ef55bf32b8
--- /dev/null
+++ b/datafusion/functions-aggregate/benches/variance.rs
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::hint::black_box;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Float64Array};
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use datafusion_expr::Accumulator;
+use datafusion_functions_aggregate::variance::VarianceAccumulator;
+use datafusion_functions_aggregate_common::stats::StatsType;
+
+const BATCH_SIZE: usize = 8192;
+
+fn batch_array(null_stride: Option<usize>) -> ArrayRef {
+ let values = (0..BATCH_SIZE)
+ .map(|idx| {
+ if null_stride.is_some_and(|stride| idx % stride == 0) {
+ None
+ } else {
+ Some(idx as f64)
+ }
+ })
+ .collect::<Vec<_>>();
+ Arc::new(Float64Array::from(values)) as ArrayRef
+}
+
+fn update_bench(c: &mut Criterion, name: &str, batch: &ArrayRef) {
+ c.bench_function(name, |b| {
+ b.iter(|| {
+ let mut acc = VarianceAccumulator::try_new(StatsType::Sample).unwrap();
+ acc.update_batch(std::slice::from_ref(batch)).unwrap();
+ black_box(acc.evaluate().unwrap())
+ })
+ });
+}
+
+fn retract_bench(c: &mut Criterion, name: &str, batch: &ArrayRef) {
+ c.bench_function(name, |b| {
+ b.iter_batched(
+ || {
+ let mut acc = VarianceAccumulator::try_new(StatsType::Sample).unwrap();
+ // Accumulate two batches so that retracting one leaves the
+ // accumulator with rows remaining, as in a sliding window.
+ acc.update_batch(std::slice::from_ref(batch)).unwrap();
+ acc.update_batch(std::slice::from_ref(batch)).unwrap();
+ acc
+ },
+ |mut acc| {
+ acc.retract_batch(std::slice::from_ref(batch)).unwrap();
+ black_box(acc.evaluate().unwrap())
+ },
+ BatchSize::SmallInput,
+ )
+ });
+}
+
+fn variance_benchmark(c: &mut Criterion) {
+ let no_nulls = batch_array(None);
+ let with_nulls = batch_array(Some(10));
+
+ update_bench(c, "variance update_batch f64 no_nulls", &no_nulls);
+ update_bench(c, "variance update_batch f64 with_nulls", &with_nulls);
+ retract_bench(c, "variance retract_batch f64 no_nulls", &no_nulls);
+ retract_bench(c, "variance retract_batch f64 with_nulls", &with_nulls);
+}
+
+criterion_group!(benches, variance_benchmark);
+criterion_main!(benches);
diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs
index df652731ff..b8e52f849a 100644
--- a/datafusion/functions-aggregate/src/variance.rs
+++ b/datafusion/functions-aggregate/src/variance.rs
@@ -326,6 +326,23 @@ fn update(count: u64, mean: f64, m2: f64, value: f64) -> (u64, f64, f64) {
(new_count, new_mean, new_m2)
}
+/// Inverse of [`update`]: removes a previously accumulated value. Retracting
+/// from a state with one or zero values resets the state to empty.
+#[inline]
+fn retract(count: u64, mean: f64, m2: f64, value: f64) -> (u64, f64, f64) {
+ if count <= 1 {
+ return (0, 0.0, 0.0);
+ }
+
+ let new_count = count - 1;
+ let delta1 = mean - value;
+ let new_mean = delta1 / new_count as f64 + mean;
+ let delta2 = new_mean - value;
+ let new_m2 = m2 - delta1 * delta2;
+
+ (new_count, new_mean, new_m2)
+}
+
impl Accumulator for VarianceAccumulator {
fn state(&mut self) -> Result<Vec<ScalarValue>> {
Ok(vec![
@@ -348,22 +365,8 @@ impl Accumulator for VarianceAccumulator {
fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
let arr = as_float64_array(&values[0])?;
for value in arr.iter().flatten() {
- if self.count <= 1 {
- self.count = 0;
- self.mean = 0.0;
- self.m2 = 0.0;
- continue;
- }
-
- let new_count = self.count - 1;
- let delta1 = self.mean - value;
- let new_mean = delta1 / new_count as f64 + self.mean;
- let delta2 = new_mean - value;
- let new_m2 = self.m2 - delta1 * delta2;
-
- self.count -= 1;
- self.mean = new_mean;
- self.m2 = new_m2;
+ (self.count, self.mean, self.m2) =
+ retract(self.count, self.mean, self.m2, value)
}
Ok(())
@@ -691,6 +694,75 @@ mod tests {
use super::*;
+ #[test]
+ fn update_batch_ignores_nulls() -> Result<()> {
+ // An array with nulls must accumulate the same values as a dense
+ // array of its non-null values.
+ let dense: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));
+ let sparse: ArrayRef = Arc::new(Float64Array::from(vec![
+ Some(1.0),
+ None,
+ Some(2.0),
+ Some(3.0),
+ None,
+ Some(4.0),
+ ]));
+
+ let mut dense_acc = VarianceAccumulator::try_new(StatsType::Sample)?;
+ dense_acc.update_batch(std::slice::from_ref(&dense))?;
+ let mut sparse_acc = VarianceAccumulator::try_new(StatsType::Sample)?;
+ sparse_acc.update_batch(std::slice::from_ref(&sparse))?;
+
+ // Sample variance of {1, 2, 3, 4} is 5/3 (all steps are exact in f64).
+ assert_eq!(dense_acc.evaluate()?, ScalarValue::Float64(Some(5.0 / 3.0)));
+ assert_eq!(dense_acc.evaluate()?, sparse_acc.evaluate()?);
+ Ok(())
+ }
+
+ #[test]
+ fn retract_batch_ignores_nulls() -> Result<()> {
+ let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));
+ let dense_retract: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));
+ let sparse_retract: ArrayRef =
+ Arc::new(Float64Array::from(vec![Some(1.0), None, Some(2.0)]));
+
+ let mut dense_acc = VarianceAccumulator::try_new(StatsType::Sample)?;
+ dense_acc.update_batch(std::slice::from_ref(&values))?;
+ dense_acc.retract_batch(std::slice::from_ref(&dense_retract))?;
+ let mut sparse_acc = VarianceAccumulator::try_new(StatsType::Sample)?;
+ sparse_acc.update_batch(std::slice::from_ref(&values))?;
+ sparse_acc.retract_batch(std::slice::from_ref(&sparse_retract))?;
+
+ // Sample variance of the remaining {3, 4} is 0.5 (all steps are exact
+ // in f64).
+ assert_eq!(dense_acc.evaluate()?, ScalarValue::Float64(Some(0.5)));
+ assert_eq!(dense_acc.evaluate()?, sparse_acc.evaluate()?);
+ Ok(())
+ }
+
+ #[test]
+ fn retract_batch_resets_when_underflowing() -> Result<()> {
+ // Retracting more values than were accumulated resets to the empty
+ // state, with or without nulls in the retracted batch.
+ let values: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0]));
+ let dense_retract: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]));
+ let sparse_retract: ArrayRef = Arc::new(Float64Array::from(vec![
+ Some(1.0),
+ None,
+ Some(2.0),
+ Some(3.0),
+ ]));
+
+ for retract in [&dense_retract, &sparse_retract] {
+ let mut acc = VarianceAccumulator::try_new(StatsType::Sample)?;
+ acc.update_batch(std::slice::from_ref(&values))?;
+ acc.retract_batch(std::slice::from_ref(retract))?;
+ assert_eq!(acc.get_count(), 0);
+ assert_eq!(acc.evaluate()?, ScalarValue::Float64(None));
+ }
+ Ok(())
+ }
+
#[test]
fn test_groups_accumulator_merge_empty_states() -> Result<()> {
let state_1 = vec![