Skip to content

Apache DataFusion SQL Query Engine

Fix aggregate accumulator capacity accounting (#24099)

kosiew8cb7c84d6c3e
diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs
index f0de9d9848..1e72d8ac3d 100644
--- a/datafusion/functions-aggregate/src/count.rs
+++ b/datafusion/functions-aggregate/src/count.rs
@@ -29,6 +29,7 @@ use arrow::{
},
};
use datafusion_common::hash_utils::RandomState;
+use datafusion_common::heap_size::{DFHeapSize, DFHeapSizeCtx};
use datafusion_common::{
HashMap, Result, ScalarValue, downcast_value, exec_err, internal_err, not_impl_err,
stats::Precision, utils::expr::COUNT_STAR_EXPANSION,
@@ -774,7 +775,7 @@ impl GroupsAccumulator for CountGroupsAccumulator {
Ok(vec![state_array])
}
fn size(&self) -> usize {
- self.counts.capacity() * size_of::<usize>()
+ self.counts.heap_size(&mut DFHeapSizeCtx::default())
}
}
@@ -932,6 +933,23 @@ mod tests {
)?)
}
+ #[test]
+ fn count_groups_size_includes_vec_capacity() -> Result<()> {
+ let mut acc = CountGroupsAccumulator::new();
+ let empty_size = acc.size();
+ assert_eq!(empty_size, 0);
+ let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3]));
+ acc.update_batch(&[values], &[0, 1, 2], None, 3)?;
+
+ assert!(acc.counts.capacity() > 0);
+ let allocated_size = acc.counts.heap_size(&mut DFHeapSizeCtx::default());
+ assert_eq!(allocated_size, acc.counts.capacity() * size_of::<i64>());
+ assert_eq!(acc.size(), allocated_size);
+ assert!(acc.size() > empty_size);
+
+ Ok(())
+ }
+
#[test]
fn count_accumulator_nulls() -> Result<()> {
let mut accumulator = CountAccumulator::new();
diff --git a/datafusion/functions-aggregate/src/sum.rs b/datafusion/functions-aggregate/src/sum.rs
index c124c7a1a0..71932c5f0b 100644
--- a/datafusion/functions-aggregate/src/sum.rs
+++ b/datafusion/functions-aggregate/src/sum.rs
@@ -48,7 +48,7 @@ use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_o
use datafusion_functions_aggregate_common::aggregate::sum_distinct::DistinctSumAccumulator;
use datafusion_macros::user_doc;
use datafusion_physical_expr::expressions::{CastExpr, Column};
-use std::mem::size_of_val;
+use std::mem::{size_of, size_of_val};
make_udaf_expr_and_func!(
Sum,
@@ -671,7 +671,8 @@ impl Accumulator for SlidingDistinctSumAccumulator {
}
fn size(&self) -> usize {
- size_of_val(self)
+ // Estimate the owned map buckets; implementation-specific control bytes are excluded.
+ size_of_val(self) + self.counts.capacity() * size_of::<(i64, usize)>()
}
fn state(&mut self) -> Result<Vec<ScalarValue>> {
@@ -722,7 +723,10 @@ mod tests {
array::{Decimal128Array, Int64Array},
buffer::{NullBuffer, ScalarBuffer},
};
- use std::sync::Arc;
+ use std::{
+ mem::{size_of, size_of_val},
+ sync::Arc,
+ };
#[test]
fn sliding_distinct_sum_ignores_null_slots() -> Result<()> {
@@ -750,6 +754,40 @@ mod tests {
Ok(())
}
+ fn expected_sliding_distinct_sum_size(acc: &SlidingDistinctSumAccumulator) -> usize {
+ size_of_val(acc) + acc.counts.capacity() * size_of::<(i64, usize)>()
+ }
+
+ #[test]
+ fn sliding_distinct_sum_size_includes_hash_map_capacity() -> Result<()> {
+ let mut acc = SlidingDistinctSumAccumulator::try_new(&DataType::Int64)?;
+ let empty_size = acc.size();
+ let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3]));
+ acc.update_batch(&[Arc::clone(&values)])?;
+
+ let expected = expected_sliding_distinct_sum_size(&acc);
+ assert!(acc.counts.capacity() > 0);
+ assert_eq!(acc.size(), expected);
+ assert!(acc.size() > empty_size);
+
+ let initial_capacity = acc.counts.capacity();
+ let additional_values: ArrayRef =
+ Arc::new(Int64Array::from_iter(4..4 + initial_capacity as i64 + 1));
+ acc.update_batch(&[Arc::clone(&additional_values)])?;
+
+ let grown_size = expected_sliding_distinct_sum_size(&acc);
+ assert!(acc.counts.capacity() > initial_capacity);
+ assert_eq!(acc.size(), grown_size);
+ assert!(acc.size() > expected);
+
+ acc.retract_batch(&[values])?;
+ acc.retract_batch(&[additional_values])?;
+ assert!(acc.counts.is_empty());
+ assert_eq!(acc.size(), grown_size);
+
+ Ok(())
+ }
+
#[test]
fn sliding_distinct_sum_returns_null_for_all_null_frame() -> Result<()> {
let mut acc = SlidingDistinctSumAccumulator::try_new(&DataType::Int64)?;