Apache DataFusion SQL Query Engine
chore: fix `SlidingDistinctCountAccumulator::size()` to include budget for distinct values (#23399)
Oleks V96e8fdf3baae
diff --git a/datafusion/expr-common/src/accumulator.rs b/datafusion/expr-common/src/accumulator.rsindex 59fb6a5952..7e9a4ae525 100644--- a/datafusion/expr-common/src/accumulator.rs+++ b/datafusion/expr-common/src/accumulator.rs@@ -92,6 +92,8 @@ pub trait Accumulator: Send + Sync + Debug + std::any::Any {////// "Allocated" means that for internal containers such as `Vec`,/// the `capacity` should be used not the `len`.+ ///+ /// May be expensive; check the implementation before calling on hot paths.fn size(&self) -> usize;/// Returns the intermediate state of the accumulator, consuming thediff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rsindex b021674cbe..13b2f853c9 100644--- a/datafusion/expr-common/src/groups_accumulator.rs+++ b/datafusion/expr-common/src/groups_accumulator.rs@@ -248,6 +248,8 @@ pub trait GroupsAccumulator: Send + std::any::Any {////// This function is called once per batch, so it should be `O(n)` to/// compute, not `O(num_groups)`+ ///+ /// May be expensive; check the implementation before calling on hot paths.fn size(&self) -> usize;}diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rsindex f0ce8c82a1..983828ea90 100644--- a/datafusion/functions-aggregate/src/count.rs+++ b/datafusion/functions-aggregate/src/count.rs@@ -555,7 +555,17 @@ impl Accumulator for SlidingDistinctCountAccumulator {}fn size(&self) -> usize {+ // Mirrors `DistinctCountAccumulator::full_size`: self + HashMap+ // bucket array + per-key inner heap + DataType inner heap.size_of_val(self)+ + (size_of::<ScalarValue>() + size_of::<usize>()) * self.counts.capacity()+ + self+ .counts+ .keys()+ .map(|k| k.size() - size_of_val(k))+ .sum::<usize>()+ + self.data_type.size()+ - size_of_val(&self.data_type)}}diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rsindex ee253e5d7a..1101d53531 100644--- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs@@ -99,7 +99,9 @@ pub trait GroupValues: Send {/// assigned.fn intern(&mut self, cols: &[ArrayRef], groups: &mut Vec<usize>) -> Result<()>;- /// Returns the number of bytes of memory used by this [`GroupValues`]+ /// Returns the number of bytes of memory used by this [`GroupValues`].+ ///+ /// May be expensive; check the implementation before calling on hot paths.fn size(&self) -> usize;/// Returns true if this [`GroupValues`] is empty