Apache DataFusion SQL Query Engine
Fix memory size accounting for grouped `median` and `avg` (#23357)
linfeng73d5d7803ba1
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rsindex 06c7694634..278a861de2 100644--- a/datafusion/functions-aggregate/src/average.rs+++ b/datafusion/functions-aggregate/src/average.rs@@ -970,6 +970,13 @@ where}fn size(&self) -> usize {- self.counts.capacity() * size_of::<u64>() + self.sums.capacity() * size_of::<T>()+ // Heap buffers+ self.counts.capacity() * size_of::<u64>()+ + self.sums.capacity() * size_of::<T::Native>()+ // Vec struct overhead (ptr, len, cap) for each field+ + size_of::<Vec<u64>>()+ + size_of::<Vec<T::Native>>()+ // Null tracking buffers+ + self.null_state.size()}}diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rsindex 4a0da10f51..9a6ef3e7e5 100644--- a/datafusion/functions-aggregate/src/median.rs+++ b/datafusion/functions-aggregate/src/median.rs@@ -543,10 +543,11 @@ impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<Tfn size(&self) -> usize {self.group_values.iter()- .map(|values| values.capacity() * size_of::<T>())+ .map(|values| values.capacity() * size_of::<T::Native>()).sum::<usize>()- // account for size of self.grou_values too- + self.group_values.capacity() * size_of::<Vec<T>>()+ // account for size of self.group_values too+ + self.group_values.capacity() * size_of::<Vec<T::Native>>()+ + size_of::<Vec<Vec<T::Native>>>()}}