Skip to content

Apache DataFusion SQL Query Engine

Fix memory size accounting for grouped `median` and `avg` (#23357)

linfeng73d5d7803ba1
diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs
index 06c7694634..278a861de2 100644
--- a/datafusion/functions-aggregate/src/average.rs
+++ b/datafusion/functions-aggregate/src/average.rs
@@ -970,6 +970,13 @@ where
}
fn size(&self) -> usize {
- self.counts.capacity() * size_of::<u64>() + self.sums.capacity() * size_of::<T>()
+ // Heap buffers
+ self.counts.capacity() * size_of::<u64>()
+ + self.sums.capacity() * size_of::<T::Native>()
+ // Vec struct overhead (ptr, len, cap) for each field
+ + size_of::<Vec<u64>>()
+ + size_of::<Vec<T::Native>>()
+ // Null tracking buffers
+ + self.null_state.size()
}
}
diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs
index 4a0da10f51..9a6ef3e7e5 100644
--- a/datafusion/functions-aggregate/src/median.rs
+++ b/datafusion/functions-aggregate/src/median.rs
@@ -543,10 +543,11 @@ impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T
fn size(&self) -> usize {
self.group_values
.iter()
- .map(|values| values.capacity() * size_of::<T>())
+ .map(|values| values.capacity() * size_of::<T::Native>())
.sum::<usize>()
- // account for size of self.grou_values too
- + self.group_values.capacity() * size_of::<Vec<T>>()
+ // account for size of self.group_values too
+ + self.group_values.capacity() * size_of::<Vec<T::Native>>()
+ + size_of::<Vec<Vec<T::Native>>>()
}
}