Skip to content

Apache DataFusion SQL Query Engine

refactor: Simplify heap size estimation for arrays (#22954)

Michael Kleen3c6734ebf014
diff --git a/datafusion/common/src/heap_size.rs b/datafusion/common/src/heap_size.rs
index 802f9d3883..869946d824 100644
--- a/datafusion/common/src/heap_size.rs
+++ b/datafusion/common/src/heap_size.rs
@@ -326,47 +326,6 @@ impl DFHeapSize for Fields {
}
}
-impl DFHeapSize for StructArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
-impl DFHeapSize for LargeListArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
-impl DFHeapSize for LargeListViewArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
-impl DFHeapSize for ListArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
-impl DFHeapSize for ListViewArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
-impl DFHeapSize for FixedSizeListArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-impl DFHeapSize for MapArray {
- fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
- self.get_array_memory_size()
- }
-}
-
impl<T: DFHeapSize> DFHeapSize for Box<T> {
fn heap_size(&self, ctx: &mut DFHeapSizeCtx) -> usize {
size_of::<T>() + self.as_ref().heap_size(ctx)
@@ -469,6 +428,29 @@ impl_zero_heap_size!(
DateTime<Utc>,
);
+/// Implement [`DFHeapSize`] for Arrow arrays types.
+macro_rules! impl_array_heap_size {
+ ($($t:ty),+ $(,)?) => {
+ $(
+ impl DFHeapSize for $t {
+ fn heap_size(&self, _: &mut DFHeapSizeCtx) -> usize {
+ self.get_array_memory_size()
+ }
+ }
+ )+
+ };
+}
+
+impl_array_heap_size!(
+ StructArray,
+ LargeListArray,
+ LargeListViewArray,
+ ListArray,
+ ListViewArray,
+ FixedSizeListArray,
+ MapArray,
+);
+
#[cfg(test)]
mod tests {
use super::*;
@@ -696,4 +678,61 @@ mod tests {
let field = Field::new("temperature", DataType::Float64, true);
assert!(size(&field) > 0);
}
+
+ #[test]
+ fn test_list_array() {
+ use arrow::array::types::Int32Type;
+
+ let array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
+ Some(vec![Some(1), Some(2), Some(3)]),
+ Some(vec![Some(4)]),
+ ]);
+ assert_eq!(size(&array), array.get_array_memory_size());
+ assert!(size(&array) > 0);
+
+ let large =
+ LargeListArray::from_iter_primitive::<Int32Type, _, _>(vec![Some(vec![
+ Some(1),
+ Some(2),
+ ])]);
+ assert_eq!(size(&large), large.get_array_memory_size());
+ assert!(size(&large) > 0);
+ }
+
+ #[test]
+ fn test_struct_array() {
+ use arrow::array::Int32Array;
+
+ let array = StructArray::from(vec![(
+ Arc::new(Field::new("a", DataType::Int32, true)),
+ Arc::new(Int32Array::from(vec![1, 2, 3])) as _,
+ )]);
+ assert_eq!(size(&array), array.get_array_memory_size());
+ assert!(size(&array) > 0);
+ }
+
+ #[test]
+ fn test_fixed_size_list_array() {
+ use arrow::array::Int32Array;
+
+ let values = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+ let field = Arc::new(Field::new("item", DataType::Int32, true));
+ let array = FixedSizeListArray::new(field, 2, values, None);
+ assert_eq!(size(&array), array.get_array_memory_size());
+ assert!(size(&array) > 0);
+ }
+
+ #[test]
+ fn test_map_array() {
+ use arrow::array::{Int32Builder, MapBuilder, StringBuilder};
+
+ let mut builder =
+ MapBuilder::new(None, StringBuilder::new(), Int32Builder::new());
+ builder.keys().append_value("key");
+ builder.values().append_value(1);
+ builder.append(true).unwrap();
+ let array = builder.finish();
+ assert_eq!(size(&array), array.get_array_memory_size());
+ assert!(size(&array) > 0);
+ }
}