Apache DataFusion SQL Query Engine
refactor: move metrics module to `datafusion-common` crate (#19247)
Yongting You7c05b208e115
diff --git a/Cargo.lock b/Cargo.lockindex 77df826392..5fc993463b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2559,7 +2559,6 @@ dependencies = ["arrow-ord","arrow-schema","async-trait",- "chrono","criterion","datafusion-common","datafusion-common-runtime",diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rsindex 028273ee98..78ec434d2b 100644--- a/datafusion/common/src/utils/memory.rs+++ b/datafusion/common/src/utils/memory.rs@@ -17,9 +17,11 @@//! This module provides a function to estimate the memory size of a HashTable prior to allocation-use crate::Result;use crate::error::_exec_datafusion_err;-use std::mem::size_of;+use crate::{HashSet, Result};+use arrow::array::ArrayData;+use arrow::record_batch::RecordBatch;+use std::{mem::size_of, ptr::NonNull};/// Estimates the memory size required for a hash table prior to allocation.///@@ -99,6 +101,74 @@ pub fn estimate_memory_size<T>(num_elements: usize, fixed_size: usize) -> Result})}+/// Calculate total used memory of this batch.+///+/// This function is used to estimate the physical memory usage of the `RecordBatch`.+/// It only counts the memory of large data `Buffer`s, and ignores metadata like+/// types and pointers.+/// The implementation will add up all unique `Buffer`'s memory+/// size, due to:+/// - The data pointer inside `Buffer` are memory regions returned by global memory+/// allocator, those regions can't have overlap.+/// - The actual used range of `ArrayRef`s inside `RecordBatch` can have overlap+/// or reuse the same `Buffer`. For example: taking a slice from `Array`.+///+/// Example:+/// For a `RecordBatch` with two columns: `col1` and `col2`, two columns are pointing+/// to a sub-region of the same buffer.+///+/// {xxxxxxxxxxxxxxxxxxx} <--- buffer+/// ^ ^ ^ ^+/// | | | |+/// col1->{ } | |+/// col2--------->{ }+///+/// In the above case, `get_record_batch_memory_size` will return the size of+/// the buffer, instead of the sum of `col1` and `col2`'s actual memory size.+///+/// Note: Current `RecordBatch`.get_array_memory_size()` will double count the+/// buffer memory size if multiple arrays within the batch are sharing the same+/// `Buffer`. This method provides temporary fix until the issue is resolved:+/// <https://github.com/apache/arrow-rs/issues/6439>+pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize {+ // Store pointers to `Buffer`'s start memory address (instead of actual+ // used data region's pointer represented by current `Array`)+ let mut counted_buffers: HashSet<NonNull<u8>> = HashSet::new();+ let mut total_size = 0;++ for array in batch.columns() {+ let array_data = array.to_data();+ count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size);+ }++ total_size+}++/// Count the memory usage of `array_data` and its children recursively.+fn count_array_data_memory_size(+ array_data: &ArrayData,+ counted_buffers: &mut HashSet<NonNull<u8>>,+ total_size: &mut usize,+) {+ // Count memory usage for `array_data`+ for buffer in array_data.buffers() {+ if counted_buffers.insert(buffer.data_ptr()) {+ *total_size += buffer.capacity();+ } // Otherwise the buffer's memory is already counted+ }++ if let Some(null_buffer) = array_data.nulls()+ && counted_buffers.insert(null_buffer.inner().inner().data_ptr())+ {+ *total_size += null_buffer.inner().inner().capacity();+ }++ // Count all children `ArrayData` recursively+ for child in array_data.child_data() {+ count_array_data_memory_size(child, counted_buffers, total_size);+ }+}+#[cfg(test)]mod tests {use std::{collections::HashSet, mem::size_of};@@ -132,3 +202,129 @@ mod tests {assert!(estimated.is_err());}}++#[cfg(test)]+mod record_batch_tests {+ use super::*;+ use arrow::array::{Float64Array, Int32Array, ListArray};+ use arrow::datatypes::{DataType, Field, Int32Type, Schema};+ use std::sync::Arc;++ #[test]+ fn test_get_record_batch_memory_size() {+ let schema = Arc::new(Schema::new(vec![+ Field::new("ints", DataType::Int32, true),+ Field::new("float64", DataType::Float64, false),+ ]));++ let int_array =+ Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4), Some(5)]);+ let float64_array = Float64Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0]);++ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(int_array), Arc::new(float64_array)],+ )+ .unwrap();++ let size = get_record_batch_memory_size(&batch);+ assert_eq!(size, 60);+ }++ #[test]+ fn test_get_record_batch_memory_size_with_null() {+ let schema = Arc::new(Schema::new(vec![+ Field::new("ints", DataType::Int32, true),+ Field::new("float64", DataType::Float64, false),+ ]));++ let int_array = Int32Array::from(vec![None, Some(2), Some(3)]);+ let float64_array = Float64Array::from(vec![1.0, 2.0, 3.0]);++ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(int_array), Arc::new(float64_array)],+ )+ .unwrap();++ let size = get_record_batch_memory_size(&batch);+ assert_eq!(size, 100);+ }++ #[test]+ fn test_get_record_batch_memory_size_empty() {+ let schema = Arc::new(Schema::new(vec![Field::new(+ "ints",+ DataType::Int32,+ false,+ )]));++ let int_array: Int32Array = Int32Array::from(vec![] as Vec<i32>);+ let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array)]).unwrap();++ let size = get_record_batch_memory_size(&batch);+ assert_eq!(size, 0, "Empty batch should have 0 memory size");+ }++ #[test]+ fn test_get_record_batch_memory_size_shared_buffer() {+ let original = Int32Array::from(vec![1, 2, 3, 4, 5]);+ let slice1 = original.slice(0, 3);+ let slice2 = original.slice(2, 3);++ let schema_origin = Arc::new(Schema::new(vec![Field::new(+ "origin_col",+ DataType::Int32,+ false,+ )]));+ let batch_origin =+ RecordBatch::try_new(schema_origin, vec![Arc::new(original)]).unwrap();++ let schema = Arc::new(Schema::new(vec![+ Field::new("slice1", DataType::Int32, false),+ Field::new("slice2", DataType::Int32, false),+ ]));++ let batch_sliced =+ RecordBatch::try_new(schema, vec![Arc::new(slice1), Arc::new(slice2)])+ .unwrap();++ let size_origin = get_record_batch_memory_size(&batch_origin);+ let size_sliced = get_record_batch_memory_size(&batch_sliced);++ assert_eq!(size_origin, size_sliced);+ }++ #[test]+ fn test_get_record_batch_memory_size_nested_array() {+ let schema = Arc::new(Schema::new(vec![+ Field::new(+ "nested_int",+ DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),+ false,+ ),+ Field::new(+ "nested_int2",+ DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),+ false,+ ),+ ]));++ let int_list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(1), Some(2), Some(3)]),+ ]);++ let int_list_array2 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![+ Some(vec![Some(4), Some(5), Some(6)]),+ ]);++ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(int_list_array), Arc::new(int_list_array2)],+ )+ .unwrap();++ let size = get_record_batch_memory_size(&batch);+ assert_eq!(size, 8208);+ }+}diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.tomlindex 2998e8ab1b..444a060d8b 100644--- a/datafusion/execution/Cargo.toml+++ b/datafusion/execution/Cargo.toml@@ -51,6 +51,7 @@ sql = [][dependencies]arrow = { workspace = true }async-trait = { workspace = true }+chrono = { workspace = true }dashmap = { workspace = true }datafusion-common = { workspace = true, default-features = false }datafusion-expr = { workspace = true, default-features = false }@@ -64,5 +65,4 @@ tempfile = { workspace = true }url = { workspace = true }[dev-dependencies]-chrono = { workspace = true }insta = { workspace = true }diff --git a/datafusion/execution/src/lib.rs b/datafusion/execution/src/lib.rsindex aced2f46d7..2ea8182823 100644--- a/datafusion/execution/src/lib.rs+++ b/datafusion/execution/src/lib.rs@@ -32,6 +32,7 @@ pub mod cache;pub mod config;pub mod disk_manager;pub mod memory_pool;+pub mod metrics;pub mod object_store;#[cfg(feature = "parquet_encryption")]pub mod parquet_encryption;diff --git a/datafusion/physical-plan/src/metrics/baseline.rs b/datafusion/execution/src/metrics/baseline.rssimilarity index 97%rename from datafusion/physical-plan/src/metrics/baseline.rsrename to datafusion/execution/src/metrics/baseline.rsindex 8dc2f30d9f..91eff2d209 100644--- a/datafusion/physical-plan/src/metrics/baseline.rs+++ b/datafusion/execution/src/metrics/baseline.rs@@ -20,18 +20,16 @@use std::task::Poll;use arrow::record_batch::RecordBatch;--use crate::spill::get_record_batch_memory_size;+use datafusion_common::{Result, utils::memory::get_record_batch_memory_size};use super::{Count, ExecutionPlanMetricsSet, MetricBuilder, Time, Timestamp};-use datafusion_common::Result;/// Helper for creating and tracking common "baseline" metrics for/// each operator////// Example:/// ```-/// use datafusion_physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};+/// use datafusion_execution::metrics::{BaselineMetrics, ExecutionPlanMetricsSet};/// let metrics = ExecutionPlanMetricsSet::new();////// let partition = 2;@@ -203,7 +201,7 @@ impl SpillMetrics {}}-/// Metrics for tracking [`crate::stream::BatchSplitStream`] activity+/// Metrics for tracking batch splitting activity#[derive(Debug, Clone)]pub struct SplitMetrics {/// Number of times an input [`RecordBatch`] was splitdiff --git a/datafusion/physical-plan/src/metrics/builder.rs b/datafusion/execution/src/metrics/builder.rssimilarity index 99%rename from datafusion/physical-plan/src/metrics/builder.rsrename to datafusion/execution/src/metrics/builder.rsindex 42ff427a53..63c18d7253 100644--- a/datafusion/physical-plan/src/metrics/builder.rs+++ b/datafusion/execution/src/metrics/builder.rs@@ -34,7 +34,7 @@ use super::{/// case of constant strings////// ```rust-/// use datafusion_physical_plan::metrics::*;+/// use datafusion_execution::metrics::*;////// let metrics = ExecutionPlanMetricsSet::new();/// let partition = 1;diff --git a/datafusion/physical-plan/src/metrics/custom.rs b/datafusion/execution/src/metrics/custom.rssimilarity index 98%rename from datafusion/physical-plan/src/metrics/custom.rsrename to datafusion/execution/src/metrics/custom.rsindex 4421db94dc..8062bbbf38 100644--- a/datafusion/physical-plan/src/metrics/custom.rs+++ b/datafusion/execution/src/metrics/custom.rs@@ -44,7 +44,7 @@ use std::{any::Any, fmt::Debug, fmt::Display, sync::Arc};/// # use std::any::Any;/// # use std::sync::atomic::{AtomicUsize, Ordering};///-/// # use datafusion_physical_plan::metrics::CustomMetricValue;+/// # use datafusion_execution::metrics::CustomMetricValue;////// #[derive(Debug, Default)]/// struct MyCounter {diff --git a/datafusion/physical-plan/src/metrics/mod.rs b/datafusion/execution/src/metrics/mod.rssimilarity index 97%rename from datafusion/physical-plan/src/metrics/mod.rsrename to datafusion/execution/src/metrics/mod.rsindex 2e4a9a1003..e76ba1de38 100644--- a/datafusion/physical-plan/src/metrics/mod.rs+++ b/datafusion/execution/src/metrics/mod.rs@@ -22,6 +22,7 @@ mod builder;mod custom;mod value;+use datafusion_common::HashMap;use parking_lot::Mutex;use std::{borrow::Cow,@@ -29,9 +30,8 @@ use std::{sync::Arc,};-use datafusion_common::HashMap;-// public exports+pub use baseline::{BaselineMetrics, RecordOutput, SpillMetrics, SplitMetrics};pub use builder::MetricBuilder;pub use custom::CustomMetricValue;@@ -40,15 +40,14 @@ pub use value::{ScopedTimerGuard, Time, Timestamp,};-/// Something that tracks a value of interest (metric) of a DataFusion-/// [`ExecutionPlan`] execution.+/// Something that tracks a value of interest (metric) during execution.////// Typically [`Metric`]s are not created directly, but instead/// are created using [`MetricBuilder`] or methods on/// [`ExecutionPlanMetricsSet`].////// ```-/// use datafusion_physical_plan::metrics::*;+/// use datafusion_execution::metrics::*;////// let metrics = ExecutionPlanMetricsSet::new();/// assert!(metrics.clone_inner().output_rows().is_none());@@ -66,8 +65,6 @@ pub use value::{/// // As well as from the metrics set/// assert_eq!(metrics.clone_inner().output_rows(), Some(13));/// ```-///-/// [`ExecutionPlan`]: super::ExecutionPlan#[derive(Debug)]pub struct Metric {@@ -204,9 +201,7 @@ impl Metric {}}-/// A snapshot of the metrics for a particular ([`ExecutionPlan`]).-///-/// [`ExecutionPlan`]: super::ExecutionPlan+/// A snapshot of the metrics for a particular execution plan.#[derive(Default, Debug, Clone)]pub struct MetricsSet {metrics: Vec<Arc<Metric>>,@@ -401,17 +396,14 @@ impl Display for MetricsSet {}}-/// A set of [`Metric`]s for an individual "operator" (e.g. `&dyn-/// ExecutionPlan`).+/// A set of [`Metric`]s for an individual operator.///-/// This structure is intended as a convenience for [`ExecutionPlan`]+/// This structure is intended as a convenience for execution plan/// implementations so they can generate different streams for multiple/// partitions but easily report them together.////// Each `clone()` of this structure will add metrics to the same/// underlying metrics set-///-/// [`ExecutionPlan`]: super::ExecutionPlan#[derive(Default, Debug, Clone)]pub struct ExecutionPlanMetricsSet {inner: Arc<Mutex<MetricsSet>>,diff --git a/datafusion/physical-plan/src/metrics/value.rs b/datafusion/execution/src/metrics/value.rssimilarity index 99%rename from datafusion/physical-plan/src/metrics/value.rsrename to datafusion/execution/src/metrics/value.rsindex 8016380be9..9a14b804a2 100644--- a/datafusion/physical-plan/src/metrics/value.rs+++ b/datafusion/execution/src/metrics/value.rs@@ -19,9 +19,8 @@use super::CustomMetricValue;use chrono::{DateTime, Utc};-use datafusion_common::instant::Instant;use datafusion_common::{- human_readable_count, human_readable_duration, human_readable_size,+ human_readable_count, human_readable_duration, human_readable_size, instant::Instant,};use parking_lot::Mutex;use std::{diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 4908510a20..398dc2e793 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -52,7 +52,6 @@ arrow = { workspace = true }arrow-ord = { workspace = true }arrow-schema = { workspace = true }async-trait = { workspace = true }-chrono = { workspace = true }datafusion-common = { workspace = true }datafusion-common-runtime = { workspace = true, default-features = true }datafusion-execution = { workspace = true }diff --git a/datafusion/physical-plan/src/metrics.rs b/datafusion/physical-plan/src/metrics.rsnew file mode 100644index 0000000000..267a3e995d--- /dev/null+++ b/datafusion/physical-plan/src/metrics.rs@@ -0,0 +1,21 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Metrics live in `datafusion-execution`; this module re-exports them to keep+//! the public APIs stable.++pub use datafusion_execution::metrics::*;diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 1e7fa54a1b..78dea99ac8 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -21,6 +21,8 @@ pub(crate) mod in_progress_spill_file;pub(crate) mod spill_manager;pub mod spill_pool;+// Moved for refactor, re-export to keep the public API stable+pub use datafusion_common::utils::memory::get_record_batch_memory_size;// Re-export SpillManager for doctests only (hidden from public docs)#[doc(hidden)]pub use spill_manager::SpillManager;@@ -29,11 +31,10 @@ use std::fs::File;use std::io::BufReader;use std::path::{Path, PathBuf};use std::pin::Pin;-use std::ptr::NonNull;use std::sync::Arc;use std::task::{Context, Poll};-use arrow::array::{ArrayData, BufferSpec, layout};+use arrow::array::{BufferSpec, layout};use arrow::datatypes::{Schema, SchemaRef};use arrow::ipc::{MetadataVersion,@@ -43,7 +44,7 @@ use arrow::ipc::{use arrow::record_batch::RecordBatch;use datafusion_common::config::SpillCompression;-use datafusion_common::{DataFusionError, HashSet, Result, exec_datafusion_err};+use datafusion_common::{DataFusionError, Result, exec_datafusion_err};use datafusion_common_runtime::SpawnedTask;use datafusion_execution::RecordBatchStream;use datafusion_execution::disk_manager::RefCountedTempFile;@@ -250,74 +251,6 @@ pub fn spill_record_batch_by_size(Ok(())}-/// Calculate total used memory of this batch.-///-/// This function is used to estimate the physical memory usage of the `RecordBatch`.-/// It only counts the memory of large data `Buffer`s, and ignores metadata like-/// types and pointers.-/// The implementation will add up all unique `Buffer`'s memory-/// size, due to:-/// - The data pointer inside `Buffer` are memory regions returned by global memory-/// allocator, those regions can't have overlap.-/// - The actual used range of `ArrayRef`s inside `RecordBatch` can have overlap-/// or reuse the same `Buffer`. For example: taking a slice from `Array`.-///-/// Example:-/// For a `RecordBatch` with two columns: `col1` and `col2`, two columns are pointing-/// to a sub-region of the same buffer.-///-/// {xxxxxxxxxxxxxxxxxxx} <--- buffer-/// ^ ^ ^ ^-/// | | | |-/// col1->{ } | |-/// col2--------->{ }-///-/// In the above case, `get_record_batch_memory_size` will return the size of-/// the buffer, instead of the sum of `col1` and `col2`'s actual memory size.-///-/// Note: Current `RecordBatch`.get_array_memory_size()` will double count the-/// buffer memory size if multiple arrays within the batch are sharing the same-/// `Buffer`. This method provides temporary fix until the issue is resolved:-/// <https://github.com/apache/arrow-rs/issues/6439>-pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize {- // Store pointers to `Buffer`'s start memory address (instead of actual- // used data region's pointer represented by current `Array`)- let mut counted_buffers: HashSet<NonNull<u8>> = HashSet::new();- let mut total_size = 0;-- for array in batch.columns() {- let array_data = array.to_data();- count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size);- }-- total_size-}--/// Count the memory usage of `array_data` and its children recursively.-fn count_array_data_memory_size(- array_data: &ArrayData,- counted_buffers: &mut HashSet<NonNull<u8>>,- total_size: &mut usize,-) {- // Count memory usage for `array_data`- for buffer in array_data.buffers() {- if counted_buffers.insert(buffer.data_ptr()) {- *total_size += buffer.capacity();- } // Otherwise the buffer's memory is already counted- }-- if let Some(null_buffer) = array_data.nulls()- && counted_buffers.insert(null_buffer.inner().inner().data_ptr())- {- *total_size += null_buffer.inner().inner().capacity();- }-- // Count all children `ArrayData` recursively- for child in array_data.child_data() {- count_array_data_memory_size(child, counted_buffers, total_size);- }-}-/// Write in Arrow IPC Stream format to a file.////// Stream format is used for spill because it supports dictionary replacement, and the random@@ -415,9 +348,9 @@ mod tests {use crate::metrics::SpillMetrics;use crate::spill::spill_manager::SpillManager;use crate::test::build_table_i32;- use arrow::array::{ArrayRef, Float64Array, Int32Array, ListArray, StringArray};+ use arrow::array::{ArrayRef, Int32Array, StringArray};use arrow::compute::cast;- use arrow::datatypes::{DataType, Field, Int32Type, Schema};+ use arrow::datatypes::{DataType, Field, Schema};use arrow::record_batch::RecordBatch;use datafusion_common::Result;use datafusion_execution::runtime_env::RuntimeEnv;@@ -665,133 +598,6 @@ mod tests {Ok(())}- #[test]- fn test_get_record_batch_memory_size() {- // Create a simple record batch with two columns- let schema = Arc::new(Schema::new(vec![- Field::new("ints", DataType::Int32, true),- Field::new("float64", DataType::Float64, false),- ]));-- let int_array =- Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4), Some(5)]);- let float64_array = Float64Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0]);-- let batch = RecordBatch::try_new(- schema,- vec![Arc::new(int_array), Arc::new(float64_array)],- )- .unwrap();-- let size = get_record_batch_memory_size(&batch);- assert_eq!(size, 60);- }-- #[test]- fn test_get_record_batch_memory_size_with_null() {- // Create a simple record batch with two columns- let schema = Arc::new(Schema::new(vec![- Field::new("ints", DataType::Int32, true),- Field::new("float64", DataType::Float64, false),- ]));-- let int_array = Int32Array::from(vec![None, Some(2), Some(3)]);- let float64_array = Float64Array::from(vec![1.0, 2.0, 3.0]);-- let batch = RecordBatch::try_new(- schema,- vec![Arc::new(int_array), Arc::new(float64_array)],- )- .unwrap();-- let size = get_record_batch_memory_size(&batch);- assert_eq!(size, 100);- }-- #[test]- fn test_get_record_batch_memory_size_empty() {- // Test with empty record batch- let schema = Arc::new(Schema::new(vec![Field::new(- "ints",- DataType::Int32,- false,- )]));-- let int_array: Int32Array = Int32Array::from(vec![] as Vec<i32>);- let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array)]).unwrap();-- let size = get_record_batch_memory_size(&batch);- assert_eq!(size, 0, "Empty batch should have 0 memory size");- }-- #[test]- fn test_get_record_batch_memory_size_shared_buffer() {- // Test with slices that share the same underlying buffer- let original = Int32Array::from(vec![1, 2, 3, 4, 5]);- let slice1 = original.slice(0, 3);- let slice2 = original.slice(2, 3);-- // `RecordBatch` with `original` array- // ----- let schema_origin = Arc::new(Schema::new(vec![Field::new(- "origin_col",- DataType::Int32,- false,- )]));- let batch_origin =- RecordBatch::try_new(schema_origin, vec![Arc::new(original)]).unwrap();-- // `RecordBatch` with all columns are reference to `original` array- // ----- let schema = Arc::new(Schema::new(vec![- Field::new("slice1", DataType::Int32, false),- Field::new("slice2", DataType::Int32, false),- ]));-- let batch_sliced =- RecordBatch::try_new(schema, vec![Arc::new(slice1), Arc::new(slice2)])- .unwrap();-- // Two sizes should all be only counting the buffer in `original` array- let size_origin = get_record_batch_memory_size(&batch_origin);- let size_sliced = get_record_batch_memory_size(&batch_sliced);-- assert_eq!(size_origin, size_sliced);- }-- #[test]- fn test_get_record_batch_memory_size_nested_array() {- let schema = Arc::new(Schema::new(vec![- Field::new(- "nested_int",- DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),- false,- ),- Field::new(- "nested_int2",- DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))),- false,- ),- ]));-- let int_list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![- Some(vec![Some(1), Some(2), Some(3)]),- ]);-- let int_list_array2 = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![- Some(vec![Some(4), Some(5), Some(6)]),- ]);-- let batch = RecordBatch::try_new(- schema,- vec![Arc::new(int_list_array), Arc::new(int_list_array2)],- )- .unwrap();-- let size = get_record_batch_memory_size(&batch);- assert_eq!(size, 8208);- }-// ==== Spill manager tests ====#[test]