Apache DataFusion SQL Query Engine
Fix massive spill files for StringView/BinaryView columns II (#21633)
Adrian Garcia Badaracco4b8c1d9d6b4e
diff --git a/Cargo.lock b/Cargo.lockindex bbec97ed7f..512fbd3c67 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2474,6 +2474,7 @@ name = "datafusion-physical-plan"version = "53.0.0"dependencies = ["arrow",+ "arrow-data","arrow-ord","arrow-schema","async-trait",diff --git a/Cargo.toml b/Cargo.tomlindex 7e75bb59b6..8d90a11858 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -100,6 +100,7 @@ arrow-avro = { version = "58.1.0", default-features = false, features = ["xz",] }arrow-buffer = { version = "58.1.0", default-features = false }+arrow-data = { version = "58.1.0", default-features = false }arrow-flight = { version = "58.1.0", features = ["flight-sql-experimental",] }diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 7acb21b8f3..374fc275a0 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -48,6 +48,7 @@ name = "datafusion_physical_plan"[dependencies]arrow = { workspace = true }+arrow-data = { workspace = true }arrow-ord = { workspace = true }arrow-schema = { workspace = true }async-trait = { workspace = true }@@ -72,6 +73,7 @@ pin-project-lite = "^0.2.7"tokio = { workspace = true }[dev-dependencies]+arrow-data = { workspace = true }criterion = { workspace = true, features = ["async_futures"] }datafusion-functions-aggregate = { workspace = true }datafusion-functions-window = { workspace = true }diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 9afe92ba0a..6c02af8dec 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -55,7 +55,7 @@ use crate::{Statistics,};-use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray};+use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};use arrow::datatypes::SchemaRef;use datafusion_common::config::SpillCompression;@@ -420,8 +420,6 @@ impl ExternalSorter {Some((self.spill_manager.create_in_progress_file("Sorting")?, 0));}- Self::organize_stringview_arrays(globally_sorted_batches)?;-debug!("Spilling sort data of ExternalSorter to disk whilst inserting");let batches_to_spill = std::mem::take(globally_sorted_batches);@@ -433,10 +431,9 @@ impl ExternalSorter {})?;for batch in batches_to_spill {- in_progress_file.append_batch(&batch)?;+ let gc_sliced_size = in_progress_file.append_batch(&batch)?;- *max_record_batch_size =- (*max_record_batch_size).max(batch.get_sliced_size()?);+ *max_record_batch_size = (*max_record_batch_size).max(gc_sliced_size);}assert_or_internal_err!(@@ -465,71 +462,6 @@ impl ExternalSorter {Ok(())}- /// Reconstruct `globally_sorted_batches` to organize the payload buffers of each- /// `StringViewArray` in sequential order by calling `gc()` on them.- ///- /// Note this is a workaround until <https://github.com/apache/arrow-rs/issues/7185> is- /// available- ///- /// # Rationale- /// After (merge-based) sorting, all batches will be sorted into a single run,- /// but physically this sorted run is chunked into many small batches. For- /// `StringViewArray`s inside each sorted run, their inner buffers are not- /// re-constructed by default, leading to non-sequential payload locations- /// (permutated by `interleave()` Arrow kernel). A single payload buffer might- /// be shared by multiple `RecordBatch`es.- /// When writing each batch to disk, the writer has to write all referenced buffers,- /// because they have to be read back one by one to reduce memory usage. This- /// causes extra disk reads and writes, and potentially execution failure.- ///- /// # Example- /// Before sorting:- /// batch1 -> buffer1- /// batch2 -> buffer2- ///- /// sorted_batch1 -> buffer1- /// -> buffer2- /// sorted_batch2 -> buffer1- /// -> buffer2- ///- /// Then when spilling each batch, the writer has to write all referenced buffers- /// repeatedly.- fn organize_stringview_arrays(- globally_sorted_batches: &mut Vec<RecordBatch>,- ) -> Result<()> {- let mut organized_batches = Vec::with_capacity(globally_sorted_batches.len());-- for batch in globally_sorted_batches.drain(..) {- let mut new_columns: Vec<Arc<dyn Array>> =- Vec::with_capacity(batch.num_columns());-- let mut arr_mutated = false;- for array in batch.columns() {- if let Some(string_view_array) =- array.as_any().downcast_ref::<StringViewArray>()- {- let new_array = string_view_array.gc();- new_columns.push(Arc::new(new_array));- arr_mutated = true;- } else {- new_columns.push(Arc::clone(array));- }- }-- let organized_batch = if arr_mutated {- RecordBatch::try_new(batch.schema(), new_columns)?- } else {- batch- };-- organized_batches.push(organized_batch);- }-- *globally_sorted_batches = organized_batches;-- Ok(())- }-/// Sorts the in-memory batches and merges them into a single sorted run, then writes/// the result to spill files.async fn sort_and_spill_in_mem_batches(&mut self) -> Result<()> {diff --git a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs b/datafusion/physical-plan/src/spill/in_progress_spill_file.rsindex 9084ea449d..e0548bd5bf 100644--- a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs+++ b/datafusion/physical-plan/src/spill/in_progress_spill_file.rs@@ -24,7 +24,10 @@ use arrow::array::RecordBatch;use datafusion_common::exec_datafusion_err;use datafusion_execution::disk_manager::RefCountedTempFile;-use super::{IPCStreamWriter, spill_manager::SpillManager};+use super::{+ IPCStreamWriter, gc_view_arrays,+ spill_manager::{GetSlicedSize, SpillManager},+};/// Represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`./// Caller is able to use this struct to incrementally append in-memory batches to@@ -51,16 +54,25 @@ impl InProgressSpillFile {/// Appends a `RecordBatch` to the spill file, initializing the writer if necessary.///+ /// Before writing, performs GC on StringView/BinaryView arrays to compact backing+ /// buffers. When a view array is sliced, it still references the original full buffers,+ /// causing massive spill files without GC (see issue #19414: 820MB → 33MB after GC).+ ///+ /// Returns the post-GC sliced memory size of the batch for memory accounting.+ ////// # Errors/// - Returns an error if the file is not active (has been finalized)/// - Returns an error if appending would exceed the disk usage limit configured/// by `max_temp_directory_size` in `DiskManager`- pub fn append_batch(&mut self, batch: &RecordBatch) -> Result<()> {+ pub fn append_batch(&mut self, batch: &RecordBatch) -> Result<usize> {if self.in_progress_file.is_none() {return Err(exec_datafusion_err!("Append operation failed: No active in-progress file. The file may have already been finalized."));}++ let gc_batch = gc_view_arrays(batch)?;+if self.writer.is_none() {// Use the SpillManager's declared schema rather than the batch's schema.// Individual batches may have different schemas (e.g., different nullability)@@ -87,7 +99,7 @@ impl InProgressSpillFile {}}if let Some(writer) = &mut self.writer {- let (spilled_rows, _) = writer.write(batch)?;+ let (spilled_rows, _) = writer.write(&gc_batch)?;if let Some(in_progress_file) = &mut self.in_progress_file {let pre_size = in_progress_file.current_disk_usage();in_progress_file.update_disk_usage()?;@@ -102,7 +114,7 @@ impl InProgressSpillFile {unreachable!() // Already checked inside current function}}- Ok(())+ gc_batch.get_sliced_size()}pub fn flush(&mut self) -> Result<()> {diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 6d51e6660e..1c900b7579 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -34,14 +34,19 @@ use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};-use arrow::array::{BufferSpec, layout};-use arrow::datatypes::{Schema, SchemaRef};+use arrow::array::{+ Array, ArrayRef, BinaryViewArray, BufferSpec, GenericByteViewArray, StringViewArray,+ layout, make_array,+};+use arrow::datatypes::DataType;+use arrow::datatypes::{ByteViewType, Schema, SchemaRef};use arrow::ipc::{MetadataVersion,reader::StreamReader,writer::{IpcWriteOptions, StreamWriter},};use arrow::record_batch::RecordBatch;+use arrow_data::ArrayDataBuilder;use datafusion_common::config::SpillCompression;use datafusion_common::{DataFusionError, Result, exec_datafusion_err};@@ -344,6 +349,174 @@ fn get_max_alignment_for_schema(schema: &Schema) -> usize {max_alignment}+/// Size of a single view structure in StringView/BinaryView arrays (in bytes).+/// Each view is 16 bytes: 4 bytes length + 4 bytes prefix + 8 bytes buffer ID/offset.+const VIEW_SIZE_BYTES: usize = 16;++/// Performs garbage collection on StringView and BinaryView arrays before spilling to reduce memory usage.+///+/// # Why GC is needed+///+/// StringView and BinaryView arrays can accumulate significant memory waste when sliced.+/// When a large array is sliced (e.g., taking first 100 rows of 1000), the view array+/// still references the original data buffers containing all 1000 rows of data.+///+/// For example, in the ClickBench benchmark (issue #19414), repeated slicing of StringView+/// arrays resulted in 820MB of spill files that could be reduced to just 33MB after GC -+/// a 96% reduction in size.+///+/// # How it works+///+/// The GC process:+/// 1. Identifies view arrays (StringView/BinaryView) in the batch+/// 2. Checks if their data buffers exceed a memory threshold+/// 3. If exceeded, calls the Arrow `gc()` method which creates new compact buffers+/// containing only the data referenced by the current views+/// 4. Returns a new batch with GC'd arrays (or original arrays if GC not needed)+///+/// # When GC is triggered+///+/// GC is only performed when data buffers exceed a threshold (currently 10KB).+/// This balances memory savings against the CPU overhead of garbage collection.+/// Small arrays are passed through unchanged since the GC overhead would exceed+/// any memory savings.+///+/// # Performance considerations+///+/// - If no view arrays need compaction, the original batch is cloned cheaply+/// - GC is skipped for small buffers to avoid unnecessary CPU overhead+/// - Nested container types are traversed recursively so view arrays inside+/// `List`, `Map`, `Union`, `Dictionary`, and other child-bearing arrays are compacted too+/// - The Arrow `gc()` method itself is optimized and only copies referenced data+pub(crate) fn gc_view_arrays(batch: &RecordBatch) -> Result<RecordBatch> {+ let mut mutated = false;+ let mut new_columns: Vec<Arc<dyn Array>> = Vec::with_capacity(batch.num_columns());++ for array in batch.columns() {+ let (gc_array, array_mutated) = gc_array(array)?;+ mutated |= array_mutated;+ new_columns.push(gc_array);+ }++ if mutated {+ Ok(RecordBatch::try_new(batch.schema(), new_columns)?)+ } else {+ Ok(batch.clone())+ }+}++fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {+ match array.data_type() {+ DataType::Utf8View => {+ let string_view = array+ .as_any()+ .downcast_ref::<StringViewArray>()+ .expect("Utf8View array should downcast to StringViewArray");+ if should_gc_view_array(string_view) {+ Ok((Arc::new(string_view.gc()) as ArrayRef, true))+ } else {+ Ok((Arc::clone(array), false))+ }+ }+ DataType::BinaryView => {+ let binary_view = array+ .as_any()+ .downcast_ref::<BinaryViewArray>()+ .expect("BinaryView array should downcast to BinaryViewArray");+ if should_gc_view_array(binary_view) {+ Ok((Arc::new(binary_view.gc()) as ArrayRef, true))+ } else {+ Ok((Arc::clone(array), false))+ }+ }+ _ => gc_array_children(array),+ }+}++fn gc_array_children(array: &ArrayRef) -> Result<(ArrayRef, bool)> {+ let data = array.to_data();+ if data.child_data().is_empty() {+ return Ok((Arc::clone(array), false));+ }++ let mut mutated = false;+ let mut child_data = Vec::with_capacity(data.child_data().len());+ for child in data.child_data() {+ let child_array = make_array(child.clone());+ let (gc_child, child_mutated) = gc_array(&child_array)?;+ mutated |= child_mutated;+ child_data.push(gc_child.to_data());+ }++ if !mutated {+ return Ok((Arc::clone(array), false));+ }++ let rebuilt = ArrayDataBuilder::new(data.data_type().clone())+ .len(data.len())+ .offset(data.offset())+ .nulls(data.nulls().cloned())+ .buffers(data.buffers().to_vec())+ .child_data(child_data)+ .build()?;++ Ok((make_array(rebuilt), true))+}++/// Determines whether a view array should be garbage collected before spilling.+///+/// Arrow's `gc()` always allocates new compact buffers (it is never a no-op), so we+/// check here to skip the allocation cost when data buffers are small. We subtract+/// the views buffer (16 bytes × n_rows) from `get_buffer_memory_size()` so the+/// threshold tracks non-inline string data rather than row count.+fn should_gc_view_array<T: ByteViewType>(array: &GenericByteViewArray<T>) -> bool {+ const MIN_BUFFER_SIZE_FOR_GC: usize = 10 * 1024; // 10KB threshold++ if array.data_buffers().is_empty() {+ return false;+ }++ let data_buffer_size = array+ .get_buffer_memory_size()+ .saturating_sub(array.len() * VIEW_SIZE_BYTES);+ data_buffer_size > MIN_BUFFER_SIZE_FOR_GC+}++#[cfg(test)]+fn calculate_string_view_waste_ratio(array: &StringViewArray) -> f64 {+ use arrow_data::MAX_INLINE_VIEW_LEN;+ calculate_view_waste_ratio(array.len(), array.data_buffers(), |i| {+ if !array.is_null(i) {+ let value = array.value(i);+ if value.len() > MAX_INLINE_VIEW_LEN as usize {+ return value.len();+ }+ }+ 0+ })+}++#[cfg(test)]+fn calculate_view_waste_ratio<F>(+ len: usize,+ data_buffers: &[arrow::buffer::Buffer],+ get_value_size: F,+) -> f64+where+ F: Fn(usize) -> usize,+{+ let total_buffer_size: usize = data_buffers.iter().map(|b| b.capacity()).sum();+ if total_buffer_size == 0 {+ return 0.0;+ }++ let mut actual_used_size = (0..len).map(get_value_size).sum::<usize>();+ actual_used_size += len * VIEW_SIZE_BYTES;++ let waste = total_buffer_size.saturating_sub(actual_used_size);+ waste as f64 / total_buffer_size as f64+}+#[cfg(test)]mod tests {use super::in_progress_spill_file::InProgressSpillFile;@@ -866,4 +1039,386 @@ mod tests {Ok(())}++ #[test]+ fn test_gc_string_view_before_spill() -> Result<()> {+ use arrow::array::StringViewArray;++ let strings: Vec<String> = (0..200)+ .map(|i| {+ if i % 2 == 0 {+ "short_string".to_string()+ } else {+ "this_is_a_much_longer_string_that_will_not_be_inlined".to_string()+ }+ })+ .collect();++ let string_array = StringViewArray::from(strings);+ let schema = Arc::new(Schema::new(vec![Field::new(+ "strings",+ DataType::Utf8View,+ false,+ )]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(string_array) as ArrayRef],+ )?;+ let sliced_batch = batch.slice(0, 20);+ let gc_batch = gc_view_arrays(&sliced_batch)?;++ assert_eq!(gc_batch.num_rows(), sliced_batch.num_rows());+ assert_eq!(gc_batch.num_columns(), sliced_batch.num_columns());++ Ok(())+ }++ #[test]+ fn test_gc_binary_view_before_spill() -> Result<()> {+ use arrow::array::BinaryViewArray;++ let binaries: Vec<Vec<u8>> = (0..200)+ .map(|i| {+ if i % 2 == 0 {+ vec![1, 2, 3, 4]+ } else {+ vec![1; 50]+ }+ })+ .collect();++ let binary_array =+ BinaryViewArray::from_iter(binaries.iter().map(|b| Some(b.as_slice())));+ let schema = Arc::new(Schema::new(vec![Field::new(+ "binaries",+ DataType::BinaryView,+ false,+ )]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(binary_array) as ArrayRef],+ )?;+ let sliced_batch = batch.slice(0, 20);+ let gc_batch = gc_view_arrays(&sliced_batch)?;++ assert_eq!(gc_batch.num_rows(), sliced_batch.num_rows());+ assert_eq!(gc_batch.num_columns(), sliced_batch.num_columns());++ Ok(())+ }++ #[test]+ fn test_gc_skips_small_arrays() -> Result<()> {+ use arrow::array::StringViewArray;++ let strings: Vec<String> = (0..10).map(|i| format!("string_{i}")).collect();++ let string_array = StringViewArray::from(strings);+ let array_ref: ArrayRef = Arc::new(string_array);++ let schema = Arc::new(Schema::new(vec![Field::new(+ "strings",+ DataType::Utf8View,+ false,+ )]));++ let batch = RecordBatch::try_new(Arc::clone(&schema), vec![array_ref])?;++ // GC should return the original batch for small arrays+ let should_gc = should_gc_view_array(+ batch+ .column(0)+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap(),+ );+ let gc_batch = gc_view_arrays(&batch)?;++ assert!(!should_gc);+ assert_eq!(gc_batch.num_rows(), batch.num_rows());+ assert!(Arc::ptr_eq(batch.column(0), gc_batch.column(0)));++ Ok(())+ }++ #[test]+ fn test_gc_with_mixed_columns() -> Result<()> {+ use arrow::array::{Int32Array, StringViewArray};++ let strings: Vec<String> = (0..200)+ .map(|i| format!("long_string_for_gc_testing_{i}"))+ .collect();++ let string_array = StringViewArray::from(strings);+ let int_array = Int32Array::from((0..200).collect::<Vec<i32>>());++ let schema = Arc::new(Schema::new(vec![+ Field::new("strings", DataType::Utf8View, false),+ Field::new("ints", DataType::Int32, false),+ ]));++ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(string_array) as ArrayRef,+ Arc::new(int_array) as ArrayRef,+ ],+ )?;++ let sliced_batch = batch.slice(0, 50);+ let gc_batch = gc_view_arrays(&sliced_batch)?;++ assert_eq!(gc_batch.num_columns(), 2);+ assert_eq!(gc_batch.num_rows(), 50);++ Ok(())+ }++ #[test]+ fn test_verify_gc_triggers_for_sliced_arrays() -> Result<()> {+ let strings: Vec<String> = (0..200)+ .map(|i| {+ format!(+ "http://example.com/very/long/path/that/exceeds/inline/threshold/{i}"+ )+ })+ .collect();++ let string_array = StringViewArray::from(strings);+ let schema = Arc::new(Schema::new(vec![Field::new(+ "url",+ DataType::Utf8View,+ false,+ )]));++ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(string_array.clone()) as ArrayRef],+ )?;++ let sliced = batch.slice(0, 20);++ let sliced_array = sliced+ .column(0)+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap();+ let should_gc = should_gc_view_array(sliced_array);+ let waste_ratio = calculate_string_view_waste_ratio(sliced_array);++ assert!(+ waste_ratio > 0.8,+ "Waste ratio should be > 0.8 for sliced array"+ );+ assert!(+ should_gc,+ "GC should trigger for sliced array with high waste"+ );++ Ok(())+ }++ #[test]+ fn test_reproduce_issue_19414_string_view_spill_without_gc() -> Result<()> {+ use arrow::array::StringViewArray;+ use std::fs;++ let num_rows = 1000;+ let mut strings = Vec::with_capacity(num_rows);++ for i in 0..num_rows {+ let url = match i % 5 {+ 0 => format!(+ "http://irr.ru/index.php?showalbum/login-leniya7777294,938303130/{i}"+ ),+ 1 => format!("http://komme%2F27.0.1453.116/very/long/path/{i}"),+ 2 => format!("https://produkty%2Fproduct/category/item/{i}"),+ 3 => format!(+ "http://irr.ru/index.php?showalbum/login-kapusta-advert2668/{i}"+ ),+ 4 => format!(+ "http://irr.ru/index.php?showalbum/login-kapustic/product/{i}"+ ),+ _ => unreachable!(),+ };+ strings.push(url);+ }++ let string_array = StringViewArray::from(strings);+ let schema = Arc::new(Schema::new(vec![Field::new(+ "URL",+ DataType::Utf8View,+ false,+ )]));++ let original_batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(string_array.clone()) as ArrayRef],+ )?;++ let total_buffer_size: usize = string_array+ .data_buffers()+ .iter()+ .map(|buffer| buffer.capacity())+ .sum();++ let mut sliced_batches = Vec::new();+ let slice_size = 100;++ for i in (0..num_rows).step_by(slice_size) {+ let len = std::cmp::min(slice_size, num_rows - i);+ let sliced = original_batch.slice(i, len);+ sliced_batches.push(sliced);+ }++ let env = Arc::new(RuntimeEnv::default());+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);+ let spill_manager = SpillManager::new(env, metrics, schema);++ let mut in_progress_file = spill_manager.create_in_progress_file("Test GC")?;++ for batch in &sliced_batches {+ in_progress_file.append_batch(batch)?;+ }++ let spill_file = in_progress_file.finish()?.unwrap();+ let file_size = fs::metadata(spill_file.path())?.len() as usize;++ let theoretical_without_gc = total_buffer_size * sliced_batches.len();+ let reduction_percent = ((theoretical_without_gc - file_size) as f64+ / theoretical_without_gc as f64)+ * 100.0;++ assert!(+ reduction_percent > 80.0,+ "GC should reduce spill file size by >80%, got {reduction_percent:.1}%"+ );++ Ok(())+ }++ #[test]+ fn test_spill_with_and_without_gc_comparison() -> Result<()> {+ let num_rows = 400;+ let strings: Vec<String> = (0..num_rows)+ .map(|i| {+ format!(+ "http://example.com/this/is/a/long/url/path/that/wont/be/inlined/{i}"+ )+ })+ .collect();++ let string_array = StringViewArray::from(strings);+ let schema = Arc::new(Schema::new(vec![Field::new(+ "url",+ DataType::Utf8View,+ false,+ )]));++ let batch =+ RecordBatch::try_new(schema, vec![Arc::new(string_array) as ArrayRef])?;++ let sliced_batch = batch.slice(0, 40);++ let array_without_gc = sliced_batch+ .column(0)+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap();+ let size_without_gc: usize = array_without_gc+ .data_buffers()+ .iter()+ .map(|buffer| buffer.capacity())+ .sum();++ let gc_batch = gc_view_arrays(&sliced_batch)?;+ let array_with_gc = gc_batch+ .column(0)+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap();+ let size_with_gc: usize = array_with_gc+ .data_buffers()+ .iter()+ .map(|buffer| buffer.capacity())+ .sum();++ let reduction_percent =+ ((size_without_gc - size_with_gc) as f64 / size_without_gc as f64) * 100.0;++ assert!(+ reduction_percent > 85.0,+ "Expected >85% reduction for 10% slice, got {reduction_percent:.1}%"+ );++ Ok(())+ }++ #[test]+ fn test_gc_recurses_into_nested_view_arrays() -> Result<()> {+ use arrow::array::{DictionaryArray, Int32Array};+ use arrow::buffer::Buffer;++ let strings: Vec<String> = (0..200)+ .map(|i| format!("http://example.com/nested/path/that/is/not/inlined/{i}"))+ .collect();+ let string_values = Arc::new(StringViewArray::from(strings)) as ArrayRef;++ let list_data = ArrayDataBuilder::new(DataType::List(Arc::new(+ Field::new_list_field(DataType::Utf8View, true),+ )))+ .len(20)+ .buffers(vec![Buffer::from_iter((0..=20).map(|i| i * 5_i32))])+ .child_data(vec![string_values.slice(0, 100).to_data()])+ .build()?;+ let list_array = make_array(list_data);++ let keys = Int32Array::from_iter_values(0..20);+ let dictionary = DictionaryArray::new(keys, string_values.slice(0, 20));+ let dictionary_array = Arc::new(dictionary) as ArrayRef;++ let schema = Arc::new(Schema::new(vec![+ Field::new(+ "list_strings",+ DataType::List(Arc::new(Field::new_list_field(DataType::Utf8View, true))),+ false,+ ),+ Field::new(+ "dictionary_strings",+ DataType::Dictionary(+ Box::new(DataType::Int32),+ Box::new(DataType::Utf8View),+ ),+ false,+ ),+ ]));+ let batch = RecordBatch::try_new(schema, vec![list_array, dictionary_array])?;+ let gc_batch = gc_view_arrays(&batch)?;++ let gc_list_values = gc_batch.column(0).to_data().child_data()[0].clone();+ let gc_list_values = make_array(gc_list_values);+ let gc_list_values = gc_list_values+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap();+ assert!(+ calculate_string_view_waste_ratio(gc_list_values) < 0.2,+ "GC should compact nested List child views"+ );++ let gc_dictionary_values = gc_batch.column(1).to_data().child_data()[0].clone();+ let gc_dictionary_values = make_array(gc_dictionary_values);+ let gc_dictionary_values = gc_dictionary_values+ .as_any()+ .downcast_ref::<StringViewArray>()+ .unwrap();+ assert!(+ calculate_string_view_waste_ratio(gc_dictionary_values) < 0.2,+ "GC should compact nested Dictionary values"+ );++ Ok(())+ }}diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex 07ba6d3989..c81c01bbe3 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -20,10 +20,9 @@use super::{SpillReaderStream, in_progress_spill_file::InProgressSpillFile};use crate::coop::cooperative;use crate::{common::spawn_buffered, metrics::SpillMetrics};-use arrow::array::StringViewArray;-use arrow::datatypes::SchemaRef;+use arrow::array::{BinaryViewArray, GenericByteViewArray, StringViewArray};+use arrow::datatypes::{ByteViewType, SchemaRef};use arrow::record_batch::RecordBatch;-use datafusion_common::utils::memory::get_record_batch_memory_size;use datafusion_common::{DataFusionError, Result, config::SpillCompression};use datafusion_execution::SendableRecordBatchStream;use datafusion_execution::disk_manager::RefCountedTempFile;@@ -127,10 +126,8 @@ impl SpillManager {if borrowed.num_rows() == 0 {return Ok(());}- in_progress_file.append_batch(borrowed)?;-- max_record_batch_size =- max_record_batch_size.max(get_record_batch_memory_size(borrowed));+ let gc_sliced_size = in_progress_file.append_batch(borrowed)?;+ max_record_batch_size = max_record_batch_size.max(gc_sliced_size);Result::<_, DataFusionError>::Ok(())})?;@@ -153,9 +150,9 @@ impl SpillManager {while let Some(batch) = stream.next().await {let batch = batch?;- in_progress_file.append_batch(&batch)?;+ let gc_sliced_size = in_progress_file.append_batch(&batch)?;- max_record_batch_size = max_record_batch_size.max(batch.get_sliced_size()?);+ max_record_batch_size = max_record_batch_size.max(gc_sliced_size);}let file = in_progress_file.finish()?;@@ -197,7 +194,7 @@ impl SpillManager {pub(crate) trait GetSlicedSize {/// Returns the size of the `RecordBatch` when sliced./// Note: if multiple arrays or even a single array share the same data buffers, we may double count each buffer.- /// Therefore, make sure we call gc() or organize_stringview_arrays() before using this method.+ /// Therefore, make sure we call gc() or gc_view_arrays() before using this method.fn get_sliced_size(&self) -> Result<usize>;}@@ -217,15 +214,24 @@ impl GetSlicedSize for RecordBatch {// "bytes needed if we materialized exactly this slice into fresh buffers".// This is a workaround until https://github.com/apache/arrow-rs/issues/8230if let Some(sv) = array.as_any().downcast_ref::<StringViewArray>() {- for buffer in sv.data_buffers() {- total += buffer.capacity();- }+ total += byte_view_data_buffer_size(sv);+ }+ if let Some(bv) = array.as_any().downcast_ref::<BinaryViewArray>() {+ total += byte_view_data_buffer_size(bv);}}Ok(total)}}+fn byte_view_data_buffer_size<T: ByteViewType>(array: &GenericByteViewArray<T>) -> usize {+ array+ .data_buffers()+ .iter()+ .map(|buffer| buffer.capacity())+ .sum()+}+#[cfg(test)]mod tests {use crate::spill::{get_record_batch_memory_size, spill_manager::GetSlicedSize};