Skip to content

Apache DataFusion SQL Query Engine

perf: prune oversized dictionaries before IPC spilling (#27015)

Frederic Branczyk6a988079ff6e
diff --git a/Cargo.lock b/Cargo.lock
index 6fd56360c5..053c7dfbd6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2539,6 +2539,7 @@ dependencies = [
"arrow-ipc",
"arrow-ord",
"arrow-schema",
+ "arrow-select",
"async-trait",
"bytes",
"criterion",
diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml
index e93f37c231..25f8e43740 100644
--- a/datafusion/physical-plan/Cargo.toml
+++ b/datafusion/physical-plan/Cargo.toml
@@ -51,6 +51,7 @@ arrow-data = { workspace = true }
arrow-ipc = { workspace = true, features = ["lz4", "zstd"] }
arrow-ord = { workspace = true }
arrow-schema = { workspace = true }
+arrow-select = { workspace = true }
async-trait = { workspace = true }
bytes = { workspace = true }
datafusion-common = { workspace = true }
diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs
index 842105c99d..262bb243c0 100644
--- a/datafusion/physical-plan/src/spill/mod.rs
+++ b/datafusion/physical-plan/src/spill/mod.rs
@@ -21,8 +21,8 @@ use std::sync::{Arc, LazyLock};
use std::task::{Context, Poll};
use arrow::array::{
- Array, ArrayRef, BinaryViewArray, BufferSpec, GenericByteViewArray, StringViewArray,
- layout, make_array,
+ Array, ArrayRef, AsArray, BinaryViewArray, BufferSpec, GenericByteViewArray,
+ StringViewArray, layout, make_array,
};
use arrow::buffer::{Buffer, ScalarBuffer};
use arrow::datatypes::DataType;
@@ -37,6 +37,7 @@ use arrow_data::{ArrayDataBuilder, ByteView, MAX_INLINE_VIEW_LEN};
#[cfg(test)]
use arrow_ipc::writer::StreamWriter;
use arrow_ipc::{CompressionType, root_as_message};
+use arrow_select::dictionary::garbage_collect_any_dictionary;
use datafusion_common::config::SpillCompression;
use datafusion_common::{Result, internal_datafusion_err};
@@ -736,7 +737,7 @@ fn get_max_alignment_for_schema(schema: &Schema) -> usize {
#[cfg(test)]
const VIEW_SIZE_BYTES: usize = 16;
-/// Performs garbage collection on StringView and BinaryView arrays before spilling to reduce memory usage.
+/// Compacts view buffers and oversized dictionaries before spilling.
///
/// # Why GC is needed
///
@@ -757,19 +758,25 @@ const VIEW_SIZE_BYTES: usize = 16;
/// containing only the data referenced by the current views
/// 4. Returns a new batch with GC'd arrays (or original arrays if GC not needed)
///
+/// Dictionary arrays with more values than keys are also compacted using Arrow's
+/// dictionary garbage collector before visiting their children. This avoids spilling
+/// whole value tables retained by sliced or merged dictionaries.
+///
/// # When GC is triggered
///
-/// GC is only performed when data buffers exceed a threshold (currently 10KB).
+/// View GC is only performed when data buffers exceed a threshold (currently 10KB).
/// This balances memory savings against the CPU overhead of garbage collection.
-/// Small arrays are passed through unchanged since the GC overhead would exceed
+/// Small view buffers are passed through unchanged since the GC overhead would exceed
/// any memory savings.
///
/// # Performance considerations
///
-/// - If no view arrays need compaction, the original batch is cloned cheaply
-/// - GC is skipped for small buffers to avoid unnecessary CPU overhead
+/// - If no arrays need compaction, the original batch is cloned cheaply
+/// - View GC is skipped for small buffers to avoid unnecessary CPU overhead
/// - Nested container types are traversed recursively so view arrays inside
/// `List`, `Map`, `Union`, `Dictionary`, and other child-bearing arrays are compacted too
+/// - Traversal uses physical children, not logical reachability. For example,
+/// unused ListView child entries can still retain dictionary values
/// - The Arrow `gc()` method itself is optimized and only copies referenced data
pub(crate) fn gc_view_arrays(batch: &RecordBatch) -> Result<RecordBatch> {
let mut mutated = false;
@@ -810,6 +817,15 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
None => (Arc::clone(array), false),
})
}
+ DataType::Dictionary(_, _)
+ if array.as_any_dictionary().values().len() > array.len() =>
+ {
+ // Struct-valued merge dictionaries can concatenate whole value tables.
+ // Prune the outer dictionary before compacting its retained children.
+ let compacted = garbage_collect_any_dictionary(array.as_any_dictionary())?;
+ let (compacted, _) = gc_array_children(&compacted)?;
+ Ok((compacted, true))
+ }
_ => gc_array_children(array),
}
}
@@ -992,9 +1008,12 @@ mod tests {
use crate::metrics::SpillMetrics;
use crate::spill::spill_manager::SpillManager;
use crate::test::build_table_i32;
- use arrow::array::{ArrayRef, Int32Array, StringArray};
+ use arrow::array::{
+ ArrayRef, DictionaryArray, Int32Array, ListArray, StringArray, StructArray,
+ };
+ use arrow::buffer::OffsetBuffer;
use arrow::compute::cast;
- use arrow::datatypes::{DataType, Field};
+ use arrow::datatypes::{DataType, Field, Int32Type};
use arrow::ipc::reader::StreamReader;
use datafusion_execution::runtime_env::RuntimeEnv;
use futures::StreamExt as _;
@@ -2693,6 +2712,64 @@ mod tests {
Ok(())
}
+ #[rstest::rstest]
+ #[case(SpillCompression::Uncompressed)]
+ #[case(SpillCompression::Lz4Frame)]
+ #[case(SpillCompression::Zstd)]
+ #[tokio::test]
+ async fn test_spill_prunes_unused_nested_dictionary_values(
+ #[case] compression: SpillCompression,
+ ) -> Result<()> {
+ let values = Arc::new(StructArray::from(vec![
+ (
+ Arc::new(Field::new("id", DataType::Int32, false)),
+ Arc::new(Int32Array::from_iter_values(0..1024)) as ArrayRef,
+ ),
+ (
+ Arc::new(Field::new("name", DataType::Utf8View, false)),
+ Arc::new(StringViewArray::from_iter_values(
+ (0..1024).map(|id| format!("{id}:{}", "x".repeat(256))),
+ )) as ArrayRef,
+ ),
+ ])) as ArrayRef;
+ let batch = |keys, values| -> Result<RecordBatch> {
+ let dictionary = DictionaryArray::<Int32Type>::try_new(keys, values)?;
+ let list = ListArray::new(
+ Arc::new(Field::new_list_field(dictionary.data_type().clone(), true)),
+ OffsetBuffer::new(vec![0_i32, 1, 3, 4].into()),
+ Arc::new(dictionary),
+ None,
+ );
+ Ok(RecordBatch::try_from_iter([(
+ "stacktrace",
+ Arc::new(list) as ArrayRef,
+ )])?)
+ };
+ let input = batch(
+ Int32Array::from(vec![Some(1023), None, Some(17), Some(1023)]),
+ Arc::clone(&values),
+ )?;
+ let expected = batch(
+ Int32Array::from(vec![Some(1), None, Some(0), Some(1)]),
+ arrow::compute::take(&values, &Int32Array::from(vec![17, 1023]), None)?,
+ )?;
+ let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0);
+ let manager =
+ SpillManager::new(Arc::new(RuntimeEnv::default()), metrics, input.schema())
+ .with_compression_type(compression);
+ let file = manager
+ .spill_record_batch_and_finish(&[input], "nested dictionary GC")?
+ .unwrap();
+ let size = file.size().unwrap();
+ assert!(
+ size < 16 * 1024,
+ "unused dictionary values spilled: {size} bytes"
+ );
+ let actual = collect(manager.read_spill_as_stream(file, None)?).await?;
+ assert_eq!(actual, vec![expected]);
+ Ok(())
+ }
+
#[tokio::test]
async fn test_spill_file_size_gc_verification_string_view() -> Result<()> {
use arrow::array::StringViewArray;