Skip to content

Apache DataFusion SQL Query Engine

fix: count buffers shared between sort input batches once (#25820)

Andy Grovee0d64fca0f5e
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs
index 2d48ad6790..0faa4a64cf 100644
--- a/datafusion/core/tests/memory_limit/mod.rs
+++ b/datafusion/core/tests/memory_limit/mod.rs
@@ -446,6 +446,36 @@ async fn group_by_count_distinct_utf8_view() {
.await
}
+/// A grouped aggregate emits its output as zero-copy slices of one batch, and
+/// the sort that buffers them must reserve their shared buffers once rather
+/// than once per slice. For 100,000 groups that needs about 6 MB instead of
+/// about 24 MB, so 12 MB separates the two. The two orderings cover both ways
+/// the buffered slices are sorted: coalesced into runs for a single sort key,
+/// and one run per slice otherwise. The in-place threshold is below the slice
+/// size, so that coalescing keeps the slices as separate runs.
+#[tokio::test]
+async fn sort_after_group_by_reserves_shared_output_once() {
+ for order_by in ["group_key", "s desc, group_key"] {
+ TestCase::new()
+ .with_query(format!(
+ "select group_key, sum(payload) as s from t group by group_key order by {order_by}"
+ ))
+ .with_scenario(Scenario::GroupedDistinctStrings {
+ groups: 100_000,
+ string_view: false,
+ })
+ .with_config(
+ SessionConfig::new()
+ .with_target_partitions(1)
+ .with_sort_in_place_threshold_bytes(64 * 1024),
+ )
+ .with_memory_limit(12_000_000)
+ .with_expected_success()
+ .run()
+ .await
+ }
+}
+
#[tokio::test]
async fn join_by_key_multiple_partitions() {
let config = SessionConfig::new().with_target_partitions(2);
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index 7c34545b77..13065bed07 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -59,9 +59,10 @@ use crate::{
use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
-use arrow::datatypes::SchemaRef;
+use arrow::datatypes::{DataType, SchemaRef};
use datafusion_common::config::SpillCompression;
use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::utils::memory::RecordBatchMemoryCounter;
use datafusion_common::{
DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
unwrap_or_internal_err,
@@ -236,6 +237,12 @@ struct ExternalSorter {
// ========================================================================
/// Unsorted input batches stored in the memory buffer
in_mem_batches: Vec<RecordBatch>,
+ /// Counts the buffers retained by `in_mem_batches`. The batches can share
+ /// buffers, for example zero-copy slices of one larger batch, and each
+ /// buffer is reserved only once. `None` if the sorted runs keep referencing
+ /// input buffers (see [`sorted_copy_keeps_buffers`]), in which case each
+ /// batch is reserved on its own.
+ in_mem_batches_counter: Option<RecordBatchMemoryCounter>,
/// During external sorting, in-memory intermediate data will be appended to
/// this file incrementally. Once finished, this file will be moved to [`Self::finished_spill_files`].
@@ -310,9 +317,16 @@ impl ExternalSorter {
)
.with_compression_type(spill_compression);
+ let in_mem_batches_counter = (!schema
+ .fields()
+ .iter()
+ .any(|field| sorted_copy_keeps_buffers(field.data_type())))
+ .then(RecordBatchMemoryCounter::new);
+
Ok(Self {
schema,
in_mem_batches: vec![],
+ in_mem_batches_counter,
in_progress_spill_file: None,
finished_spill_files: vec![],
expr,
@@ -643,6 +657,9 @@ impl ExternalSorter {
return Ok(self.observe_if_output(empty_stream, is_output_stream));
}
+ // Every path below consumes all of `in_mem_batches`
+ self.in_mem_batches_counter = self.new_buffered_batches_counter();
+
// The elapsed compute timer is updated when the value is dropped.
// There is no need for an explicit call to drop.
let elapsed_compute = self.metrics.baseline.elapsed_compute().clone();
@@ -657,7 +674,7 @@ impl ExternalSorter {
if self.in_mem_batches.len() == 1 {
let batch = self.in_mem_batches.swap_remove(0);
let reservation = self.reservation.take();
- let sorted_stream = self.sort_batch_stream(batch, reservation)?;
+ let sorted_stream = self.sort_batch_stream(batch, reservation, None)?;
return Ok(self.observe_if_output(sorted_stream, is_output_stream));
}
@@ -674,7 +691,7 @@ impl ExternalSorter {
.try_resize(get_reserved_bytes_for_record_batch(&batch)?)
.map_err(Self::err_with_oom_context)?;
let reservation = self.reservation.take();
- let sorted_stream = self.sort_batch_stream(batch, reservation)?;
+ let sorted_stream = self.sort_batch_stream(batch, reservation, None)?;
return Ok(self.observe_if_output(sorted_stream, is_output_stream));
}
@@ -690,16 +707,11 @@ impl ExternalSorter {
batches
};
- let streams = runs
+ let streams = self
+ .sort_run_streams(runs)?
.into_iter()
- .map(|batch| {
- let reservation = self
- .reservation
- .split(get_reserved_bytes_for_record_batch(&batch)?);
- let input = self.sort_batch_stream(batch, reservation)?;
- Ok(spawn_buffered(input, 1))
- })
- .collect::<Result<_>>()?;
+ .map(|stream| spawn_buffered(stream, 1))
+ .collect();
StreamingMergeBuilder::new()
.with_streams(streams)
@@ -728,6 +740,8 @@ impl ExternalSorter {
let mut runs: Vec<RecordBatch> = Vec::new();
let mut group: Vec<RecordBatch> = Vec::new();
let mut group_bytes = 0usize;
+ // Count the buffers the batches share as when they were buffered
+ let mut counter = self.new_buffered_batches_counter();
// Flush a group into a run, skipping the copy for a single-batch group.
let flush = |group: &mut Vec<RecordBatch>,
@@ -746,7 +760,8 @@ impl ExternalSorter {
};
for batch in batches {
- let bytes = get_reserved_bytes_for_record_batch(&batch)?;
+ let bytes =
+ get_reserved_bytes_for_next_record_batch(&batch, counter.as_mut())?;
if !group.is_empty() && group_bytes.saturating_add(bytes) > target {
flush(&mut group, &mut runs, &self.schema)?;
group_bytes = 0;
@@ -757,9 +772,10 @@ impl ExternalSorter {
flush(&mut group, &mut runs, &self.schema)?;
// Realign the reservation: concatenation may shift the footprint slightly.
+ let mut counter = self.new_buffered_batches_counter();
let total: usize = runs
.iter()
- .map(get_reserved_bytes_for_record_batch)
+ .map(|run| get_reserved_bytes_for_next_record_batch(run, counter.as_mut()))
.sum::<Result<usize>>()?;
self.reservation
.try_resize(total)
@@ -768,10 +784,64 @@ impl ExternalSorter {
Ok(runs)
}
+ /// Returns a counter for sizing batches that are buffered together, or
+ /// `None` if each batch is sized on its own (see
+ /// [`Self::in_mem_batches_counter`]).
+ fn new_buffered_batches_counter(&self) -> Option<RecordBatchMemoryCounter> {
+ self.in_mem_batches_counter
+ .as_ref()
+ .map(|_| RecordBatchMemoryCounter::new())
+ }
+
+ /// Sorts each run into its own stream, splitting `self.reservation` between
+ /// them.
+ ///
+ /// Runs can share buffers, for example zero-copy slices of one larger batch,
+ /// and `self.reservation` covers each shared buffer once. A shared buffer
+ /// stays alive until the last run holding it is sorted, so if any buffers are
+ /// shared, the reservation for every run's input is held until all the runs
+ /// are sorted. Otherwise each run releases its input once it is sorted.
+ fn sort_run_streams(
+ &self,
+ runs: Vec<RecordBatch>,
+ ) -> Result<Vec<SendableRecordBatchStream>> {
+ let mut counter = self.new_buffered_batches_counter();
+ let mut shares_buffers = false;
+ let mut sizes = Vec::with_capacity(runs.len());
+ for run in &runs {
+ let memory_size = get_record_batch_memory_size(run);
+ let retained = counter
+ .as_mut()
+ .map_or(memory_size, |counter| counter.count_batch(run));
+ shares_buffers |= retained < memory_size;
+ sizes.push((retained, run.get_sliced_size()?));
+ }
+
+ let shared_input = shares_buffers.then(|| {
+ let retained = sizes.iter().map(|(retained, _)| retained).sum();
+ Arc::new(self.reservation.split(retained))
+ });
+
+ runs.into_iter()
+ .zip(sizes)
+ .map(|(run, (retained, sliced))| {
+ let size = match shared_input {
+ Some(_) => sliced,
+ None => get_reserved_bytes_for_record_batch_size(retained, sliced),
+ };
+ let reservation = self.reservation.split(size);
+ self.sort_batch_stream(run, reservation, shared_input.clone())
+ })
+ .collect()
+ }
+
/// Sorts a single `RecordBatch` into a single stream.
///
/// This may output multiple batches depending on the size of the
/// sorted data and the target batch size.
+ /// If given, `shared_input` covers the input buffers of every run, which can
+ /// share buffers, and `reservation` only covers sorting `batch`. This stream
+ /// drops its handle to `shared_input` once `batch` is sorted.
/// For single-batch output cases, `reservation` will be freed immediately after sorting,
/// as the batch will be output and is expected to be reserved by the consumer of the stream.
/// For multi-batch output cases, `reservation` and any borrowed spill workspace
@@ -781,11 +851,14 @@ impl ExternalSorter {
&self,
batch: RecordBatch,
reservation: MemoryReservation,
+ shared_input: Option<Arc<MemoryReservation>>,
) -> Result<SendableRecordBatchStream> {
- assert_eq!(
- get_reserved_bytes_for_record_batch(&batch)?,
- reservation.size()
- );
+ let reserved_bytes = get_reserved_bytes_for_record_batch(&batch)?;
+ match &shared_input {
+ None => assert_eq!(reserved_bytes, reservation.size()),
+ // `shared_input` covers every buffer of every run's input
+ Some(shared) => assert!(reservation.size() + shared.size() >= reserved_bytes),
+ }
let schema = batch.schema();
let expressions = self.expr.clone();
@@ -793,6 +866,9 @@ impl ExternalSorter {
let merge_pool = Arc::clone(&self.merge_pool);
let stream = futures::stream::once(async move {
+ // Held until `batch` is sorted, when this block ends. The last run to
+ // drop it releases the reservation for the runs' input.
+ let _shared_input = shared_input;
let schema = batch.schema();
// Sort the batch immediately and get all output batches
@@ -865,7 +941,10 @@ impl ExternalSorter {
&mut self,
input: &RecordBatch,
) -> Result<()> {
- let size = get_reserved_bytes_for_record_batch(input)?;
+ let size = get_reserved_bytes_for_next_record_batch(
+ input,
+ self.in_mem_batches_counter.as_mut(),
+ )?;
match self.reservation.try_grow(size) {
Ok(_) => Ok(()),
@@ -874,8 +953,13 @@ impl ExternalSorter {
return Err(Self::err_with_oom_context(e));
}
- // Spill and try again.
+ // Spill and try again. Nothing is buffered after the spill, so
+ // `input` shares no buffers with a buffered batch.
self.sort_and_spill_in_mem_batches().await?;
+ let size = get_reserved_bytes_for_next_record_batch(
+ input,
+ self.in_mem_batches_counter.as_mut(),
+ )?;
self.reservation
.try_grow(size)
.map_err(Self::err_with_oom_context)
@@ -946,6 +1030,59 @@ pub(crate) fn get_reserved_bytes_for_record_batch(batch: &RecordBatch) -> Result
})
}
+/// Estimate how much memory is needed to sort `batch` when it is buffered with
+/// the batches `counter` has already counted.
+///
+/// Like [`get_reserved_bytes_for_record_batch`], but only counts the buffers of
+/// `batch` that `counter` has not seen. Buffered batches can share buffers, for
+/// example zero-copy slices of one larger batch, and a shared buffer only needs
+/// to be reserved once. Without a `counter`, every buffer of `batch` is counted.
+fn get_reserved_bytes_for_next_record_batch(
+ batch: &RecordBatch,
+ counter: Option<&mut RecordBatchMemoryCounter>,
+) -> Result<usize> {
+ let Some(counter) = counter else {
+ return get_reserved_bytes_for_record_batch(batch);
+ };
+ let sliced_size = batch.get_sliced_size()?;
+ Ok(get_reserved_bytes_for_record_batch_size(
+ counter.count_batch(batch),
+ sliced_size,
+ ))
+}
+
+/// Returns whether the sorted copy of an array of `data_type` keeps
+/// referencing some of the array's buffers.
+///
+/// Sorting takes the selected rows into new buffers, except that `take` keeps
+/// the data buffers of view arrays and the values of dictionaries and list
+/// views. Every sorted run is charged for those buffers in full, so buffered
+/// batches that share them must be charged for them each, too. Otherwise the
+/// sorter buffers more batches than the merge of their sorted runs can hold.
+fn sorted_copy_keeps_buffers(data_type: &DataType) -> bool {
+ match data_type {
+ DataType::Utf8View
+ | DataType::BinaryView
+ | DataType::Dictionary(_, _)
+ | DataType::ListView(_)
+ | DataType::LargeListView(_) => true,
+ DataType::List(field)
+ | DataType::LargeList(field)
+ | DataType::FixedSizeList(field, _)
+ | DataType::Map(field, _) => sorted_copy_keeps_buffers(field.data_type()),
+ DataType::Struct(fields) => fields
+ .iter()
+ .any(|field| sorted_copy_keeps_buffers(field.data_type())),
+ DataType::Union(fields, _) => fields
+ .iter()
+ .any(|(_, field)| sorted_copy_keeps_buffers(field.data_type())),
+ DataType::RunEndEncoded(_, values) => {
+ sorted_copy_keeps_buffers(values.data_type())
+ }
+ _ => false,
+ }
+}
+
impl Debug for ExternalSorter {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
f.debug_struct("ExternalSorter")
diff --git a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
index f41ca4507c..60b151030c 100644
--- a/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
+++ b/datafusion/physical-plan/src/sorts/sort/spill_tests.rs
@@ -146,21 +146,32 @@ fn fixture() -> Result<Fixture> {
Field::new("sales", DataType::Decimal128(28, 2), true),
Field::new("payload", DataType::Utf8, false),
]));
- let categories = StringArray::from_iter(
- (0..total).map(|i| (i % 11 != 0).then(|| format!("category-{:04}", i % 100))),
- );
- // Every non-null sales value is distinct, so sorting has no ambiguous ties.
- let sales = Decimal128Array::from_iter(
- (0..total).map(|i| (i != 1).then_some(((total - i) * 100) as i128)),
- )
- .with_precision_and_scale(28, 2)?;
- let payload = StringArray::from_iter_values(
- (0..total).map(|i| format!("row-{i:08}-{}", "x".repeat(64))),
- );
- let columns: Vec<ArrayRef> =
- vec![Arc::new(categories), Arc::new(sales), Arc::new(payload)];
- let parent = RecordBatch::try_new(schema, columns)?;
- let batches: Vec<_> = (0..count).map(|i| parent.slice(i * rows, rows)).collect();
+ // Build each batch separately so it owns its buffers. Zero-copy slices of
+ // one parent would share them, and the sort reserves shared buffers once.
+ let batches = (0..count)
+ .map(|batch| {
+ let range = batch * rows..(batch + 1) * rows;
+ let categories = StringArray::from_iter(
+ range
+ .clone()
+ .map(|i| (i % 11 != 0).then(|| format!("category-{:04}", i % 100))),
+ );
+ // Every non-null sales value is distinct, so sorting has no ambiguous ties.
+ let sales = Decimal128Array::from_iter(
+ range
+ .clone()
+ .map(|i| (i != 1).then_some(((total - i) * 100) as i128)),
+ )
+ .with_precision_and_scale(28, 2)?;
+ let payload = StringArray::from_iter_values(
+ range.map(|i| format!("row-{i:08}-{}", "x".repeat(64))),
+ );
+ let columns: Vec<ArrayRef> =
+ vec![Arc::new(categories), Arc::new(sales), Arc::new(payload)];
+ Ok(RecordBatch::try_new(Arc::clone(&schema), columns)?)
+ })
+ .collect::<Result<Vec<_>>>()?;
+ let parent = concat_batches(&schema, &batches)?;
let ordering = [
PhysicalSortExpr::new(
Arc::new(Column::new("category", 0)),