Skip to content

Apache DataFusion SQL Query Engine

perf: reuse GroupsAccumulatorAdapter scratch buffers across batches (#25605)

Adrian Garcia Badaraccoca0a6a42d799
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
index d49cae6a16..dbf2af512b 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
@@ -111,6 +111,63 @@ pub struct GroupsAccumulatorAdapter {
/// Optional aggregate-owned metric timed once for a grouped update batch.
grouped_update_metric: OnceLock<Option<Arc<dyn AggregateMetric>>>,
+
+ /// Buffers reused by every call to `invoke_per_accumulator`
+ scratch: Scratch,
+}
+
+/// Per-batch buffers of [`GroupsAccumulatorAdapter`], kept between batches so
+/// that each batch does not allocate them again. Their capacity is part of
+/// [`GroupsAccumulatorAdapter::allocation_bytes`].
+#[derive(Default)]
+struct Scratch {
+ /// Group indexes that have rows in the batch, in order of first
+ /// appearance in the batch
+ groups_with_rows: Vec<usize>,
+
+ /// `offsets[i]` is the index into `batch_indices` where the rows for
+ /// `groups_with_rows[i]` start
+ offsets: Vec<usize>,
+
+ /// Indices into the batch rows, with the rows of each group contiguous
+ batch_indices: Vec<u32>,
+}
+
+/// A scratch buffer keeps its capacity after a batch unless that capacity is
+/// more than this many entries and more than [`SCRATCH_RETAIN_RATIO`] times
+/// what the batch used. Then it shrinks to the smaller of the two. This stops
+/// one unusually large batch from holding memory for all the batches after it.
+const MAX_RETAINED_SCRATCH_ENTRIES: usize = 64 * 1024;
+
+/// See [`MAX_RETAINED_SCRATCH_ENTRIES`]
+const SCRATCH_RETAIN_RATIO: usize = 4;
+
+impl Scratch {
+ fn allocated_size(&self) -> usize {
+ self.groups_with_rows.allocated_size()
+ + self.offsets.allocated_size()
+ + self.batch_indices.allocated_size()
+ }
+
+ /// Shrink each buffer that is much larger than the batch that just used
+ /// it. Call this after the batch, before the buffers are cleared for the
+ /// next batch.
+ fn release_oversized(&mut self) {
+ fn release_if_oversized<T>(buffer: &mut Vec<T>) {
+ let keep = SCRATCH_RETAIN_RATIO * buffer.len();
+ if buffer.capacity() > MAX_RETAINED_SCRATCH_ENTRIES
+ && buffer.capacity() > keep
+ {
+ // The batch is done with the contents, so clear them first
+ // and the shrink has nothing to copy
+ buffer.clear();
+ buffer.shrink_to(MAX_RETAINED_SCRATCH_ENTRIES.min(keep));
+ }
+ }
+ release_if_oversized(&mut self.groups_with_rows);
+ release_if_oversized(&mut self.offsets);
+ release_if_oversized(&mut self.batch_indices);
+ }
}
/// Maximum number of prepared group inputs retained while timing an
@@ -154,6 +211,7 @@ impl GroupsAccumulatorAdapter {
allocation_bytes: 0,
metrics: None,
grouped_update_metric: OnceLock::new(),
+ scratch: Scratch::default(),
}
}
@@ -235,10 +293,45 @@ impl GroupsAccumulatorAdapter {
assert_eq!(values[0].len(), group_indices.len());
+ // Take the scratch buffers out of `self` for the batch, and put them
+ // back on every return path, including errors
+ let mut scratch = std::mem::take(&mut self.scratch);
+ let scratch_size_pre = scratch.allocated_size();
+ let result = self.invoke_per_accumulator_with_scratch(
+ &mut scratch,
+ values,
+ group_indices,
+ opt_filter,
+ f,
+ );
+ scratch.release_oversized();
+ self.adjust_allocation(scratch_size_pre, scratch.allocated_size());
+ self.scratch = scratch;
+ result
+ }
+
+ /// Body of [`Self::invoke_per_accumulator`], with its scratch buffers
+ fn invoke_per_accumulator_with_scratch<F>(
+ &mut self,
+ scratch: &mut Scratch,
+ values: &[ArrayRef],
+ group_indices: &[usize],
+ opt_filter: Option<&BooleanArray>,
+ f: F,
+ ) -> Result<()>
+ where
+ F: Fn(&mut dyn Accumulator, &[ArrayRef]) -> Result<()>,
+ {
+ let Scratch {
+ groups_with_rows,
+ offsets,
+ batch_indices,
+ } = scratch;
+
// groups_with_rows holds a list of group indexes that have any rows
// that need to be accumulated, stored in order of first appearance in
// this batch
- let mut groups_with_rows = vec![];
+ groups_with_rows.clear();
// figure out which input rows correspond to which groups.
// Note that self.state.indices starts empty for all groups (it is
@@ -263,11 +356,18 @@ impl GroupsAccumulatorAdapter {
self.add_allocation(indices_allocation_delta);
// batch_indices holds indices into values, each group is contiguous
- let mut batch_indices = Vec::with_capacity(group_indices.len());
+ batch_indices.clear();
+ batch_indices
+ .try_reserve(group_indices.len())
+ .map_err(|e| {
+ arrow_datafusion_err!(arrow::error::ArrowError::MemoryError(
+ e.to_string()
+ ))
+ })?;
// offsets[i] is index into batch_indices where the rows for
// groups_with_rows[i] start
- let mut offsets = Vec::with_capacity(groups_with_rows.len() + 1);
+ offsets.clear();
offsets.push(0);
let mut offset_so_far = 0;
@@ -277,13 +377,20 @@ impl GroupsAccumulatorAdapter {
offset_so_far += indices.len();
offsets.push(offset_so_far);
}
- let batch_indices = batch_indices.into();
+ // Move the buffer into an array without a copy, for the take kernels
+ let indices_array =
+ PrimitiveArray::<UInt32Type>::from(std::mem::take(batch_indices));
// reorder the values and opt_filter by batch_indices so that
// all values for each group are contiguous, then invoke the
// accumulator once per group with values
- let values = take_arrays(values, &batch_indices, None)?;
- let opt_filter = get_filter_at_indices(opt_filter, &batch_indices)?;
+ let values = take_arrays(values, &indices_array, None)?;
+ let opt_filter = get_filter_at_indices(opt_filter, &indices_array)?;
+
+ // The take kernels only borrow the indices, so the array holds the
+ // only reference to the buffer and we can get it back without a copy
+ let (_, indices_buffer, _) = indices_array.into_parts();
+ *batch_indices = indices_buffer.into_inner().into_vec().unwrap_or_default();
let grouped_update_metric = self.grouped_metric();
@@ -398,6 +505,15 @@ impl GroupsAccumulatorAdapter {
self.allocation_bytes = self.allocation_bytes.saturating_sub(size)
}
+ /// Release the scratch buffers when no group is left, so that an adapter
+ /// that has emitted every group holds no memory
+ fn free_scratch_if_empty(&mut self) {
+ if self.states.is_empty() {
+ let scratch = std::mem::take(&mut self.scratch);
+ self.free_allocation(scratch.allocated_size());
+ }
+ }
+
/// Release the allocation held by a state that is being emitted.
fn free_state_allocation(&mut self, state: &AccumulatorState) {
self.free_allocation(state.size());
@@ -456,6 +572,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
let result = ScalarValue::iter_to_array(results);
self.adjust_allocation(vec_size_pre, self.states.allocated_size());
+ self.free_scratch_if_empty();
result
}
@@ -520,6 +637,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
}
}
self.adjust_allocation(vec_size_pre, self.states.allocated_size());
+ self.free_scratch_if_empty();
Ok(arrays)
}
@@ -820,7 +938,14 @@ mod tests {
let retained_indices = adapter.states[0].indices.allocated_size();
assert!(retained_indices > 0);
assert!(adapter.states[0].indices.is_empty());
- assert_eq!(adapter.size(), allocation_before_update + retained_indices);
+ // The indices buffer comes back from the take kernels without a copy
+ assert!(adapter.scratch.batch_indices.capacity() >= 4);
+ let retained_scratch = adapter.scratch.allocated_size();
+ assert!(retained_scratch > 0);
+ assert_eq!(
+ adapter.size(),
+ allocation_before_update + retained_indices + retained_scratch
+ );
let allocation_after_first_update = adapter.size();
adapter.update_batch(&[values], &[0, 0, 0, 0], None, 1)?;
@@ -1085,6 +1210,7 @@ mod tests {
.iter()
.map(AccumulatorState::size)
.sum::<usize>()
+ + accumulator.scratch.allocated_size()
);
accumulator.update_batch(&[values], &[0, 1], None, 2)?;
@@ -1129,6 +1255,7 @@ mod tests {
.iter()
.map(AccumulatorState::size)
.sum::<usize>()
+ + accumulator.scratch.allocated_size()
);
Ok(())
}
@@ -1158,6 +1285,49 @@ mod tests {
Ok(())
}
+ /// One large batch must not make the adapter keep its scratch capacity
+ /// for the smaller batches after it.
+ #[test]
+ fn adapter_releases_oversized_scratch() -> Result<()> {
+ const NUM_GROUPS: usize = 2 * MAX_RETAINED_SCRATCH_ENTRIES;
+ let mut adapter = GroupsAccumulatorAdapter::new(|| {
+ Ok(Box::new(MaxAccumulator::try_new(&DataType::Int64)?)
+ as Box<dyn Accumulator>)
+ });
+
+ // every row is a different group, so the scratch buffers need one
+ // entry for each row
+ let group_indices: Vec<usize> = (0..NUM_GROUPS).collect();
+ let values: ArrayRef =
+ Arc::new(Int64Array::from_iter_values(0..NUM_GROUPS as i64));
+ adapter.update_batch(&[values], &group_indices, None, NUM_GROUPS)?;
+ let large_scratch = adapter.scratch.allocated_size();
+ assert!(large_scratch >= NUM_GROUPS * size_of::<usize>());
+ let size_after_large_batch = adapter.size();
+
+ // a batch with one row shrinks each large buffer to
+ // SCRATCH_RETAIN_RATIO times what it used: one group, two offsets and
+ // one row index
+ let values: ArrayRef = Arc::new(Int64Array::from(vec![1]));
+ adapter.update_batch(&[values], &[0], None, NUM_GROUPS)?;
+ let small_scratch = adapter.scratch.allocated_size();
+ assert!(small_scratch < large_scratch);
+ assert_eq!(
+ adapter.scratch.groups_with_rows.capacity(),
+ SCRATCH_RETAIN_RATIO
+ );
+ assert_eq!(adapter.scratch.offsets.capacity(), 2 * SCRATCH_RETAIN_RATIO);
+ assert_eq!(
+ adapter.scratch.batch_indices.capacity(),
+ SCRATCH_RETAIN_RATIO
+ );
+ assert_eq!(
+ adapter.size(),
+ size_after_large_batch - large_scratch + small_scratch
+ );
+ Ok(())
+ }
+
#[test]
fn adapter_preserving_evaluation_uses_accumulator_contract() -> Result<()> {
let mut accumulator = GroupsAccumulatorAdapter::new(|| {