Skip to content

Apache DataFusion SQL Query Engine

fix: charge retained scratch indices capacity in GroupsAccumulatorAdapter (#24858)

Adrian Garcia Badaraccoa01dc7007b14
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs
index 1636b676ba..1e86c1fb7f 100644
--- a/datafusion/core/tests/memory_limit/mod.rs
+++ b/datafusion/core/tests/memory_limit/mod.rs
@@ -1070,6 +1070,107 @@ async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> {
Ok(())
}
+
+/// Number of groups the `covar_samp` queries below produce.
+const ADAPTER_GROUPS: i64 = 128;
+/// Rows per input batch for the `covar_samp` queries below.
+const ADAPTER_BATCH_SIZE: i64 = 8192;
+/// Bytes of scratch row indices `GroupsAccumulatorAdapter` retains for the
+/// `covar_samp` queries below: one `u32` per row of the largest batch each
+/// group has ever received, kept for the lifetime of the group.
+const ADAPTER_RETAINED_BYTES: usize =
+ (ADAPTER_GROUPS * ADAPTER_BATCH_SIZE) as usize * size_of::<u32>();
+
+/// Runs a `GROUP BY` `covar_samp` query under `memory_limit` and returns the
+/// executed plan so the caller can inspect its metrics.
+///
+/// `covar_samp` has no native [`GroupsAccumulator`], so its per-group state is
+/// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch `Vec<u32>`
+/// of row indices per group, grown to the largest number of rows that group
+/// has ever taken from a single input batch and retained (cleared, but not
+/// deallocated) for the lifetime of the group.
+///
+/// The query hands each of the [`ADAPTER_GROUPS`] groups `batches_per_group`
+/// consecutive full batches of [`ADAPTER_BATCH_SIZE`] rows, so the adapter
+/// retains [`ADAPTER_RETAINED_BYTES`] (4 MiB) of scratch capacity however many
+/// batches each group receives. Everything else the aggregate holds is two
+/// orders of magnitude smaller.
+///
+/// `target_partitions = 1` puts the aggregate in `Single` mode, which spills
+/// under memory pressure instead of emitting groups early, so the accounting
+/// is observable as a spill.
+///
+/// [`GroupsAccumulator`]: datafusion_expr::GroupsAccumulator
+async fn run_adapter_query(
+ memory_limit: usize,
+ batches_per_group: i64,
+) -> Result<Arc<dyn ExecutionPlan>> {
+ let runtime = RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(memory_limit)))
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
+ )
+ .build_arc()?;
+
+ let config = SessionConfig::new()
+ .with_target_partitions(1)
+ .with_batch_size(ADAPTER_BATCH_SIZE as usize);
+ let ctx = SessionContext::new_with_config_rt(config, runtime);
+
+ let rows_per_group = ADAPTER_BATCH_SIZE * batches_per_group;
+ let sql = format!(
+ "SELECT v / {rows_per_group} AS g, covar_samp(v, v) AS c \
+ FROM generate_series(0, {}) AS t(v) \
+ GROUP BY v / {rows_per_group}",
+ ADAPTER_GROUPS * rows_per_group - 1
+ );
+
+ let plan = ctx.sql(&sql).await?.create_physical_plan().await?;
+ let batches = collect_batches(Arc::clone(&plan), ctx.task_ctx()).await?;
+
+ let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();
+ assert_eq!(rows, ADAPTER_GROUPS as usize);
+
+ Ok(plan)
+}
+
+/// The scratch capacity `GroupsAccumulatorAdapter` retains is four times the
+/// memory limit, so the aggregate must spill. Without the capacity everything
+/// the aggregate reports is far under the limit, and the query runs to
+/// completion without ever asking the pool for what it is really using.
+#[tokio::test]
+async fn aggregate_adapter_spills_on_retained_indices() -> Result<()> {
+ let plan = run_adapter_query(ADAPTER_RETAINED_BYTES / 4, 1).await?;
+
+ let spill_count = plan_spill_count(plan.as_ref());
+ assert!(
+ spill_count > 0,
+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
+ against a limit of a quarter of that, so it must spill, \
+ but spill_count was {spill_count}"
+ );
+
+ Ok(())
+}
+
+/// The retained scratch capacity is charged once, not once per batch. Every
+/// group receives four batches, so charging the capacity per batch would
+/// report four times the retained bytes, and the memory limit of twice the
+/// retained bytes fits the aggregate only if it is charged once.
+#[tokio::test]
+async fn aggregate_adapter_charges_retained_indices_once() -> Result<()> {
+ let plan = run_adapter_query(ADAPTER_RETAINED_BYTES * 2, 4).await?;
+
+ let spill_count = plan_spill_count(plan.as_ref());
+ assert_eq!(
+ spill_count, 0,
+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \
+ under a limit of twice that, so it must not spill"
+ );
+
+ Ok(())
+}
+
/// Run the query with the specified memory limit,
/// and verifies the expected errors are returned
#[derive(Clone, Debug)]
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
index 6704b068ac..ad01fbce5b 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
@@ -102,6 +102,10 @@ pub struct GroupsAccumulatorAdapter {
/// bottleneck in earlier implementations when there were many
/// distinct groups.
allocation_bytes: usize,
+
+ /// The portion of [`Self::allocation_bytes`] that is the scratch
+ /// [`AccumulatorState::indices`] capacity held by [`Self::states`].
+ indices_allocation_bytes: usize,
}
struct AccumulatorState {
@@ -139,6 +143,7 @@ impl GroupsAccumulatorAdapter {
factory: Box::new(factory),
states: vec![],
allocation_bytes: 0,
+ indices_allocation_bytes: 0,
}
}
@@ -221,8 +226,12 @@ impl GroupsAccumulatorAdapter {
let mut offsets = vec![0];
let mut offset_so_far = 0;
+ let mut indices_allocation_bytes = 0;
for (group_index, state) in self.states.iter_mut().enumerate() {
let indices = &state.indices;
+ // this pass already visits every group, so totalling the scratch
+ // capacity here costs a field read rather than a `size()` call
+ indices_allocation_bytes += indices.allocated_size();
if indices.is_empty() {
continue;
}
@@ -234,6 +243,13 @@ impl GroupsAccumulatorAdapter {
}
let batch_indices = batch_indices.into();
+ // The push loop above is the only place `indices` grows. Charge the
+ // growth since the previous batch here: the pre/post deltas below
+ // observe the identical capacity on both sides, because `f` does not
+ // touch `indices` and the `clear()` after it retains the capacity.
+ self.adjust_allocation(self.indices_allocation_bytes, indices_allocation_bytes);
+ self.indices_allocation_bytes = indices_allocation_bytes;
+
// reorder the values and opt_filter by batch_indices so that
// all values for each group are contiguous, then invoke the
// accumulator once per group with values
@@ -284,6 +300,18 @@ impl GroupsAccumulatorAdapter {
self.allocation_bytes = self.allocation_bytes.saturating_sub(size)
}
+ /// Release the allocation held by a state that is being emitted.
+ ///
+ /// [`AccumulatorState::size`] covers the scratch `indices` capacity, so
+ /// this also drops it from [`Self::indices_allocation_bytes`] to keep that
+ /// running total equal to the capacity still held by [`Self::states`].
+ fn free_state_allocation(&mut self, state: &AccumulatorState) {
+ self.free_allocation(state.size());
+ self.indices_allocation_bytes = self
+ .indices_allocation_bytes
+ .saturating_sub(state.indices.allocated_size());
+ }
+
/// Adjusts the allocation for something that started with
/// start_size and now has new_size avoiding overflow
///
@@ -325,7 +353,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
let results: Vec<ScalarValue> = states
.into_iter()
.map(|mut state| {
- self.free_allocation(state.size());
+ self.free_state_allocation(&state);
state.accumulator.evaluate()
})
.collect::<Result<_>>()?;
@@ -375,7 +403,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
let mut results: Vec<Vec<ScalarValue>> = vec![];
for mut state in states {
- self.free_allocation(state.size());
+ self.free_state_allocation(&state);
let accumulator_state = state.accumulator.state()?;
results.resize_with(accumulator_state.len(), Vec::new);
for (idx, state_val) in accumulator_state.into_iter().enumerate() {