Apache DataFusion SQL Query Engine
fix: charge retained scratch indices capacity in GroupsAccumulatorAdapter (#24858)
Adrian Garcia Badaraccoa01dc7007b14
diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rsindex 1636b676ba..1e86c1fb7f 100644--- a/datafusion/core/tests/memory_limit/mod.rs+++ b/datafusion/core/tests/memory_limit/mod.rs@@ -1070,6 +1070,107 @@ async fn test_spill_file_compressed_with_lz4_frame() -> Result<()> {Ok(())}++/// Number of groups the `covar_samp` queries below produce.+const ADAPTER_GROUPS: i64 = 128;+/// Rows per input batch for the `covar_samp` queries below.+const ADAPTER_BATCH_SIZE: i64 = 8192;+/// Bytes of scratch row indices `GroupsAccumulatorAdapter` retains for the+/// `covar_samp` queries below: one `u32` per row of the largest batch each+/// group has ever received, kept for the lifetime of the group.+const ADAPTER_RETAINED_BYTES: usize =+ (ADAPTER_GROUPS * ADAPTER_BATCH_SIZE) as usize * size_of::<u32>();++/// Runs a `GROUP BY` `covar_samp` query under `memory_limit` and returns the+/// executed plan so the caller can inspect its metrics.+///+/// `covar_samp` has no native [`GroupsAccumulator`], so its per-group state is+/// held by `GroupsAccumulatorAdapter`. The adapter keeps one scratch `Vec<u32>`+/// of row indices per group, grown to the largest number of rows that group+/// has ever taken from a single input batch and retained (cleared, but not+/// deallocated) for the lifetime of the group.+///+/// The query hands each of the [`ADAPTER_GROUPS`] groups `batches_per_group`+/// consecutive full batches of [`ADAPTER_BATCH_SIZE`] rows, so the adapter+/// retains [`ADAPTER_RETAINED_BYTES`] (4 MiB) of scratch capacity however many+/// batches each group receives. Everything else the aggregate holds is two+/// orders of magnitude smaller.+///+/// `target_partitions = 1` puts the aggregate in `Single` mode, which spills+/// under memory pressure instead of emitting groups early, so the accounting+/// is observable as a spill.+///+/// [`GroupsAccumulator`]: datafusion_expr::GroupsAccumulator+async fn run_adapter_query(+ memory_limit: usize,+ batches_per_group: i64,+) -> Result<Arc<dyn ExecutionPlan>> {+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::new(GreedyMemoryPool::new(memory_limit)))+ .with_disk_manager_builder(+ DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),+ )+ .build_arc()?;++ let config = SessionConfig::new()+ .with_target_partitions(1)+ .with_batch_size(ADAPTER_BATCH_SIZE as usize);+ let ctx = SessionContext::new_with_config_rt(config, runtime);++ let rows_per_group = ADAPTER_BATCH_SIZE * batches_per_group;+ let sql = format!(+ "SELECT v / {rows_per_group} AS g, covar_samp(v, v) AS c \+ FROM generate_series(0, {}) AS t(v) \+ GROUP BY v / {rows_per_group}",+ ADAPTER_GROUPS * rows_per_group - 1+ );++ let plan = ctx.sql(&sql).await?.create_physical_plan().await?;+ let batches = collect_batches(Arc::clone(&plan), ctx.task_ctx()).await?;++ let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum();+ assert_eq!(rows, ADAPTER_GROUPS as usize);++ Ok(plan)+}++/// The scratch capacity `GroupsAccumulatorAdapter` retains is four times the+/// memory limit, so the aggregate must spill. Without the capacity everything+/// the aggregate reports is far under the limit, and the query runs to+/// completion without ever asking the pool for what it is really using.+#[tokio::test]+async fn aggregate_adapter_spills_on_retained_indices() -> Result<()> {+ let plan = run_adapter_query(ADAPTER_RETAINED_BYTES / 4, 1).await?;++ let spill_count = plan_spill_count(plan.as_ref());+ assert!(+ spill_count > 0,+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \+ against a limit of a quarter of that, so it must spill, \+ but spill_count was {spill_count}"+ );++ Ok(())+}++/// The retained scratch capacity is charged once, not once per batch. Every+/// group receives four batches, so charging the capacity per batch would+/// report four times the retained bytes, and the memory limit of twice the+/// retained bytes fits the aggregate only if it is charged once.+#[tokio::test]+async fn aggregate_adapter_charges_retained_indices_once() -> Result<()> {+ let plan = run_adapter_query(ADAPTER_RETAINED_BYTES * 2, 4).await?;++ let spill_count = plan_spill_count(plan.as_ref());+ assert_eq!(+ spill_count, 0,+ "the aggregate retains {ADAPTER_RETAINED_BYTES} bytes of scratch indices \+ under a limit of twice that, so it must not spill"+ );++ Ok(())+}+/// Run the query with the specified memory limit,/// and verifies the expected errors are returned#[derive(Clone, Debug)]diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rsindex 6704b068ac..ad01fbce5b 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs@@ -102,6 +102,10 @@ pub struct GroupsAccumulatorAdapter {/// bottleneck in earlier implementations when there were many/// distinct groups.allocation_bytes: usize,++ /// The portion of [`Self::allocation_bytes`] that is the scratch+ /// [`AccumulatorState::indices`] capacity held by [`Self::states`].+ indices_allocation_bytes: usize,}struct AccumulatorState {@@ -139,6 +143,7 @@ impl GroupsAccumulatorAdapter {factory: Box::new(factory),states: vec![],allocation_bytes: 0,+ indices_allocation_bytes: 0,}}@@ -221,8 +226,12 @@ impl GroupsAccumulatorAdapter {let mut offsets = vec![0];let mut offset_so_far = 0;+ let mut indices_allocation_bytes = 0;for (group_index, state) in self.states.iter_mut().enumerate() {let indices = &state.indices;+ // this pass already visits every group, so totalling the scratch+ // capacity here costs a field read rather than a `size()` call+ indices_allocation_bytes += indices.allocated_size();if indices.is_empty() {continue;}@@ -234,6 +243,13 @@ impl GroupsAccumulatorAdapter {}let batch_indices = batch_indices.into();+ // The push loop above is the only place `indices` grows. Charge the+ // growth since the previous batch here: the pre/post deltas below+ // observe the identical capacity on both sides, because `f` does not+ // touch `indices` and the `clear()` after it retains the capacity.+ self.adjust_allocation(self.indices_allocation_bytes, indices_allocation_bytes);+ self.indices_allocation_bytes = indices_allocation_bytes;+// reorder the values and opt_filter by batch_indices so that// all values for each group are contiguous, then invoke the// accumulator once per group with values@@ -284,6 +300,18 @@ impl GroupsAccumulatorAdapter {self.allocation_bytes = self.allocation_bytes.saturating_sub(size)}+ /// Release the allocation held by a state that is being emitted.+ ///+ /// [`AccumulatorState::size`] covers the scratch `indices` capacity, so+ /// this also drops it from [`Self::indices_allocation_bytes`] to keep that+ /// running total equal to the capacity still held by [`Self::states`].+ fn free_state_allocation(&mut self, state: &AccumulatorState) {+ self.free_allocation(state.size());+ self.indices_allocation_bytes = self+ .indices_allocation_bytes+ .saturating_sub(state.indices.allocated_size());+ }+/// Adjusts the allocation for something that started with/// start_size and now has new_size avoiding overflow///@@ -325,7 +353,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {let results: Vec<ScalarValue> = states.into_iter().map(|mut state| {- self.free_allocation(state.size());+ self.free_state_allocation(&state);state.accumulator.evaluate()}).collect::<Result<_>>()?;@@ -375,7 +403,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {let mut results: Vec<Vec<ScalarValue>> = vec![];for mut state in states {- self.free_allocation(state.size());+ self.free_state_allocation(&state);let accumulator_state = state.accumulator.state()?;results.resize_with(accumulator_state.len(), Vec::new);for (idx, state_val) in accumulator_state.into_iter().enumerate() {