Apache DataFusion SQL Query Engine
fix: release per-batch accumulator scratch after a clustered aggregate emit under memory pressure (#26782)
Liang-Chi Hsieh926901f90019
diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rsindex e7301f85e0..83d9ee2367 100644--- a/datafusion/expr-common/src/groups_accumulator.rs+++ b/datafusion/expr-common/src/groups_accumulator.rs@@ -368,6 +368,14 @@ pub trait GroupsAccumulator: Send + std::any::Any {////// May be expensive; check the implementation before calling on hot paths.fn size(&self) -> usize;++ /// Keeps the stored groups but releases capacity they do not need, such as+ /// per-batch scratch buffers, so that [`Self::size`] reflects the stored+ /// groups. For example, after [`EmitTo::First`] removes most groups, an+ /// accumulator may keep a buffer it sized for the last input batch.+ ///+ /// The default does nothing.+ fn shrink_to_fit(&mut self) {}}#[cfg(test)]diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rsindex 8c86d59e44..0b968a6bd9 100644--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs@@ -496,11 +496,16 @@ impl GroupsAccumulatorAdapter {/// that has emitted every group holds no memoryfn free_scratch_if_empty(&mut self) {if self.states.is_empty() {- let scratch = std::mem::take(&mut self.scratch);- self.free_allocation(scratch.allocated_size());+ self.free_scratch();}}+ /// Release the scratch buffers. The next batch allocates them again.+ fn free_scratch(&mut self) {+ let scratch = std::mem::take(&mut self.scratch);+ self.free_allocation(scratch.allocated_size());+ }+/// Release the allocation held by a state that is being emitted.fn free_state_allocation(&mut self, state: &AccumulatorState) {self.free_allocation(state.size());@@ -653,6 +658,19 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {self.allocation_bytes}+ fn shrink_to_fit(&mut self) {+ self.free_scratch();+ // Each group keeps the capacity of its row indices after a batch, so+ // a group that had many rows in the last batch still holds them+ let mut freed = 0;+ for state in &mut self.states {+ if state.indices.is_empty() {+ freed += std::mem::take(&mut state.indices).allocated_size();+ }+ }+ self.free_allocation(freed);+ }+fn convert_to_state(&self,values: &[ArrayRef],diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rsindex a925a994c3..c60725b5d0 100644--- a/datafusion/functions-aggregate/src/approx_distinct.rs+++ b/datafusion/functions-aggregate/src/approx_distinct.rs@@ -629,6 +629,10 @@ impl GroupsAccumulator for HllGroupsAccumulator {+ self.allocated_bytes+ self.hashes.capacity() * size_of::<u64>()}++ fn shrink_to_fit(&mut self) {+ self.hashes = Vec::new();+ }}impl Debug for ApproxDistinct {diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rsindex 7cb7ebc314..e7f2d8b7ea 100644--- a/datafusion/functions-aggregate/src/first_last.rs+++ b/datafusion/functions-aggregate/src/first_last.rs@@ -882,6 +882,18 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S+ self.extreme_of_each_group_buf.0.capacity() * size_of::<usize>()+ self.extreme_of_each_group_buf.1.capacity() / 8}++ fn shrink_to_fit(&mut self) {+ // `take_state` truncates the per-batch scoreboard, which keeps its+ // capacity for every group seen before the emit. Every user of the+ // scoreboard accepts it all-false, so rebuild it at the current size.+ let num_groups = self.extreme_of_each_group_buf.1.len();+ let mut scoreboard = BooleanBufferBuilder::new(num_groups);+ scoreboard.append_n(num_groups, false);+ self.extreme_of_each_group_buf.1 = scoreboard;+ self.extreme_buf_dirty = false;+ }+fn convert_to_state(&self,values: &[ArrayRef],diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex 12a7e95491..0c7d5f123a 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -385,11 +385,14 @@ impl<AggrMode> AggregateHashTable<AggrMode> {debug_assert!(batch.num_rows() > 0);// `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the- // key/index buffers too so the memory reservation can be released- // before the batch is sorted for spilling.+ // key/index buffers and accumulator scratch too so the memory+ // reservation can be released before the batch is sorted for spilling.state.group_values.clear_shrink(0);state.batch_group_indices.clear();state.batch_group_indices.shrink_to_fit();+ for acc in &mut state.accumulators {+ acc.shrink_to_fit();+ }Ok(Some(batch))}@@ -816,6 +819,11 @@ impl HashAggregateAccumulator {self.accumulator.size()}+ /// See [`GroupsAccumulator::shrink_to_fit`].+ pub(super) fn shrink_to_fit(&mut self) {+ self.accumulator.shrink_to_fit();+ }+pub(super) fn update_batch(&mut self,values: &CompactedAccumulatorArgs,diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rsindex e75ed3012e..dbe572c615 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs@@ -287,14 +287,16 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {}/// Releases the capacity that the buffer holds beyond its stored groups,- /// such as hash table slots and the per-batch `group_indices` scratch, so- /// that [`Self::memory_size`] reflects the remaining groups.- ///- /// Accumulator capacity is not released.+ /// such as hash table slots, the per-batch `group_indices` scratch and+ /// per-batch accumulator scratch, so that [`Self::memory_size`] reflects+ /// the remaining groups.pub(in crate::aggregates) fn shrink_to_fit(&mut self) {self.buffer.group_values.shrink_to_fit();self.buffer.group_indices.clear();self.buffer.group_indices.shrink_to_fit();+ for acc in &mut self.buffer.accumulators {+ acc.shrink_to_fit();+ }}pub(in crate::aggregates) fn metrics(&self) -> ClusteredAggregateTableMetrics {@@ -337,12 +339,16 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {debug_assert!(batch.num_rows() > 0);// `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the- // key/index buffers too so the memory reservation can be released- // before the batch is passed downstream or sorted for spilling.+ // key/index buffers and accumulator scratch too so the memory+ // reservation can be released before the batch is passed downstream or+ // sorted for spilling.self.buffer.group_values.clear_shrink(0);self.buffer.group_indices.clear();self.buffer.group_indices.shrink_to_fit();self.buffer.group_clustering.reset();+ for acc in &mut self.buffer.accumulators {+ acc.shrink_to_fit();+ }Ok(Some(batch))}diff --git a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rsindex 0232a6df8c..e1f6c1458d 100644--- a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs+++ b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs@@ -424,7 +424,9 @@ mod tests {use crate::aggregates::GroupClusteringMode;use crate::aggregates::PhysicalGroupBy;use crate::aggregates::clustered_test_util::{- assert_completed_groups_emitted_under_limit, full_batches, sum_over_sorted_input,+ aggregate_expr_over_sorted_input, aggregate_over_sorted_input,+ assert_completed_groups_emitted_under_limit, assert_output_unchanged_under_limit,+ collect_under_limit, distinct_key_batches, full_batches, sum_over_sorted_input,};use crate::test::TestMemoryExec;use arrow::array::{ArrayRef, Float64Array, Int64Array, ListArray};@@ -434,6 +436,9 @@ mod tests {GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,};use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_functions_aggregate::any_value::any_value_udaf;+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;+ use datafusion_functions_aggregate::first_last::first_value_udaf;use datafusion_functions_aggregate::sum::sum_udaf;use datafusion_physical_expr::PhysicalSortExpr;use datafusion_physical_expr::aggregate::AggregateExprBuilder;@@ -559,4 +564,89 @@ mod tests {assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);assert_completed_groups_emitted_under_limit(&plan).await}++ /// Like [`completed_groups_under_memory_pressure`], but with accumulators+ /// that keep per-batch scratch. For example, `APPROX_DISTINCT` keeps the+ /// hashes of the last batch (64 KB for 8192 rows), and the+ /// `GroupsAccumulatorAdapter` of `ANY_VALUE` keeps its per-batch indices+ /// (~160 KB). After the completed groups are emitted, the accumulators must+ /// release that scratch too. The limit fits the active group, but not the+ /// scratch.+ #[tokio::test]+ async fn completed_groups_release_accumulator_scratch() -> Result<()> {+ for udaf in [approx_distinct_udaf(), any_value_udaf()] {+ let plan = aggregate_over_sorted_input(+ AggregateMode::Partial,+ &["k"],+ distinct_key_batches(),+ udaf,+ )?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_output_unchanged_under_limit(&plan, 64 * 1024).await?;+ }+ Ok(())+ }++ /// Like [`completed_groups_release_accumulator_scratch`], but the active+ /// group holds most rows of the batch. The `GroupsAccumulatorAdapter` of+ /// `ANY_VALUE` keeps the row indices it collected for each group (32 KB for+ /// the 8191 rows of the active group), so it must release them too.+ #[tokio::test]+ async fn active_group_releases_adapter_indices() -> Result<()> {+ let keys = std::iter::once(0)+ .chain(std::iter::repeat_n(1, 8191))+ .collect();+ let plan = aggregate_over_sorted_input(+ AggregateMode::Partial,+ &["k"],+ vec![vec![keys]],+ any_value_udaf(),+ )?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_output_unchanged_under_limit(&plan, 16 * 1024).await+ }++ /// Like [`completed_groups_release_accumulator_scratch`], with the native+ /// `FIRST_VALUE(v ORDER BY v)` accumulator. It keeps a bitmap of the+ /// groups each batch updated, which grows to one bit per group (16 KB for+ /// 131072 groups), so it must release that bitmap too.+ #[tokio::test]+ async fn completed_groups_release_first_value_bitmap() -> Result<()> {+ let plan = aggregate_expr_over_sorted_input(+ AggregateMode::Partial,+ &["k"],+ vec![vec![(0..131072).collect()]],+ first_value_udaf(),+ |builder, schema| {+ Ok(builder+ .order_by(vec![PhysicalSortExpr::new_default(col("v", schema)?)]))+ },+ )?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_output_unchanged_under_limit(&plan, 16 * 1024).await+ }++ /// With partial group clustering, the stream emits every state under+ /// memory pressure, so the output is not compared. After the emit, the+ /// accumulators must release their per-batch scratch, or reserving the+ /// empty table fails the limit.+ #[tokio::test]+ async fn emitted_states_release_accumulator_scratch() -> Result<()> {+ let batches = [0..8193, 8192..16385, 16384..24576]+ .into_iter()+ .map(|range| vec![range.clone().collect(), range.collect()])+ .collect();+ let plan = aggregate_over_sorted_input(+ AggregateMode::Partial,+ &["k", "k2"],+ batches,+ approx_distinct_udaf(),+ )?;+ assert!(matches!(+ plan.group_clustering_mode(),+ GroupClusteringMode::Partial(_)+ ));+ collect_under_limit(&plan, 64 * 1024).await?;+ Ok(())+ }}diff --git a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs b/datafusion/physical-plan/src/aggregates/clustered_test_util.rsindex 79e566843f..1d5670773d 100644--- a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs+++ b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs@@ -7,7 +7,7 @@use std::sync::Arc;use arrow::array::{ArrayRef, Int64Array};-use arrow::datatypes::{DataType, Field, Schema};+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::test_util::batches_to_string;use datafusion_common::{DataFusionError, Result};@@ -18,6 +18,7 @@ use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,};use datafusion_execution::runtime_env::RuntimeEnvBuilder;+use datafusion_expr::AggregateUDF;use datafusion_functions_aggregate::sum::sum_udaf;use datafusion_physical_expr::PhysicalSortExpr;use datafusion_physical_expr::aggregate::AggregateExprBuilder;@@ -63,6 +64,30 @@ pub(super) fn sum_over_sorted_input(mode: AggregateMode,keys: &[&str],batches: Vec<Vec<Vec<i64>>>,+) -> Result<AggregateExec> {+ aggregate_over_sorted_input(mode, keys, batches, sum_udaf())+}++/// Like [`sum_over_sorted_input`], but aggregates `v` with `udaf`. For `Final`+/// mode, the state of `udaf` must be a single `Int64` column.+pub(super) fn aggregate_over_sorted_input(+ mode: AggregateMode,+ keys: &[&str],+ batches: Vec<Vec<Vec<i64>>>,+ udaf: Arc<AggregateUDF>,+) -> Result<AggregateExec> {+ aggregate_expr_over_sorted_input(mode, keys, batches, udaf, |builder, _| Ok(builder))+}++/// Like [`aggregate_over_sorted_input`], but `configure` can change the+/// aggregate expression, given the input schema. For example, it can add+/// `ORDER BY v`.+pub(super) fn aggregate_expr_over_sorted_input(+ mode: AggregateMode,+ keys: &[&str],+ batches: Vec<Vec<Vec<i64>>>,+ udaf: Arc<AggregateUDF>,+ configure: impl FnOnce(AggregateExprBuilder, &SchemaRef) -> Result<AggregateExprBuilder>,) -> Result<AggregateExec> {let fields = keys.iter()@@ -76,10 +101,13 @@ pub(super) fn sum_over_sorted_input(.collect::<Result<_>>()?,);let aggregate_exprs = vec![Arc::new(- AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])- .schema(Arc::clone(&schema))- .alias("sum")- .build()?,+ configure(+ AggregateExprBuilder::new(Arc::clone(&udaf), vec![col("v", &schema)?]),+ &schema,+ )?+ .schema(Arc::clone(&schema))+ .alias(udaf.name())+ .build()?,)];let (input_schema, group_by) = if mode == AggregateMode::Final {@@ -158,6 +186,16 @@ async fn collect_with_held_memory(Ok((output, held))}+/// Runs `plan` under `limit`.+pub(super) async fn collect_under_limit(+ plan: &AggregateExec,+ limit: usize,+) -> Result<Vec<RecordBatch>> {+ let (output, _) =+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(limit))).await?;+ Ok(output)+}+/// Returns the memory `plan` holds when handing off output without a limit.pub(super) async fn held_memory(plan: &AggregateExec) -> Result<usize> {let (_, held) =diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rsindex ddd6162a21..ca85ccb59f 100644--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs@@ -873,13 +873,14 @@ mod tests {use super::*;use crate::aggregates::GroupClusteringMode;+ use crate::aggregates::clustered_test_util::collect_under_limit;use crate::aggregates::{AggregateMode, PhysicalGroupBy};use crate::common::collect;use crate::execution_plan::ExecutionPlan;use crate::test::TestMemoryExec;use crate::test::exec::BarrierExec;- use arrow::array::{AsArray, Int32Array, Int64Array, StringViewArray};+ use arrow::array::{ArrayRef, AsArray, Int32Array, Int64Array, StringViewArray};use arrow::datatypes::{DataType, Field, Int32Type, Schema};use datafusion_common::Result;use datafusion_execution::config::SessionConfig;@@ -887,6 +888,7 @@ mod tests {GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,};use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;use datafusion_functions_aggregate::count::count_udaf;use datafusion_functions_aggregate::{min_max::min_udaf, sum::sum_udaf};use datafusion_physical_expr::aggregate::AggregateExprBuilder;@@ -1678,4 +1680,59 @@ mod tests {assert_eq!(pool.reserved(), 0);Ok(())}++ /// Every row is a distinct group, so under memory pressure the stream+ /// emits every state. After the emit, `APPROX_DISTINCT` must release the+ /// hashes of the last batch (64 KB for 8192 rows), or reserving the empty+ /// table fails the limit.+ #[tokio::test]+ async fn emitted_states_release_accumulator_scratch() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("k", DataType::Int64, false),+ Field::new("v", DataType::Int64, false),+ ]));+ let batches = [0..8192, 8192..16384, 16384..24576]+ .into_iter()+ .map(|range| {+ let values: ArrayRef = Arc::new(Int64Array::from_iter_values(range));+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::clone(&values), values],+ )+ })+ .collect::<Result<Vec<_>, _>>()?;+ let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;+ let aggregate = AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".into())]),+ vec![Arc::new(+ AggregateExprBuilder::new(+ approx_distinct_udaf(),+ vec![col("v", &schema)?],+ )+ .schema(Arc::clone(&schema))+ .alias("approx_distinct")+ .build()?,+ )],+ vec![None],+ input,+ schema,+ )?;+ assert_eq!(+ aggregate.group_clustering_mode(),+ &GroupClusteringMode::None+ );+ let output = collect_under_limit(&aggregate, 64 * 1024).await?;+ assert_eq!(output.iter().map(|b| b.num_rows()).sum::<usize>(), 24576);+ assert!(+ aggregate+ .metrics()+ .unwrap()+ .sum_by_name("early_emit_count")+ .unwrap()+ .as_usize()+ > 0+ );+ Ok(())+ }}