Skip to content

Apache DataFusion SQL Query Engine

fix: release per-batch accumulator scratch after a clustered aggregate emit under memory pressure (#26782)

Liang-Chi Hsieh926901f90019
diff --git a/datafusion/expr-common/src/groups_accumulator.rs b/datafusion/expr-common/src/groups_accumulator.rs
index e7301f85e0..83d9ee2367 100644
--- a/datafusion/expr-common/src/groups_accumulator.rs
+++ b/datafusion/expr-common/src/groups_accumulator.rs
@@ -368,6 +368,14 @@ pub trait GroupsAccumulator: Send + std::any::Any {
///
/// May be expensive; check the implementation before calling on hot paths.
fn size(&self) -> usize;
+
+ /// Keeps the stored groups but releases capacity they do not need, such as
+ /// per-batch scratch buffers, so that [`Self::size`] reflects the stored
+ /// groups. For example, after [`EmitTo::First`] removes most groups, an
+ /// accumulator may keep a buffer it sized for the last input batch.
+ ///
+ /// The default does nothing.
+ fn shrink_to_fit(&mut self) {}
}
#[cfg(test)]
diff --git a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
index 8c86d59e44..0b968a6bd9 100644
--- a/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
+++ b/datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs
@@ -496,11 +496,16 @@ impl GroupsAccumulatorAdapter {
/// that has emitted every group holds no memory
fn free_scratch_if_empty(&mut self) {
if self.states.is_empty() {
- let scratch = std::mem::take(&mut self.scratch);
- self.free_allocation(scratch.allocated_size());
+ self.free_scratch();
}
}
+ /// Release the scratch buffers. The next batch allocates them again.
+ fn free_scratch(&mut self) {
+ let scratch = std::mem::take(&mut self.scratch);
+ self.free_allocation(scratch.allocated_size());
+ }
+
/// Release the allocation held by a state that is being emitted.
fn free_state_allocation(&mut self, state: &AccumulatorState) {
self.free_allocation(state.size());
@@ -653,6 +658,19 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
self.allocation_bytes
}
+ fn shrink_to_fit(&mut self) {
+ self.free_scratch();
+ // Each group keeps the capacity of its row indices after a batch, so
+ // a group that had many rows in the last batch still holds them
+ let mut freed = 0;
+ for state in &mut self.states {
+ if state.indices.is_empty() {
+ freed += std::mem::take(&mut state.indices).allocated_size();
+ }
+ }
+ self.free_allocation(freed);
+ }
+
fn convert_to_state(
&self,
values: &[ArrayRef],
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index a925a994c3..c60725b5d0 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -629,6 +629,10 @@ impl GroupsAccumulator for HllGroupsAccumulator {
+ self.allocated_bytes
+ self.hashes.capacity() * size_of::<u64>()
}
+
+ fn shrink_to_fit(&mut self) {
+ self.hashes = Vec::new();
+ }
}
impl Debug for ApproxDistinct {
diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs
index 7cb7ebc314..e7f2d8b7ea 100644
--- a/datafusion/functions-aggregate/src/first_last.rs
+++ b/datafusion/functions-aggregate/src/first_last.rs
@@ -882,6 +882,18 @@ impl<S: ValueState + 'static> GroupsAccumulator for FirstLastGroupsAccumulator<S
+ self.extreme_of_each_group_buf.0.capacity() * size_of::<usize>()
+ self.extreme_of_each_group_buf.1.capacity() / 8
}
+
+ fn shrink_to_fit(&mut self) {
+ // `take_state` truncates the per-batch scoreboard, which keeps its
+ // capacity for every group seen before the emit. Every user of the
+ // scoreboard accepts it all-false, so rebuild it at the current size.
+ let num_groups = self.extreme_of_each_group_buf.1.len();
+ let mut scoreboard = BooleanBufferBuilder::new(num_groups);
+ scoreboard.append_n(num_groups, false);
+ self.extreme_of_each_group_buf.1 = scoreboard;
+ self.extreme_buf_dirty = false;
+ }
+
fn convert_to_state(
&self,
values: &[ArrayRef],
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
index 12a7e95491..0c7d5f123a 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -385,11 +385,14 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
debug_assert!(batch.num_rows() > 0);
// `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the
- // key/index buffers too so the memory reservation can be released
- // before the batch is sorted for spilling.
+ // key/index buffers and accumulator scratch too so the memory
+ // reservation can be released before the batch is sorted for spilling.
state.group_values.clear_shrink(0);
state.batch_group_indices.clear();
state.batch_group_indices.shrink_to_fit();
+ for acc in &mut state.accumulators {
+ acc.shrink_to_fit();
+ }
Ok(Some(batch))
}
@@ -816,6 +819,11 @@ impl HashAggregateAccumulator {
self.accumulator.size()
}
+ /// See [`GroupsAccumulator::shrink_to_fit`].
+ pub(super) fn shrink_to_fit(&mut self) {
+ self.accumulator.shrink_to_fit();
+ }
+
pub(super) fn update_batch(
&mut self,
values: &CompactedAccumulatorArgs,
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs
index e75ed3012e..dbe572c615 100644
--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs
@@ -287,14 +287,16 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {
}
/// Releases the capacity that the buffer holds beyond its stored groups,
- /// such as hash table slots and the per-batch `group_indices` scratch, so
- /// that [`Self::memory_size`] reflects the remaining groups.
- ///
- /// Accumulator capacity is not released.
+ /// such as hash table slots, the per-batch `group_indices` scratch and
+ /// per-batch accumulator scratch, so that [`Self::memory_size`] reflects
+ /// the remaining groups.
pub(in crate::aggregates) fn shrink_to_fit(&mut self) {
self.buffer.group_values.shrink_to_fit();
self.buffer.group_indices.clear();
self.buffer.group_indices.shrink_to_fit();
+ for acc in &mut self.buffer.accumulators {
+ acc.shrink_to_fit();
+ }
}
pub(in crate::aggregates) fn metrics(&self) -> ClusteredAggregateTableMetrics {
@@ -337,12 +339,16 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {
debug_assert!(batch.num_rows() > 0);
// `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the
- // key/index buffers too so the memory reservation can be released
- // before the batch is passed downstream or sorted for spilling.
+ // key/index buffers and accumulator scratch too so the memory
+ // reservation can be released before the batch is passed downstream or
+ // sorted for spilling.
self.buffer.group_values.clear_shrink(0);
self.buffer.group_indices.clear();
self.buffer.group_indices.shrink_to_fit();
self.buffer.group_clustering.reset();
+ for acc in &mut self.buffer.accumulators {
+ acc.shrink_to_fit();
+ }
Ok(Some(batch))
}
diff --git a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
index 0232a6df8c..e1f6c1458d 100644
--- a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
@@ -424,7 +424,9 @@ mod tests {
use crate::aggregates::GroupClusteringMode;
use crate::aggregates::PhysicalGroupBy;
use crate::aggregates::clustered_test_util::{
- assert_completed_groups_emitted_under_limit, full_batches, sum_over_sorted_input,
+ aggregate_expr_over_sorted_input, aggregate_over_sorted_input,
+ assert_completed_groups_emitted_under_limit, assert_output_unchanged_under_limit,
+ collect_under_limit, distinct_key_batches, full_batches, sum_over_sorted_input,
};
use crate::test::TestMemoryExec;
use arrow::array::{ArrayRef, Float64Array, Int64Array, ListArray};
@@ -434,6 +436,9 @@ mod tests {
GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
};
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+ use datafusion_functions_aggregate::any_value::any_value_udaf;
+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;
+ use datafusion_functions_aggregate::first_last::first_value_udaf;
use datafusion_functions_aggregate::sum::sum_udaf;
use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
@@ -559,4 +564,89 @@ mod tests {
assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
assert_completed_groups_emitted_under_limit(&plan).await
}
+
+ /// Like [`completed_groups_under_memory_pressure`], but with accumulators
+ /// that keep per-batch scratch. For example, `APPROX_DISTINCT` keeps the
+ /// hashes of the last batch (64 KB for 8192 rows), and the
+ /// `GroupsAccumulatorAdapter` of `ANY_VALUE` keeps its per-batch indices
+ /// (~160 KB). After the completed groups are emitted, the accumulators must
+ /// release that scratch too. The limit fits the active group, but not the
+ /// scratch.
+ #[tokio::test]
+ async fn completed_groups_release_accumulator_scratch() -> Result<()> {
+ for udaf in [approx_distinct_udaf(), any_value_udaf()] {
+ let plan = aggregate_over_sorted_input(
+ AggregateMode::Partial,
+ &["k"],
+ distinct_key_batches(),
+ udaf,
+ )?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_output_unchanged_under_limit(&plan, 64 * 1024).await?;
+ }
+ Ok(())
+ }
+
+ /// Like [`completed_groups_release_accumulator_scratch`], but the active
+ /// group holds most rows of the batch. The `GroupsAccumulatorAdapter` of
+ /// `ANY_VALUE` keeps the row indices it collected for each group (32 KB for
+ /// the 8191 rows of the active group), so it must release them too.
+ #[tokio::test]
+ async fn active_group_releases_adapter_indices() -> Result<()> {
+ let keys = std::iter::once(0)
+ .chain(std::iter::repeat_n(1, 8191))
+ .collect();
+ let plan = aggregate_over_sorted_input(
+ AggregateMode::Partial,
+ &["k"],
+ vec![vec![keys]],
+ any_value_udaf(),
+ )?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_output_unchanged_under_limit(&plan, 16 * 1024).await
+ }
+
+ /// Like [`completed_groups_release_accumulator_scratch`], with the native
+ /// `FIRST_VALUE(v ORDER BY v)` accumulator. It keeps a bitmap of the
+ /// groups each batch updated, which grows to one bit per group (16 KB for
+ /// 131072 groups), so it must release that bitmap too.
+ #[tokio::test]
+ async fn completed_groups_release_first_value_bitmap() -> Result<()> {
+ let plan = aggregate_expr_over_sorted_input(
+ AggregateMode::Partial,
+ &["k"],
+ vec![vec![(0..131072).collect()]],
+ first_value_udaf(),
+ |builder, schema| {
+ Ok(builder
+ .order_by(vec![PhysicalSortExpr::new_default(col("v", schema)?)]))
+ },
+ )?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_output_unchanged_under_limit(&plan, 16 * 1024).await
+ }
+
+ /// With partial group clustering, the stream emits every state under
+ /// memory pressure, so the output is not compared. After the emit, the
+ /// accumulators must release their per-batch scratch, or reserving the
+ /// empty table fails the limit.
+ #[tokio::test]
+ async fn emitted_states_release_accumulator_scratch() -> Result<()> {
+ let batches = [0..8193, 8192..16385, 16384..24576]
+ .into_iter()
+ .map(|range| vec![range.clone().collect(), range.collect()])
+ .collect();
+ let plan = aggregate_over_sorted_input(
+ AggregateMode::Partial,
+ &["k", "k2"],
+ batches,
+ approx_distinct_udaf(),
+ )?;
+ assert!(matches!(
+ plan.group_clustering_mode(),
+ GroupClusteringMode::Partial(_)
+ ));
+ collect_under_limit(&plan, 64 * 1024).await?;
+ Ok(())
+ }
}
diff --git a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs
index 79e566843f..1d5670773d 100644
--- a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs
+++ b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs
@@ -7,7 +7,7 @@
use std::sync::Arc;
use arrow::array::{ArrayRef, Int64Array};
-use arrow::datatypes::{DataType, Field, Schema};
+use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::test_util::batches_to_string;
use datafusion_common::{DataFusionError, Result};
@@ -18,6 +18,7 @@ use datafusion_execution::memory_pool::{
GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
};
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+use datafusion_expr::AggregateUDF;
use datafusion_functions_aggregate::sum::sum_udaf;
use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
@@ -63,6 +64,30 @@ pub(super) fn sum_over_sorted_input(
mode: AggregateMode,
keys: &[&str],
batches: Vec<Vec<Vec<i64>>>,
+) -> Result<AggregateExec> {
+ aggregate_over_sorted_input(mode, keys, batches, sum_udaf())
+}
+
+/// Like [`sum_over_sorted_input`], but aggregates `v` with `udaf`. For `Final`
+/// mode, the state of `udaf` must be a single `Int64` column.
+pub(super) fn aggregate_over_sorted_input(
+ mode: AggregateMode,
+ keys: &[&str],
+ batches: Vec<Vec<Vec<i64>>>,
+ udaf: Arc<AggregateUDF>,
+) -> Result<AggregateExec> {
+ aggregate_expr_over_sorted_input(mode, keys, batches, udaf, |builder, _| Ok(builder))
+}
+
+/// Like [`aggregate_over_sorted_input`], but `configure` can change the
+/// aggregate expression, given the input schema. For example, it can add
+/// `ORDER BY v`.
+pub(super) fn aggregate_expr_over_sorted_input(
+ mode: AggregateMode,
+ keys: &[&str],
+ batches: Vec<Vec<Vec<i64>>>,
+ udaf: Arc<AggregateUDF>,
+ configure: impl FnOnce(AggregateExprBuilder, &SchemaRef) -> Result<AggregateExprBuilder>,
) -> Result<AggregateExec> {
let fields = keys
.iter()
@@ -76,10 +101,13 @@ pub(super) fn sum_over_sorted_input(
.collect::<Result<_>>()?,
);
let aggregate_exprs = vec![Arc::new(
- AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("sum")
- .build()?,
+ configure(
+ AggregateExprBuilder::new(Arc::clone(&udaf), vec![col("v", &schema)?]),
+ &schema,
+ )?
+ .schema(Arc::clone(&schema))
+ .alias(udaf.name())
+ .build()?,
)];
let (input_schema, group_by) = if mode == AggregateMode::Final {
@@ -158,6 +186,16 @@ async fn collect_with_held_memory(
Ok((output, held))
}
+/// Runs `plan` under `limit`.
+pub(super) async fn collect_under_limit(
+ plan: &AggregateExec,
+ limit: usize,
+) -> Result<Vec<RecordBatch>> {
+ let (output, _) =
+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(limit))).await?;
+ Ok(output)
+}
+
/// Returns the memory `plan` holds when handing off output without a limit.
pub(super) async fn held_memory(plan: &AggregateExec) -> Result<usize> {
let (_, held) =
diff --git a/datafusion/physical-plan/src/aggregates/hash_stream.rs b/datafusion/physical-plan/src/aggregates/hash_stream.rs
index ddd6162a21..ca85ccb59f 100644
--- a/datafusion/physical-plan/src/aggregates/hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_stream.rs
@@ -873,13 +873,14 @@ mod tests {
use super::*;
use crate::aggregates::GroupClusteringMode;
+ use crate::aggregates::clustered_test_util::collect_under_limit;
use crate::aggregates::{AggregateMode, PhysicalGroupBy};
use crate::common::collect;
use crate::execution_plan::ExecutionPlan;
use crate::test::TestMemoryExec;
use crate::test::exec::BarrierExec;
- use arrow::array::{AsArray, Int32Array, Int64Array, StringViewArray};
+ use arrow::array::{ArrayRef, AsArray, Int32Array, Int64Array, StringViewArray};
use arrow::datatypes::{DataType, Field, Int32Type, Schema};
use datafusion_common::Result;
use datafusion_execution::config::SessionConfig;
@@ -887,6 +888,7 @@ mod tests {
GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
};
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+ use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;
use datafusion_functions_aggregate::count::count_udaf;
use datafusion_functions_aggregate::{min_max::min_udaf, sum::sum_udaf};
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
@@ -1678,4 +1680,59 @@ mod tests {
assert_eq!(pool.reserved(), 0);
Ok(())
}
+
+ /// Every row is a distinct group, so under memory pressure the stream
+ /// emits every state. After the emit, `APPROX_DISTINCT` must release the
+ /// hashes of the last batch (64 KB for 8192 rows), or reserving the empty
+ /// table fails the limit.
+ #[tokio::test]
+ async fn emitted_states_release_accumulator_scratch() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("k", DataType::Int64, false),
+ Field::new("v", DataType::Int64, false),
+ ]));
+ let batches = [0..8192, 8192..16384, 16384..24576]
+ .into_iter()
+ .map(|range| {
+ let values: ArrayRef = Arc::new(Int64Array::from_iter_values(range));
+ RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::clone(&values), values],
+ )
+ })
+ .collect::<Result<Vec<_>, _>>()?;
+ let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;
+ let aggregate = AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".into())]),
+ vec![Arc::new(
+ AggregateExprBuilder::new(
+ approx_distinct_udaf(),
+ vec![col("v", &schema)?],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("approx_distinct")
+ .build()?,
+ )],
+ vec![None],
+ input,
+ schema,
+ )?;
+ assert_eq!(
+ aggregate.group_clustering_mode(),
+ &GroupClusteringMode::None
+ );
+ let output = collect_under_limit(&aggregate, 64 * 1024).await?;
+ assert_eq!(output.iter().map(|b| b.num_rows()).sum::<usize>(), 24576);
+ assert!(
+ aggregate
+ .metrics()
+ .unwrap()
+ .sum_by_name("early_emit_count")
+ .unwrap()
+ .as_usize()
+ > 0
+ );
+ Ok(())
+ }
}