Skip to content

Apache DataFusion SQL Query Engine

fix: emit completed groups when fully sorted final and single aggrega… (#26637)

Yash Aggarwalad160042e784
diff --git a/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs
index 9c71c5ee13..b5890e8891 100644
--- a/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs
@@ -261,7 +261,9 @@ impl ClusteredFinalAggregateStream {
/// 1. Start.
/// 2. Merge one input batch:
/// - If memory fits and no groups are complete, continue reading input.
- /// - If OOM, spill.
+ /// - If OOM with a spill context, spill.
+ /// - If OOM without one, emit the completed groups (edge 3), or return
+ /// the error when no group is complete.
/// 3. Prepare output:
/// - Before any spill, a prefix of groups is complete: materialize the
/// entire prefix once, retaining the input and active groups to resume
@@ -336,27 +338,33 @@ impl Aggregating {
let timer = elapsed_compute.timer();
self.table.aggregate_batch(&batch)?;
- match reservation.try_resize(self.reservation_size()) {
- Ok(()) => {}
+ // Check the reservation before taking completed groups, as the partial
+ // stream does, so spilling is unchanged. Without spilling, the refusal
+ // is returned below only if no group is complete.
+ let oom = match reservation.try_resize(self.reservation_size()) {
+ Ok(()) => None,
Err(oom @ DataFusionError::ResourcesExhausted(_)) => {
- let Some(spill_context) = self.spill_context.as_mut() else {
- return Err(oom);
- };
- if self.table.is_empty() {
- return Err(oom);
+ match self.spill_context.as_mut() {
+ None => Some(oom),
+ Some(spill_context) => {
+ if self.table.is_empty() {
+ return Err(oom);
+ }
+ spill_context
+ .sort_and_spill(self.table.take_state_batch()?)?;
+ reservation.try_resize(self.table.memory_size()).map_err(
+ |e| {
+ e.context(
+ "Decreasing allocation after spilling should succeed",
+ )
+ },
+ )?;
+ continue;
+ }
}
- spill_context.sort_and_spill(self.table.take_state_batch()?)?;
- reservation
- .try_resize(self.table.memory_size())
- .map_err(|e| {
- e.context(
- "Decreasing allocation after spilling should succeed",
- )
- })?;
- continue;
}
Err(e) => return Err(e),
- }
+ };
if self.spill_context.as_ref().is_some_and(|s| s.has_spills()) {
// Spilled groups may recur, so all remaining states must go
@@ -364,7 +372,10 @@ impl Aggregating {
continue;
}
let Some(batch) = self.table.take_completed_result_batch()? else {
- continue;
+ match oom {
+ Some(oom) => return Err(oom),
+ None => continue,
+ }
};
timer.done();
return Ok(Some(ExecutionStage::Outputting(Outputting {
@@ -460,6 +471,11 @@ mod tests {
use crate::ExecutionPlan;
use crate::aggregates::GroupClusteringMode;
use crate::aggregates::PhysicalGroupBy;
+ use crate::aggregates::clustered_test_util::{
+ assert_completed_groups_emitted_under_limit,
+ assert_resources_exhausted_under_limit, full_batches, held_memory,
+ single_key_batches, sum_over_sorted_input,
+ };
use crate::common::collect;
use crate::test::TestMemoryExec;
use arrow::array::{Int64Array, StringViewArray};
@@ -732,4 +748,21 @@ mod tests {
assert_eq!(pool.reserved(), 0);
Ok(())
}
+
+ #[tokio::test]
+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
+ let plan = sum_over_sorted_input(AggregateMode::Final, &["k"], full_batches())?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_completed_groups_emitted_under_limit(&plan).await
+ }
+
+ /// A single key never completes a group before the input ends. The limit
+ /// fits only the initial table.
+ #[tokio::test]
+ async fn memory_pressure_without_completed_groups() -> Result<()> {
+ let plan =
+ sum_over_sorted_input(AggregateMode::Final, &["k"], single_key_batches())?;
+ let limit = held_memory(&plan).await?;
+ assert_resources_exhausted_under_limit(&plan, limit).await
+ }
}
diff --git a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
index f60f7ed30c..17ea53e29d 100644
--- a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs
@@ -422,78 +422,15 @@ impl Outputting {
#[cfg(test)]
mod tests {
use super::*;
- use crate::aggregates::PhysicalGroupBy;
- use crate::test::TestMemoryExec;
- use arrow::array::Int64Array;
- use arrow::datatypes::{DataType, Field, Schema};
- use datafusion_common::test_util::batches_to_string;
- use datafusion_execution::memory_pool::{
- GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
+ use crate::aggregates::GroupClusteringMode;
+ use crate::aggregates::clustered_test_util::{
+ assert_completed_groups_emitted_under_limit, full_batches, sum_over_sorted_input,
};
- use datafusion_execution::runtime_env::RuntimeEnvBuilder;
- use datafusion_functions_aggregate::sum::sum_udaf;
- use datafusion_physical_expr::PhysicalSortExpr;
- use datafusion_physical_expr::aggregate::AggregateExprBuilder;
- use datafusion_physical_expr::expressions::col;
- use datafusion_physical_expr_common::sort_expr::LexOrdering;
- /// Runs a partial `SUM(k) GROUP BY k` over two batches sorted on `k` that
- /// share a boundary key. Returns the output and the most memory the stream
- /// held between batches.
- async fn run(pool: Arc<dyn MemoryPool>) -> Result<(Vec<RecordBatch>, usize)> {
- let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)]));
- let batches = [0..=256, 256..=512]
- .into_iter()
- .map(|keys| {
- let k = Int64Array::from_iter_values(keys);
- RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k)])
- })
- .collect::<Result<Vec<_>, _>>()?;
- let input = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None)?
- .try_with_sort_information(vec![
- LexOrdering::new([PhysicalSortExpr::new_default(col("k", &schema)?)])
- .unwrap(),
- ])?;
- let aggregate = AggregateExec::try_new(
- AggregateMode::Partial,
- PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".into())]),
- vec![Arc::new(
- AggregateExprBuilder::new(sum_udaf(), vec![col("k", &schema)?])
- .schema(Arc::clone(&schema))
- .alias("sum")
- .build()?,
- )],
- vec![None],
- Arc::new(input),
- Arc::clone(&schema),
- )?;
- let context = Arc::new(
- TaskContext::default().with_runtime(
- RuntimeEnvBuilder::new()
- .with_memory_pool(Arc::clone(&pool))
- .build_arc()?,
- ),
- );
- let mut stream =
- ClusteredPartialAggregateStream::new(&aggregate, &context, 0)?.into_stream();
- // Each output batch is a single slice, so the stream releases it before
- // handing it off and holds only the table.
- let mut held = pool.reserved();
- let mut output = vec![];
- while let Some(batch) = stream.next().await.transpose()? {
- held = held.max(pool.reserved());
- output.push(batch);
- }
- Ok((output, held))
- }
-
- /// The limit fits the table with only the active group, but not the table
- /// before the completed groups are emitted. The result must not change.
#[tokio::test]
- async fn completed_groups_under_memory_pressure() -> Result<()> {
- let (expected, held) = run(Arc::new(UnboundedMemoryPool::default())).await?;
- let (actual, _) = run(Arc::new(GreedyMemoryPool::new(held))).await?;
- assert_eq!(batches_to_string(&actual), batches_to_string(&expected));
- Ok(())
+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
+ let plan = sum_over_sorted_input(AggregateMode::Partial, &["k"], full_batches())?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_completed_groups_emitted_under_limit(&plan).await
}
}
diff --git a/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs
index 7095921ac7..c83f0f3887 100644
--- a/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs
@@ -82,8 +82,9 @@ use crate::{RecordBatchStream, SendableRecordBatchStream};
/// only the current group remains active after completed groups are emitted, and
/// memory usage does not grow with the total number of groups.
///
-/// If a memory reservation nevertheless fails, the stream returns the error
-/// directly, indicating an unexpected behavior.
+/// If the reservation after an input batch fails, `ClusteredSingleAggregateStream`
+/// still emits the completed groups, and returns the error only when no group is
+/// complete.
///
/// ## Partial group clustering
///
@@ -329,30 +330,35 @@ impl ClusteredSingleAggregateStream {
spill_context.as_deref(),
));
timer.done();
- match resize_result {
- Ok(()) => {}
+ // Check the reservation before taking completed groups, as the partial
+ // stream does, so spilling is unchanged. Without spilling, the refusal
+ // is returned below only if no group is complete.
+ let oom = match resize_result {
+ Ok(()) => None,
Err(e @ DataFusionError::ResourcesExhausted(_)) => {
- let Some(spill_context) = spill_context else {
- // `None` means spilling is not supported, see comments
- // at `ClusteredSingleAggregateState` for details.
- return Self::break_with_err(e);
- };
- if table.is_empty() {
- return Self::break_with_internal_err(
- "Clustered single aggregate ran out of memory with no aggregated groups",
- );
+ match spill_context {
+ Some(spill_context) => {
+ if table.is_empty() {
+ return Self::break_with_internal_err(
+ "Clustered single aggregate ran out of memory with no aggregated groups",
+ );
+ }
+ return ControlFlow::Continue(
+ ClusteredSingleAggregateState::Spilling {
+ table,
+ spill_context,
+ },
+ );
+ }
+ // Spilling is not supported, see comments at
+ // `ClusteredSingleAggregateState` for details.
+ None => Some(e),
}
- return ControlFlow::Continue(
- ClusteredSingleAggregateState::Spilling {
- table,
- spill_context,
- },
- );
}
Err(e) => {
return Self::break_with_err(e);
}
- }
+ };
let result = if spill_context
.as_ref()
@@ -368,10 +374,10 @@ impl ClusteredSingleAggregateStream {
result
};
- match result {
+ match (result, oom) {
// Some finalized groups can be emitted. Yield them in
// slices, then continue aggregating input.
- Ok(Some(batch)) => {
+ (Ok(Some(batch)), _) => {
let table_memory = Self::reservation_size_for_table(
&table,
spill_context.as_deref(),
@@ -385,14 +391,15 @@ impl ClusteredSingleAggregateStream {
},
)
}
+ (Ok(None), Some(oom)) => Self::break_with_err(oom),
// Can't do early emit, continue aggregating.
- Ok(None) => ControlFlow::Continue(
+ (Ok(None), None) => ControlFlow::Continue(
ClusteredSingleAggregateState::ReadingInput {
table,
spill_context,
},
),
- Err(e) => Self::break_with_err(e),
+ (Err(e), _) => Self::break_with_err(e),
}
}
Poll::Ready(Some(Err(e))) => Self::break_with_err(e),
@@ -762,6 +769,11 @@ impl RecordBatchStream for ClusteredSingleAggregateStream {
mod tests {
use super::*;
use crate::aggregates::PhysicalGroupBy;
+ use crate::aggregates::clustered_test_util::{
+ assert_completed_groups_emitted_under_limit,
+ assert_resources_exhausted_under_limit, full_batches, held_memory,
+ single_key_batches, sum_over_sorted_input,
+ };
use crate::collect;
use crate::test::TestMemoryExec;
use arrow::array::Int64Array;
@@ -851,4 +863,21 @@ mod tests {
);
Ok(())
}
+
+ #[tokio::test]
+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
+ let plan = sum_over_sorted_input(AggregateMode::Single, &["k"], full_batches())?;
+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
+ assert_completed_groups_emitted_under_limit(&plan).await
+ }
+
+ /// A single key never completes a group before the input ends. The limit
+ /// fits only the initial table.
+ #[tokio::test]
+ async fn memory_pressure_without_completed_groups() -> Result<()> {
+ let plan =
+ sum_over_sorted_input(AggregateMode::Single, &["k"], single_key_batches())?;
+ let limit = held_memory(&plan).await?;
+ assert_resources_exhausted_under_limit(&plan, limit).await
+ }
}
diff --git a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs
new file mode 100644
index 0000000000..ad2006bd9c
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs
@@ -0,0 +1,197 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Test helpers for the clustered aggregate streams under memory pressure.
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int64Array};
+use arrow::datatypes::{DataType, Field, Schema};
+use arrow::record_batch::RecordBatch;
+use datafusion_common::test_util::batches_to_string;
+use datafusion_common::{DataFusionError, Result};
+use datafusion_execution::TaskContext;
+use datafusion_execution::config::SessionConfig;
+use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
+use datafusion_execution::memory_pool::{
+ GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
+};
+use datafusion_execution::runtime_env::RuntimeEnvBuilder;
+use datafusion_functions_aggregate::sum::sum_udaf;
+use datafusion_physical_expr::PhysicalSortExpr;
+use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+use datafusion_physical_expr::expressions::col;
+use datafusion_physical_expr_common::sort_expr::LexOrdering;
+use futures::StreamExt;
+
+use super::{AggregateExec, AggregateMode, PhysicalGroupBy};
+use crate::ExecutionPlan;
+use crate::test::TestMemoryExec;
+
+/// Key columns of two batches sorted on `k` that share the boundary key 256, so
+/// groups complete within each batch under full group clustering.
+pub(super) fn full_batches() -> Vec<Vec<Vec<i64>>> {
+ vec![vec![(0..=256).collect()], vec![(256..=512).collect()]]
+}
+
+/// Key column of two batches that hold the same single key, so no group
+/// completes before the input ends.
+pub(super) fn single_key_batches() -> Vec<Vec<Vec<i64>>> {
+ vec![vec![vec![0; 512]]; 2]
+}
+
+/// Builds `SUM(v) GROUP BY <keys>` in `mode` over `batches` of `(keys..., v)`
+/// columns, where `v` equals the last key. The input is declared sorted on the
+/// first key only, so one key gives full group clustering and more keys give
+/// partial group clustering.
+///
+/// For `Final` mode, the batches are the partial states, since the state of
+/// `SUM(v)` is a single `Int64` column.
+pub(super) fn sum_over_sorted_input(
+ mode: AggregateMode,
+ keys: &[&str],
+ batches: Vec<Vec<Vec<i64>>>,
+) -> Result<AggregateExec> {
+ let fields = keys
+ .iter()
+ .chain(&["v"])
+ .map(|name| Field::new(*name, DataType::Int64, false))
+ .collect::<Vec<_>>();
+ let schema = Arc::new(Schema::new(fields));
+ let group_by = PhysicalGroupBy::new_single(
+ keys.iter()
+ .map(|key| Ok((col(key, &schema)?, key.to_string())))
+ .collect::<Result<_>>()?,
+ );
+ let aggregate_exprs = vec![Arc::new(
+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("sum")
+ .build()?,
+ )];
+
+ let (input_schema, group_by) = if mode == AggregateMode::Final {
+ let empty = TestMemoryExec::try_new_exec(&[vec![]], Arc::clone(&schema), None)?;
+ let partial = AggregateExec::try_new(
+ AggregateMode::Partial,
+ group_by.clone(),
+ aggregate_exprs.clone(),
+ vec![None],
+ empty,
+ Arc::clone(&schema),
+ )?;
+ (partial.schema(), group_by.as_final())
+ } else {
+ (Arc::clone(&schema), group_by)
+ };
+
+ let batches = batches
+ .into_iter()
+ .map(|columns| {
+ let mut columns = columns
+ .into_iter()
+ .map(|values| Arc::new(Int64Array::from(values)) as ArrayRef)
+ .collect::<Vec<_>>();
+ columns.push(Arc::clone(columns.last().unwrap()));
+ RecordBatch::try_new(Arc::clone(&input_schema), columns)
+ })
+ .collect::<Result<Vec<_>, _>>()?;
+ let ordering =
+ LexOrdering::new([PhysicalSortExpr::new_default(col(keys[0], &input_schema)?)])
+ .unwrap();
+ let input = TestMemoryExec::try_new(&[batches], Arc::clone(&input_schema), None)?
+ .try_with_sort_information(vec![ordering])?;
+ AggregateExec::try_new(
+ mode,
+ group_by,
+ aggregate_exprs,
+ vec![None],
+ Arc::new(TestMemoryExec::update_cache(&Arc::new(input))),
+ schema,
+ )
+}
+
+/// Executes partition 0 of `plan` on the clustered aggregate path, with `pool`
+/// and temporary files disabled. Returns the output and the most memory the
+/// plan held when it handed off a batch, including its initial reservation.
+async fn collect_with_held_memory(
+ plan: &AggregateExec,
+ pool: Arc<dyn MemoryPool>,
+) -> Result<(Vec<RecordBatch>, usize)> {
+ let context = Arc::new(
+ TaskContext::default()
+ .with_session_config(
+ SessionConfig::new()
+ .set_bool("datafusion.execution.enable_migration_aggregate", true),
+ )
+ .with_runtime(
+ RuntimeEnvBuilder::new()
+ .with_memory_pool(Arc::clone(&pool))
+ .with_disk_manager_builder(
+ DiskManagerBuilder::default()
+ .with_mode(DiskManagerMode::Disabled),
+ )
+ .build_arc()?,
+ ),
+ );
+ let mut stream = plan.execute(0, context)?;
+ // Each output batch is a single slice, so the stream releases it before
+ // handing it off and holds only the table.
+ let mut held = pool.reserved();
+ let mut output = vec![];
+ while let Some(batch) = stream.next().await.transpose()? {
+ held = held.max(pool.reserved());
+ output.push(batch);
+ }
+ Ok((output, held))
+}
+
+/// Returns the memory `plan` holds when handing off output without a limit.
+pub(super) async fn held_memory(plan: &AggregateExec) -> Result<usize> {
+ let (_, held) =
+ collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;
+ Ok(held)
+}
+
+/// Runs `plan` without a memory limit, then with a limit that fits the table
+/// with only the active groups, but not the table before completed groups are
+/// emitted. The result must not change.
+pub(super) async fn assert_completed_groups_emitted_under_limit(
+ plan: &AggregateExec,
+) -> Result<()> {
+ let (expected, held) =
+ collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;
+ let (actual, _) =
+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(held))).await?;
+ assert_eq!(batches_to_string(&actual), batches_to_string(&expected));
+ Ok(())
+}
+
+/// Runs `plan` under `limit`, where emitting completed groups cannot bring the
+/// table under the limit, so the refused reservation must be returned.
+pub(super) async fn assert_resources_exhausted_under_limit(
+ plan: &AggregateExec,
+ limit: usize,
+) -> Result<()> {
+ let result =
+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(limit))).await;
+ assert!(
+ matches!(result, Err(DataFusionError::ResourcesExhausted(_))),
+ "expected ResourcesExhausted, got {result:?}"
+ );
+ Ok(())
+}
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index e62cf2972a..f2b5e70f80 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -216,6 +216,8 @@ mod aggregate_stream;
mod clustered_final_stream;
mod clustered_partial_stream;
mod clustered_single_stream;
+#[cfg(test)]
+mod clustered_test_util;
pub mod group_values;
mod grouped_hash_stream;
mod grouped_topk_stream;