Apache DataFusion SQL Query Engine
fix: emit completed groups when fully sorted final and single aggrega… (#26637)
Yash Aggarwalad160042e784
diff --git a/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_final_stream.rsindex 9c71c5ee13..b5890e8891 100644--- a/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs+++ b/datafusion/physical-plan/src/aggregates/clustered_final_stream.rs@@ -261,7 +261,9 @@ impl ClusteredFinalAggregateStream {/// 1. Start./// 2. Merge one input batch:/// - If memory fits and no groups are complete, continue reading input.- /// - If OOM, spill.+ /// - If OOM with a spill context, spill.+ /// - If OOM without one, emit the completed groups (edge 3), or return+ /// the error when no group is complete./// 3. Prepare output:/// - Before any spill, a prefix of groups is complete: materialize the/// entire prefix once, retaining the input and active groups to resume@@ -336,27 +338,33 @@ impl Aggregating {let timer = elapsed_compute.timer();self.table.aggregate_batch(&batch)?;- match reservation.try_resize(self.reservation_size()) {- Ok(()) => {}+ // Check the reservation before taking completed groups, as the partial+ // stream does, so spilling is unchanged. Without spilling, the refusal+ // is returned below only if no group is complete.+ let oom = match reservation.try_resize(self.reservation_size()) {+ Ok(()) => None,Err(oom @ DataFusionError::ResourcesExhausted(_)) => {- let Some(spill_context) = self.spill_context.as_mut() else {- return Err(oom);- };- if self.table.is_empty() {- return Err(oom);+ match self.spill_context.as_mut() {+ None => Some(oom),+ Some(spill_context) => {+ if self.table.is_empty() {+ return Err(oom);+ }+ spill_context+ .sort_and_spill(self.table.take_state_batch()?)?;+ reservation.try_resize(self.table.memory_size()).map_err(+ |e| {+ e.context(+ "Decreasing allocation after spilling should succeed",+ )+ },+ )?;+ continue;+ }}- spill_context.sort_and_spill(self.table.take_state_batch()?)?;- reservation- .try_resize(self.table.memory_size())- .map_err(|e| {- e.context(- "Decreasing allocation after spilling should succeed",- )- })?;- continue;}Err(e) => return Err(e),- }+ };if self.spill_context.as_ref().is_some_and(|s| s.has_spills()) {// Spilled groups may recur, so all remaining states must go@@ -364,7 +372,10 @@ impl Aggregating {continue;}let Some(batch) = self.table.take_completed_result_batch()? else {- continue;+ match oom {+ Some(oom) => return Err(oom),+ None => continue,+ }};timer.done();return Ok(Some(ExecutionStage::Outputting(Outputting {@@ -460,6 +471,11 @@ mod tests {use crate::ExecutionPlan;use crate::aggregates::GroupClusteringMode;use crate::aggregates::PhysicalGroupBy;+ use crate::aggregates::clustered_test_util::{+ assert_completed_groups_emitted_under_limit,+ assert_resources_exhausted_under_limit, full_batches, held_memory,+ single_key_batches, sum_over_sorted_input,+ };use crate::common::collect;use crate::test::TestMemoryExec;use arrow::array::{Int64Array, StringViewArray};@@ -732,4 +748,21 @@ mod tests {assert_eq!(pool.reserved(), 0);Ok(())}++ #[tokio::test]+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {+ let plan = sum_over_sorted_input(AggregateMode::Final, &["k"], full_batches())?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_completed_groups_emitted_under_limit(&plan).await+ }++ /// A single key never completes a group before the input ends. The limit+ /// fits only the initial table.+ #[tokio::test]+ async fn memory_pressure_without_completed_groups() -> Result<()> {+ let plan =+ sum_over_sorted_input(AggregateMode::Final, &["k"], single_key_batches())?;+ let limit = held_memory(&plan).await?;+ assert_resources_exhausted_under_limit(&plan, limit).await+ }}diff --git a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rsindex f60f7ed30c..17ea53e29d 100644--- a/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs+++ b/datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs@@ -422,78 +422,15 @@ impl Outputting {#[cfg(test)]mod tests {use super::*;- use crate::aggregates::PhysicalGroupBy;- use crate::test::TestMemoryExec;- use arrow::array::Int64Array;- use arrow::datatypes::{DataType, Field, Schema};- use datafusion_common::test_util::batches_to_string;- use datafusion_execution::memory_pool::{- GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,+ use crate::aggregates::GroupClusteringMode;+ use crate::aggregates::clustered_test_util::{+ assert_completed_groups_emitted_under_limit, full_batches, sum_over_sorted_input,};- use datafusion_execution::runtime_env::RuntimeEnvBuilder;- use datafusion_functions_aggregate::sum::sum_udaf;- use datafusion_physical_expr::PhysicalSortExpr;- use datafusion_physical_expr::aggregate::AggregateExprBuilder;- use datafusion_physical_expr::expressions::col;- use datafusion_physical_expr_common::sort_expr::LexOrdering;- /// Runs a partial `SUM(k) GROUP BY k` over two batches sorted on `k` that- /// share a boundary key. Returns the output and the most memory the stream- /// held between batches.- async fn run(pool: Arc<dyn MemoryPool>) -> Result<(Vec<RecordBatch>, usize)> {- let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, false)]));- let batches = [0..=256, 256..=512]- .into_iter()- .map(|keys| {- let k = Int64Array::from_iter_values(keys);- RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(k)])- })- .collect::<Result<Vec<_>, _>>()?;- let input = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None)?- .try_with_sort_information(vec![- LexOrdering::new([PhysicalSortExpr::new_default(col("k", &schema)?)])- .unwrap(),- ])?;- let aggregate = AggregateExec::try_new(- AggregateMode::Partial,- PhysicalGroupBy::new_single(vec![(col("k", &schema)?, "k".into())]),- vec![Arc::new(- AggregateExprBuilder::new(sum_udaf(), vec![col("k", &schema)?])- .schema(Arc::clone(&schema))- .alias("sum")- .build()?,- )],- vec![None],- Arc::new(input),- Arc::clone(&schema),- )?;- let context = Arc::new(- TaskContext::default().with_runtime(- RuntimeEnvBuilder::new()- .with_memory_pool(Arc::clone(&pool))- .build_arc()?,- ),- );- let mut stream =- ClusteredPartialAggregateStream::new(&aggregate, &context, 0)?.into_stream();- // Each output batch is a single slice, so the stream releases it before- // handing it off and holds only the table.- let mut held = pool.reserved();- let mut output = vec![];- while let Some(batch) = stream.next().await.transpose()? {- held = held.max(pool.reserved());- output.push(batch);- }- Ok((output, held))- }-- /// The limit fits the table with only the active group, but not the table- /// before the completed groups are emitted. The result must not change.#[tokio::test]- async fn completed_groups_under_memory_pressure() -> Result<()> {- let (expected, held) = run(Arc::new(UnboundedMemoryPool::default())).await?;- let (actual, _) = run(Arc::new(GreedyMemoryPool::new(held))).await?;- assert_eq!(batches_to_string(&actual), batches_to_string(&expected));- Ok(())+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {+ let plan = sum_over_sorted_input(AggregateMode::Partial, &["k"], full_batches())?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_completed_groups_emitted_under_limit(&plan).await}}diff --git a/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs b/datafusion/physical-plan/src/aggregates/clustered_single_stream.rsindex 7095921ac7..c83f0f3887 100644--- a/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs+++ b/datafusion/physical-plan/src/aggregates/clustered_single_stream.rs@@ -82,8 +82,9 @@ use crate::{RecordBatchStream, SendableRecordBatchStream};/// only the current group remains active after completed groups are emitted, and/// memory usage does not grow with the total number of groups.///-/// If a memory reservation nevertheless fails, the stream returns the error-/// directly, indicating an unexpected behavior.+/// If the reservation after an input batch fails, `ClusteredSingleAggregateStream`+/// still emits the completed groups, and returns the error only when no group is+/// complete.////// ## Partial group clustering///@@ -329,30 +330,35 @@ impl ClusteredSingleAggregateStream {spill_context.as_deref(),));timer.done();- match resize_result {- Ok(()) => {}+ // Check the reservation before taking completed groups, as the partial+ // stream does, so spilling is unchanged. Without spilling, the refusal+ // is returned below only if no group is complete.+ let oom = match resize_result {+ Ok(()) => None,Err(e @ DataFusionError::ResourcesExhausted(_)) => {- let Some(spill_context) = spill_context else {- // `None` means spilling is not supported, see comments- // at `ClusteredSingleAggregateState` for details.- return Self::break_with_err(e);- };- if table.is_empty() {- return Self::break_with_internal_err(- "Clustered single aggregate ran out of memory with no aggregated groups",- );+ match spill_context {+ Some(spill_context) => {+ if table.is_empty() {+ return Self::break_with_internal_err(+ "Clustered single aggregate ran out of memory with no aggregated groups",+ );+ }+ return ControlFlow::Continue(+ ClusteredSingleAggregateState::Spilling {+ table,+ spill_context,+ },+ );+ }+ // Spilling is not supported, see comments at+ // `ClusteredSingleAggregateState` for details.+ None => Some(e),}- return ControlFlow::Continue(- ClusteredSingleAggregateState::Spilling {- table,- spill_context,- },- );}Err(e) => {return Self::break_with_err(e);}- }+ };let result = if spill_context.as_ref()@@ -368,10 +374,10 @@ impl ClusteredSingleAggregateStream {result};- match result {+ match (result, oom) {// Some finalized groups can be emitted. Yield them in// slices, then continue aggregating input.- Ok(Some(batch)) => {+ (Ok(Some(batch)), _) => {let table_memory = Self::reservation_size_for_table(&table,spill_context.as_deref(),@@ -385,14 +391,15 @@ impl ClusteredSingleAggregateStream {},)}+ (Ok(None), Some(oom)) => Self::break_with_err(oom),// Can't do early emit, continue aggregating.- Ok(None) => ControlFlow::Continue(+ (Ok(None), None) => ControlFlow::Continue(ClusteredSingleAggregateState::ReadingInput {table,spill_context,},),- Err(e) => Self::break_with_err(e),+ (Err(e), _) => Self::break_with_err(e),}}Poll::Ready(Some(Err(e))) => Self::break_with_err(e),@@ -762,6 +769,11 @@ impl RecordBatchStream for ClusteredSingleAggregateStream {mod tests {use super::*;use crate::aggregates::PhysicalGroupBy;+ use crate::aggregates::clustered_test_util::{+ assert_completed_groups_emitted_under_limit,+ assert_resources_exhausted_under_limit, full_batches, held_memory,+ single_key_batches, sum_over_sorted_input,+ };use crate::collect;use crate::test::TestMemoryExec;use arrow::array::Int64Array;@@ -851,4 +863,21 @@ mod tests {);Ok(())}++ #[tokio::test]+ async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {+ let plan = sum_over_sorted_input(AggregateMode::Single, &["k"], full_batches())?;+ assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);+ assert_completed_groups_emitted_under_limit(&plan).await+ }++ /// A single key never completes a group before the input ends. The limit+ /// fits only the initial table.+ #[tokio::test]+ async fn memory_pressure_without_completed_groups() -> Result<()> {+ let plan =+ sum_over_sorted_input(AggregateMode::Single, &["k"], single_key_batches())?;+ let limit = held_memory(&plan).await?;+ assert_resources_exhausted_under_limit(&plan, limit).await+ }}diff --git a/datafusion/physical-plan/src/aggregates/clustered_test_util.rs b/datafusion/physical-plan/src/aggregates/clustered_test_util.rsnew file mode 100644index 0000000000..ad2006bd9c--- /dev/null+++ b/datafusion/physical-plan/src/aggregates/clustered_test_util.rs@@ -0,0 +1,197 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Test helpers for the clustered aggregate streams under memory pressure.++use std::sync::Arc;++use arrow::array::{ArrayRef, Int64Array};+use arrow::datatypes::{DataType, Field, Schema};+use arrow::record_batch::RecordBatch;+use datafusion_common::test_util::batches_to_string;+use datafusion_common::{DataFusionError, Result};+use datafusion_execution::TaskContext;+use datafusion_execution::config::SessionConfig;+use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};+use datafusion_execution::memory_pool::{+ GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,+};+use datafusion_execution::runtime_env::RuntimeEnvBuilder;+use datafusion_functions_aggregate::sum::sum_udaf;+use datafusion_physical_expr::PhysicalSortExpr;+use datafusion_physical_expr::aggregate::AggregateExprBuilder;+use datafusion_physical_expr::expressions::col;+use datafusion_physical_expr_common::sort_expr::LexOrdering;+use futures::StreamExt;++use super::{AggregateExec, AggregateMode, PhysicalGroupBy};+use crate::ExecutionPlan;+use crate::test::TestMemoryExec;++/// Key columns of two batches sorted on `k` that share the boundary key 256, so+/// groups complete within each batch under full group clustering.+pub(super) fn full_batches() -> Vec<Vec<Vec<i64>>> {+ vec![vec![(0..=256).collect()], vec![(256..=512).collect()]]+}++/// Key column of two batches that hold the same single key, so no group+/// completes before the input ends.+pub(super) fn single_key_batches() -> Vec<Vec<Vec<i64>>> {+ vec![vec![vec![0; 512]]; 2]+}++/// Builds `SUM(v) GROUP BY <keys>` in `mode` over `batches` of `(keys..., v)`+/// columns, where `v` equals the last key. The input is declared sorted on the+/// first key only, so one key gives full group clustering and more keys give+/// partial group clustering.+///+/// For `Final` mode, the batches are the partial states, since the state of+/// `SUM(v)` is a single `Int64` column.+pub(super) fn sum_over_sorted_input(+ mode: AggregateMode,+ keys: &[&str],+ batches: Vec<Vec<Vec<i64>>>,+) -> Result<AggregateExec> {+ let fields = keys+ .iter()+ .chain(&["v"])+ .map(|name| Field::new(*name, DataType::Int64, false))+ .collect::<Vec<_>>();+ let schema = Arc::new(Schema::new(fields));+ let group_by = PhysicalGroupBy::new_single(+ keys.iter()+ .map(|key| Ok((col(key, &schema)?, key.to_string())))+ .collect::<Result<_>>()?,+ );+ let aggregate_exprs = vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("sum")+ .build()?,+ )];++ let (input_schema, group_by) = if mode == AggregateMode::Final {+ let empty = TestMemoryExec::try_new_exec(&[vec![]], Arc::clone(&schema), None)?;+ let partial = AggregateExec::try_new(+ AggregateMode::Partial,+ group_by.clone(),+ aggregate_exprs.clone(),+ vec![None],+ empty,+ Arc::clone(&schema),+ )?;+ (partial.schema(), group_by.as_final())+ } else {+ (Arc::clone(&schema), group_by)+ };++ let batches = batches+ .into_iter()+ .map(|columns| {+ let mut columns = columns+ .into_iter()+ .map(|values| Arc::new(Int64Array::from(values)) as ArrayRef)+ .collect::<Vec<_>>();+ columns.push(Arc::clone(columns.last().unwrap()));+ RecordBatch::try_new(Arc::clone(&input_schema), columns)+ })+ .collect::<Result<Vec<_>, _>>()?;+ let ordering =+ LexOrdering::new([PhysicalSortExpr::new_default(col(keys[0], &input_schema)?)])+ .unwrap();+ let input = TestMemoryExec::try_new(&[batches], Arc::clone(&input_schema), None)?+ .try_with_sort_information(vec![ordering])?;+ AggregateExec::try_new(+ mode,+ group_by,+ aggregate_exprs,+ vec![None],+ Arc::new(TestMemoryExec::update_cache(&Arc::new(input))),+ schema,+ )+}++/// Executes partition 0 of `plan` on the clustered aggregate path, with `pool`+/// and temporary files disabled. Returns the output and the most memory the+/// plan held when it handed off a batch, including its initial reservation.+async fn collect_with_held_memory(+ plan: &AggregateExec,+ pool: Arc<dyn MemoryPool>,+) -> Result<(Vec<RecordBatch>, usize)> {+ let context = Arc::new(+ TaskContext::default()+ .with_session_config(+ SessionConfig::new()+ .set_bool("datafusion.execution.enable_migration_aggregate", true),+ )+ .with_runtime(+ RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::clone(&pool))+ .with_disk_manager_builder(+ DiskManagerBuilder::default()+ .with_mode(DiskManagerMode::Disabled),+ )+ .build_arc()?,+ ),+ );+ let mut stream = plan.execute(0, context)?;+ // Each output batch is a single slice, so the stream releases it before+ // handing it off and holds only the table.+ let mut held = pool.reserved();+ let mut output = vec![];+ while let Some(batch) = stream.next().await.transpose()? {+ held = held.max(pool.reserved());+ output.push(batch);+ }+ Ok((output, held))+}++/// Returns the memory `plan` holds when handing off output without a limit.+pub(super) async fn held_memory(plan: &AggregateExec) -> Result<usize> {+ let (_, held) =+ collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;+ Ok(held)+}++/// Runs `plan` without a memory limit, then with a limit that fits the table+/// with only the active groups, but not the table before completed groups are+/// emitted. The result must not change.+pub(super) async fn assert_completed_groups_emitted_under_limit(+ plan: &AggregateExec,+) -> Result<()> {+ let (expected, held) =+ collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;+ let (actual, _) =+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(held))).await?;+ assert_eq!(batches_to_string(&actual), batches_to_string(&expected));+ Ok(())+}++/// Runs `plan` under `limit`, where emitting completed groups cannot bring the+/// table under the limit, so the refused reservation must be returned.+pub(super) async fn assert_resources_exhausted_under_limit(+ plan: &AggregateExec,+ limit: usize,+) -> Result<()> {+ let result =+ collect_with_held_memory(plan, Arc::new(GreedyMemoryPool::new(limit))).await;+ assert!(+ matches!(result, Err(DataFusionError::ResourcesExhausted(_))),+ "expected ResourcesExhausted, got {result:?}"+ );+ Ok(())+}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex e62cf2972a..f2b5e70f80 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -216,6 +216,8 @@ mod aggregate_stream;mod clustered_final_stream;mod clustered_partial_stream;mod clustered_single_stream;+#[cfg(test)]+mod clustered_test_util;pub mod group_values;mod grouped_hash_stream;mod grouped_topk_stream;