Apache DataFusion SQL Query Engine
Implement disk spilling for all grouping ordering modes in GroupedHashAggregateStream (#19287)
Pepijn Van Eeckhoudteb30c19b303e
diff --git a/datafusion/physical-expr-common/src/sort_expr.rs b/datafusion/physical-expr-common/src/sort_expr.rsindex e8558c7643..db30dd6ed2 100644--- a/datafusion/physical-expr-common/src/sort_expr.rs+++ b/datafusion/physical-expr-common/src/sort_expr.rs@@ -457,6 +457,17 @@ impl LexOrdering {req.expr.eq(&cur.expr) && is_reversed_sort_options(&req.options, &cur.options)})}++ /// Returns the sort options for the given expression if one is defined in this `LexOrdering`.+ pub fn get_sort_options(&self, expr: &dyn PhysicalExpr) -> Option<SortOptions> {+ for e in self {+ if e.expr.as_ref().dyn_eq(expr) {+ return Some(e.options);+ }+ }++ None+ }}/// Check if two SortOptions represent reversed orderings.diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rsindex f419328d11..2f3b1a19e7 100644--- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs@@ -22,7 +22,7 @@ use arrow::array::types::{Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType,TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType,};-use arrow::array::{ArrayRef, RecordBatch, downcast_primitive};+use arrow::array::{ArrayRef, downcast_primitive};use arrow::datatypes::{DataType, SchemaRef, TimeUnit};use datafusion_common::Result;@@ -112,7 +112,7 @@ pub trait GroupValues: Send {fn emit(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>>;/// Clear the contents and shrink the capacity to the size of the batch (free up memory usage)- fn clear_shrink(&mut self, batch: &RecordBatch);+ fn clear_shrink(&mut self, num_rows: usize);}/// Return a specialized implementation of [`GroupValues`] for the given schema.diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rsindex b62bc11aff..4c9e376fc4 100644--- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs@@ -30,7 +30,7 @@ use crate::aggregates::group_values::multi_group_by::{bytes_view::ByteViewGroupValueBuilder, primitive::PrimitiveGroupValueBuilder,};use ahash::RandomState;-use arrow::array::{Array, ArrayRef, RecordBatch};+use arrow::array::{Array, ArrayRef};use arrow::compute::cast;use arrow::datatypes::{BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Float32Type,@@ -1181,14 +1181,13 @@ impl<const STREAMING: bool> GroupValues for GroupValuesColumn<STREAMING> {Ok(output)}- fn clear_shrink(&mut self, batch: &RecordBatch) {- let count = batch.num_rows();+ fn clear_shrink(&mut self, num_rows: usize) {self.group_values.clear();self.map.clear();- self.map.shrink_to(count, |_| 0); // hasher does not matter since the map is cleared+ self.map.shrink_to(num_rows, |_| 0); // hasher does not matter since the map is clearedself.map_size = self.map.capacity() * size_of::<(u64, usize)>();self.hashes_buffer.clear();- self.hashes_buffer.shrink_to(count);+ self.hashes_buffer.shrink_to(num_rows);// Such structures are only used in `non-streaming` caseif !STREAMING {diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rsindex a5e5c16006..dd794c9573 100644--- a/datafusion/physical-plan/src/aggregates/group_values/row.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs@@ -17,7 +17,7 @@use crate::aggregates::group_values::GroupValues;use ahash::RandomState;-use arrow::array::{Array, ArrayRef, ListArray, RecordBatch, StructArray};+use arrow::array::{Array, ArrayRef, ListArray, StructArray};use arrow::compute::cast;use arrow::datatypes::{DataType, SchemaRef};use arrow::row::{RowConverter, Rows, SortField};@@ -243,17 +243,16 @@ impl GroupValues for GroupValuesRows {Ok(output)}- fn clear_shrink(&mut self, batch: &RecordBatch) {- let count = batch.num_rows();+ fn clear_shrink(&mut self, num_rows: usize) {self.group_values = self.group_values.take().map(|mut rows| {rows.clear();rows});self.map.clear();- self.map.shrink_to(count, |_| 0); // hasher does not matter since the map is cleared+ self.map.shrink_to(num_rows, |_| 0); // hasher does not matter since the map is clearedself.map_size = self.map.capacity() * size_of::<(u64, usize)>();self.hashes_buffer.clear();- self.hashes_buffer.shrink_to(count);+ self.hashes_buffer.shrink_to(num_rows);}}diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/boolean.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/boolean.rsindex 44b763a91f..e993c0c53d 100644--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/boolean.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/boolean.rs@@ -19,7 +19,6 @@ use crate::aggregates::group_values::GroupValues;use arrow::array::{ArrayRef, AsArray as _, BooleanArray, BooleanBufferBuilder, NullBufferBuilder,- RecordBatch,};use datafusion_common::Result;use datafusion_expr::EmitTo;@@ -146,7 +145,7 @@ impl GroupValues for GroupValuesBoolean {Ok(vec![Arc::new(BooleanArray::new(values, nulls)) as _])}- fn clear_shrink(&mut self, _batch: &RecordBatch) {+ fn clear_shrink(&mut self, _num_rows: usize) {self.false_group = None;self.true_group = None;self.null_group = None;diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes.rsindex b901aee313..b881a51b25 100644--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes.rs@@ -19,7 +19,7 @@ use std::mem::size_of;use crate::aggregates::group_values::GroupValues;-use arrow::array::{Array, ArrayRef, OffsetSizeTrait, RecordBatch};+use arrow::array::{Array, ArrayRef, OffsetSizeTrait};use datafusion_common::Result;use datafusion_expr::EmitTo;use datafusion_physical_expr_common::binary_map::{ArrowBytesMap, OutputType};@@ -120,7 +120,7 @@ impl<O: OffsetSizeTrait> GroupValues for GroupValuesBytes<O> {Ok(vec![group_values])}- fn clear_shrink(&mut self, _batch: &RecordBatch) {+ fn clear_shrink(&mut self, _num_rows: usize) {// in theory we could potentially avoid this reallocation and clear the// contents of the maps, but for now we just reset the map from the beginningself.map.take();diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes_view.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes_view.rsindex be9a0334e3..7a56f7c52c 100644--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes_view.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/bytes_view.rs@@ -16,7 +16,7 @@// under the License.use crate::aggregates::group_values::GroupValues;-use arrow::array::{Array, ArrayRef, RecordBatch};+use arrow::array::{Array, ArrayRef};use datafusion_expr::EmitTo;use datafusion_physical_expr::binary_map::OutputType;use datafusion_physical_expr_common::binary_view_map::ArrowBytesViewMap;@@ -122,7 +122,7 @@ impl GroupValues for GroupValuesBytesView {Ok(vec![group_values])}- fn clear_shrink(&mut self, _batch: &RecordBatch) {+ fn clear_shrink(&mut self, _num_rows: usize) {// in theory we could potentially avoid this reallocation and clear the// contents of the maps, but for now we just reset the map from the beginningself.map.take();diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rsindex 41d34218f6..c46cde8786 100644--- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs+++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs@@ -23,7 +23,6 @@ use arrow::array::{cast::AsArray,};use arrow::datatypes::{DataType, i256};-use arrow::record_batch::RecordBatch;use datafusion_common::Result;use datafusion_execution::memory_pool::proxy::VecAllocExt;use datafusion_expr::EmitTo;@@ -213,11 +212,10 @@ whereOk(vec![Arc::new(array.with_data_type(self.data_type.clone()))])}- fn clear_shrink(&mut self, batch: &RecordBatch) {- let count = batch.num_rows();+ fn clear_shrink(&mut self, num_rows: usize) {self.values.clear();- self.values.shrink_to(count);+ self.values.shrink_to(num_rows);self.map.clear();- self.map.shrink_to(count, |_| 0); // hasher does not matter since the map is cleared+ self.map.shrink_to(num_rows, |_| 0); // hasher does not matter since the map is cleared}}diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex b0d432a9de..06f12a9019 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -31,7 +31,6 @@ use crate::filter_pushdown::{FilterPushdownPropagation, PushedDownPredicate,};use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};-use crate::windows::get_ordered_partition_by_indices;use crate::{DisplayFormatType, Distribution, ExecutionPlan, InputOrderMode,SendableRecordBatchStream, Statistics,@@ -613,12 +612,13 @@ impl AggregateExec {// If existing ordering satisfies a prefix of the GROUP BY expressions,// prefix requirements with this section. In this case, aggregation will// work more efficiently.- let indices = get_ordered_partition_by_indices(&groupby_exprs, &input)?;- let mut new_requirements = indices- .iter()- .map(|&idx| {- PhysicalSortRequirement::new(Arc::clone(&groupby_exprs[idx]), None)- })+ // Copy the `PhysicalSortExpr`s to retain the sort options.+ let (new_sort_exprs, indices) =+ input_eq_properties.find_longest_permutation(&groupby_exprs)?;++ let mut new_requirements = new_sort_exprs+ .into_iter()+ .map(PhysicalSortRequirement::from).collect::<Vec<_>>();let req = get_finer_aggregate_exprs_requirement(@@ -1815,7 +1815,7 @@ mod tests {use crate::test::exec::{BlockingExec, assert_strong_count_converges_to_zero};use arrow::array::{- DictionaryArray, Float32Array, Float64Array, Int32Array, StructArray,+ DictionaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StructArray,UInt32Array, UInt64Array,};use arrow::compute::{SortOptions, concat_batches};@@ -1837,6 +1837,8 @@ mod tests {use datafusion_physical_expr::expressions::Literal;use datafusion_physical_expr::expressions::lit;+ use crate::projection::ProjectionExec;+ use datafusion_physical_expr::projection::ProjectionExpr;use futures::{FutureExt, Stream};use insta::{allow_duplicates, assert_snapshot};@@ -2484,7 +2486,7 @@ mod tests {] {let n_aggr = aggregates.len();let partial_aggregate = Arc::new(AggregateExec::try_new(- AggregateMode::Partial,+ AggregateMode::Single,groups,aggregates,vec![None; n_aggr],@@ -3420,6 +3422,117 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_grouped_aggregation_respects_memory_limit() -> Result<()> {+ // test with spill+ fn create_record_batch(+ schema: &Arc<Schema>,+ data: (Vec<u32>, Vec<f64>),+ ) -> Result<RecordBatch> {+ Ok(RecordBatch::try_new(+ Arc::clone(schema),+ vec![+ Arc::new(UInt32Array::from(data.0)),+ Arc::new(Float64Array::from(data.1)),+ ],+ )?)+ }++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::UInt32, false),+ Field::new("b", DataType::Float64, false),+ ]));++ let batches = vec![+ create_record_batch(&schema, (vec![2, 3, 4, 4], vec![1.0, 2.0, 3.0, 4.0]))?,+ create_record_batch(&schema, (vec![2, 3, 4, 4], vec![1.0, 2.0, 3.0, 4.0]))?,+ ];+ let plan: Arc<dyn ExecutionPlan> =+ TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?;+ let proj = ProjectionExec::try_new(+ vec![+ ProjectionExpr::new(lit("0"), "l".to_string()),+ ProjectionExpr::new_from_expression(col("a", &schema)?, &schema)?,+ ProjectionExpr::new_from_expression(col("b", &schema)?, &schema)?,+ ],+ plan,+ )?;+ let plan: Arc<dyn ExecutionPlan> = Arc::new(proj);+ let schema = plan.schema();++ let grouping_set = PhysicalGroupBy::new(+ vec![+ (col("l", &schema)?, "l".to_string()),+ (col("a", &schema)?, "a".to_string()),+ ],+ vec![],+ vec![vec![false, false]],+ false,+ );++ // Test with MIN for simple intermediate state (min) and AVG for multiple intermediate states (partial sum, partial count).+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![+ Arc::new(+ AggregateExprBuilder::new(+ datafusion_functions_aggregate::min_max::min_udaf(),+ vec![col("b", &schema)?],+ )+ .schema(Arc::clone(&schema))+ .alias("MIN(b)")+ .build()?,+ ),+ Arc::new(+ AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("AVG(b)")+ .build()?,+ ),+ ];++ let single_aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Single,+ grouping_set,+ aggregates,+ vec![None, None],+ plan,+ Arc::clone(&schema),+ )?);++ let batch_size = 2;+ let memory_pool = Arc::new(FairSpillPool::new(2000));+ let task_ctx = Arc::new(+ TaskContext::default()+ .with_session_config(SessionConfig::new().with_batch_size(batch_size))+ .with_runtime(Arc::new(+ RuntimeEnvBuilder::new()+ .with_memory_pool(memory_pool)+ .build()?,+ )),+ );++ let result = collect(single_aggregate.execute(0, Arc::clone(&task_ctx))?).await;+ match result {+ Ok(result) => {+ assert_spill_count_metric(true, single_aggregate);++ allow_duplicates! {+ assert_snapshot!(batches_to_string(&result), @r"+ +---+---+--------+--------++ | l | a | MIN(b) | AVG(b) |+ +---+---+--------+--------++ | 0 | 2 | 1.0 | 1.0 |+ | 0 | 3 | 2.0 | 2.0 |+ | 0 | 4 | 3.0 | 3.5 |+ +---+---+--------+--------++ ");+ }+ }+ Err(e) => assert!(matches!(e, DataFusionError::ResourcesExhausted(_))),+ }++ Ok(())+ }+#[tokio::test]async fn test_aggregate_statistics_edge_cases() -> Result<()> {use crate::test::exec::StatisticsExec;@@ -3492,4 +3605,87 @@ mod tests {Ok(())}++ #[tokio::test]+ async fn test_order_is_retained_when_spilling() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, false),+ Field::new("c", DataType::Int64, false),+ ]));++ let batches = vec![vec![+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![2])),+ Arc::new(Int64Array::from(vec![2])),+ Arc::new(Int64Array::from(vec![1])),+ ],+ )?,+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![1])),+ Arc::new(Int64Array::from(vec![1])),+ Arc::new(Int64Array::from(vec![1])),+ ],+ )?,+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![0])),+ Arc::new(Int64Array::from(vec![0])),+ Arc::new(Int64Array::from(vec![1])),+ ],+ )?,+ ]];+ let scan = TestMemoryExec::try_new(&batches, Arc::clone(&schema), None)?;+ let scan = scan.try_with_sort_information(vec![+ LexOrdering::new([PhysicalSortExpr::new(+ col("b", schema.as_ref())?,+ SortOptions::default().desc(),+ )])+ .unwrap(),+ ])?;++ let aggr = Arc::new(AggregateExec::try_new(+ AggregateMode::Single,+ PhysicalGroupBy::new(+ vec![+ (col("b", schema.as_ref())?, "b".to_string()),+ (col("c", schema.as_ref())?, "c".to_string()),+ ],+ vec![],+ vec![vec![false, false]],+ false,+ ),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("c", schema.as_ref())?])+ .schema(Arc::clone(&schema))+ .alias("SUM(c)")+ .build()?,+ )],+ vec![None],+ Arc::new(scan) as Arc<dyn ExecutionPlan>,+ Arc::clone(&schema),+ )?);++ let task_ctx = new_spill_ctx(1, 600);+ let result = collect(aggr.execute(0, Arc::clone(&task_ctx))?).await?;+ assert_spill_count_metric(true, aggr);++ allow_duplicates! {+ assert_snapshot!(batches_to_string(&result), @r"+ +---+---+--------++ | b | c | SUM(c) |+ +---+---+--------++ | 2 | 1 | 1 |+ | 1 | 1 | 1 |+ | 0 | 1 | 1 |+ +---+---+--------++ ");+ }+ Ok(())+ }}diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rsindex 1e7757de4a..cb22fbf9a0 100644--- a/datafusion/physical-plan/src/aggregates/row_hash.rs+++ b/datafusion/physical-plan/src/aggregates/row_hash.rs@@ -33,7 +33,6 @@ use crate::metrics::{BaselineMetrics, MetricBuilder, RecordOutput};use crate::sorts::sort::sort_batch;use crate::sorts::streaming_merge::{SortedSpillFile, StreamingMergeBuilder};use crate::spill::spill_manager::SpillManager;-use crate::stream::RecordBatchStreamAdapter;use crate::{PhysicalExpr, aggregates, metrics};use crate::{RecordBatchStream, SendableRecordBatchStream};@@ -210,6 +209,17 @@ impl SkipAggregationProbe {}}+/// Controls the behavior when an out-of-memory condition occurs.+#[derive(PartialEq, Debug)]+enum OutOfMemoryMode {+ /// When out of memory occurs, spill state to disk+ Spill,+ /// When out of memory occurs, attempt to emit group values early+ EmitEarly,+ /// When out of memory occurs, immediately report the error+ ReportError,+}+/// HashTable based Grouping Aggregator////// # Design Goals@@ -433,6 +443,9 @@ pub(crate) struct GroupedHashAggregateStream {/// The memory reservation for this groupingreservation: MemoryReservation,+ /// The behavior to trigger when out of memory occurs+ oom_mode: OutOfMemoryMode,+/// Execution metricsbaseline_metrics: BaselineMetrics,@@ -510,12 +523,12 @@ impl GroupedHashAggregateStream {// Therefore, when we spill these intermediate states or pass them to another// aggregation operator, we must use a schema that includes both the group// columns **and** the partial-state columns.- let partial_agg_schema = create_schema(+ let spill_schema = Arc::new(create_schema(&agg.input().schema(),&agg_group_by,&aggregate_exprs,AggregateMode::Partial,- )?;+ )?);// Need to update the GROUP BY expressions to point to the correct column after schema changelet merging_group_by_expr = agg_group_by@@ -527,20 +540,25 @@ impl GroupedHashAggregateStream {}).collect();- let partial_agg_schema = Arc::new(partial_agg_schema);+ let output_ordering = agg.cache.output_ordering();- let spill_expr =+ let spill_sort_exprs =group_schema.fields.into_iter().enumerate().map(|(idx, field)| {- PhysicalSortExpr::new_default(Arc::new(Column::new(- field.name().as_str(),- idx,- )) as _)+ let output_expr = Column::new(field.name().as_str(), idx);++ // Try to use the sort options from the output ordering, if available.+ // This ensures that spilled state is sorted in the required order as well.+ let sort_options = output_ordering+ .and_then(|o| o.get_sort_options(&output_expr))+ .unwrap_or_default();++ PhysicalSortExpr::new(Arc::new(output_expr), sort_options)});- let Some(spill_expr) = LexOrdering::new(spill_expr) else {+ let Some(spill_ordering) = LexOrdering::new(spill_sort_exprs) else {return internal_err!("Spill expression is empty");};@@ -550,11 +568,35 @@ impl GroupedHashAggregateStream {.collect::<Vec<_>>().join(", ");let name = format!("GroupedHashAggregateStream[{partition}] ({agg_fn_names})");- let reservation = MemoryConsumer::new(name)- .with_can_spill(true)- .register(context.memory_pool());let group_ordering = GroupOrdering::try_new(&agg.input_order_mode)?;+ let oom_mode = match (agg.mode, &group_ordering) {+ // In partial aggregation mode, always prefer to emit incomplete results early.+ (AggregateMode::Partial, _) => OutOfMemoryMode::EmitEarly,+ // For non-partial aggregation modes, emitting incomplete results is not an option.+ // Instead, use disk spilling to store sorted, incomplete results, and merge them+ // afterwards.+ (_, GroupOrdering::None | GroupOrdering::Partial(_))+ if context.runtime_env().disk_manager.tmp_files_enabled() =>+ {+ OutOfMemoryMode::Spill+ }+ // For `GroupOrdering::Full`, the incoming stream is already sorted. This ensures the+ // number of incomplete groups can be kept small at all times. If we still hit+ // an out-of-memory condition, spilling to disk would not be beneficial since the same+ // situation is likely to reoccur when reading back the spilled data.+ // Therefore, we fall back to simply reporting the error immediately.+ // This mode will also be used if the `DiskManager` is not configured to allow spilling+ // to disk.+ _ => OutOfMemoryMode::ReportError,+ };+let group_values = new_group_values(group_schema, &group_ordering)?;+ let reservation = MemoryConsumer::new(name)+ // We interpret 'can spill' as 'can handle memory back pressure'.+ // This value needs to be set to true for the default memory pool implementations+ // to ensure fair application of back pressure amongst the memory consumers.+ .with_can_spill(oom_mode != OutOfMemoryMode::ReportError)+ .register(context.memory_pool());timer.done();let exec_state = ExecutionState::ReadingInput;@@ -562,14 +604,14 @@ impl GroupedHashAggregateStream {let spill_manager = SpillManager::new(context.runtime_env(),metrics::SpillMetrics::new(&agg.metrics, partition),- Arc::clone(&partial_agg_schema),+ Arc::clone(&spill_schema),).with_compression_type(context.session_config().spill_compression());let spill_state = SpillState {spills: vec![],- spill_expr,- spill_schema: partial_agg_schema,+ spill_expr: spill_ordering,+ spill_schema,is_stream_merging: false,merging_aggregate_arguments,merging_group_by: PhysicalGroupBy::new_single(merging_group_by_expr),@@ -627,6 +669,7 @@ impl GroupedHashAggregateStream {filter_expressions,group_by: agg_group_by,reservation,+ oom_mode,group_values,current_group_indices: Default::default(),exec_state,@@ -676,21 +719,24 @@ impl Stream for GroupedHashAggregateStream {match &self.exec_state {ExecutionState::ReadingInput => 'reading_input: {match ready!(self.input.poll_next_unpin(cx)) {- // New batch to aggregate in partial aggregation operator- Some(Ok(batch)) if self.mode == AggregateMode::Partial => {+ // New batch to aggregate+ Some(Ok(batch)) => {let timer = elapsed_compute.timer();let input_rows = batch.num_rows();- if let Some(reduction_factor) = self.reduction_factor.as_ref()+ if self.mode == AggregateMode::Partial+ && let Some(reduction_factor) =+ self.reduction_factor.as_ref(){reduction_factor.add_total(input_rows);}- // Do the grouping+ // Do the grouping.+ // `group_aggregate_batch` will _not_ have updated the memory reservation yet.+ // The rest of the code will first try to reduce memory usage by+ // already emitting results.self.group_aggregate_batch(&batch)?;- // If we can begin emitting rows, do so,- // otherwise keep consuming inputassert!(!self.input_done);// If the number of group values equals or exceeds the soft limit,@@ -702,7 +748,13 @@ impl Stream for GroupedHashAggregateStream {break 'reading_input;}- if let Some(to_emit) = self.group_ordering.emit_to() {+ // Try to emit completed groups if possible.+ // If we already started spilling, we can no longer emit since+ // this might lead to incorrect output ordering+ if (self.spill_state.spills.is_empty()+ || self.spill_state.is_stream_merging)+ && let Some(to_emit) = self.group_ordering.emit_to()+ {timer.done();if let Some(batch) = self.emit(to_emit, false)? {self.exec_state =@@ -712,18 +764,28 @@ impl Stream for GroupedHashAggregateStream {break 'reading_input;}- // Check if we should switch to skip aggregation mode- // It's important that we do this before we early emit since we've- // already updated the probe.- self.update_skip_aggregation_probe(input_rows);- if let Some(new_state) = self.switch_to_skip_aggregation()? {- timer.done();- self.exec_state = new_state;- break 'reading_input;+ if self.mode == AggregateMode::Partial {+ // Spilling should never be activated in partial aggregation mode.+ assert!(!self.spill_state.is_stream_merging);++ // Check if we should switch to skip aggregation mode+ // It's important that we do this before we early emit since we've+ // already updated the probe.+ self.update_skip_aggregation_probe(input_rows);+ if let Some(new_state) =+ self.switch_to_skip_aggregation()?+ {+ timer.done();+ self.exec_state = new_state;+ break 'reading_input;+ }}- // Check if we need to emit early due to memory pressure- if let Some(new_state) = self.emit_early_if_necessary()? {+ // If we reach this point, try to update the memory reservation+ // handling out-of-memory conditions as determined by the OOM mode.+ if let Some(new_state) =+ self.try_update_memory_reservation()?+ {timer.done();self.exec_state = new_state;break 'reading_input;@@ -732,43 +794,6 @@ impl Stream for GroupedHashAggregateStream {timer.done();}- // New batch to aggregate in terminal aggregation operator- // (Final/FinalPartitioned/Single/SinglePartitioned)- Some(Ok(batch)) => {- let timer = elapsed_compute.timer();-- // Make sure we have enough capacity for `batch`, otherwise spill- self.spill_previous_if_necessary(&batch)?;-- // Do the grouping- self.group_aggregate_batch(&batch)?;-- // If we can begin emitting rows, do so,- // otherwise keep consuming input- assert!(!self.input_done);-- // If the number of group values equals or exceeds the soft limit,- // emit all groups and switch to producing output- if self.hit_soft_group_limit() {- timer.done();- self.set_input_done_and_produce_output()?;- // make sure the exec_state just set is not overwritten below- break 'reading_input;- }-- if let Some(to_emit) = self.group_ordering.emit_to() {- timer.done();- if let Some(batch) = self.emit(to_emit, false)? {- self.exec_state =- ExecutionState::ProducingOutput(batch);- };- // make sure the exec_state just set is not overwritten below- break 'reading_input;- }-- timer.done();- }-// Found error from input streamSome(Err(e)) => {// inner had error, return to caller@@ -987,25 +1012,56 @@ impl GroupedHashAggregateStream {}}- match self.update_memory_reservation() {- // Here we can ignore `insufficient_capacity_err` because we will spill later,- // but at least one batch should fit in the memory- Err(DataFusionError::ResourcesExhausted(_))- if self.group_values.len() >= self.batch_size =>- {- Ok(())+ Ok(())+ }++ /// Attempts to update the memory reservation. If that fails due to a+ /// [DataFusionError::ResourcesExhausted] error, an attempt will be made to resolve+ /// the out-of-memory condition based on the [out-of-memory handling mode](OutOfMemoryMode).+ ///+ /// If the out-of-memory condition can not be resolved, an `Err` value will be returned+ ///+ /// Returns `Ok(Some(ExecutionState))` if the state should be changed, `Ok(None)` otherwise.+ fn try_update_memory_reservation(&mut self) -> Result<Option<ExecutionState>> {+ let oom = match self.update_memory_reservation() {+ Err(e @ DataFusionError::ResourcesExhausted(_)) => e,+ Err(e) => return Err(e),+ Ok(_) => return Ok(None),+ };++ match self.oom_mode {+ OutOfMemoryMode::Spill if !self.group_values.is_empty() => {+ self.spill()?;+ self.clear_shrink(self.batch_size);+ self.update_memory_reservation()?;+ Ok(None)}- other => other,+ OutOfMemoryMode::EmitEarly if self.group_values.len() > 1 => {+ let n = if self.group_values.len() >= self.batch_size {+ // Try to emit an integer multiple of batch size if possible+ self.group_values.len() / self.batch_size * self.batch_size+ } else {+ // Otherwise emit whatever we can+ self.group_values.len()+ };++ if let Some(batch) = self.emit(EmitTo::First(n), false)? {+ Ok(Some(ExecutionState::ProducingOutput(batch)))+ } else {+ Err(oom)+ }+ }+ _ => Err(oom),}}fn update_memory_reservation(&mut self) -> Result<()> {let acc = self.accumulators.iter().map(|x| x.size()).sum::<usize>();- let reservation_result = self.reservation.try_resize(- acc + self.group_values.size()- + self.group_ordering.size()- + self.current_group_indices.allocated_size(),- );+ let new_size = acc+ + self.group_values.size()+ + self.group_ordering.size()+ + self.current_group_indices.allocated_size();+ let reservation_result = self.reservation.try_resize(new_size);if reservation_result.is_ok() {self.spill_state@@ -1060,24 +1116,6 @@ impl GroupedHashAggregateStream {Ok(Some(batch))}- /// Optimistically, [`Self::group_aggregate_batch`] allows to exceed the memory target slightly- /// (~ 1 [`RecordBatch`]) for simplicity. In such cases, spill the data to disk and clear the- /// memory. Currently only [`GroupOrdering::None`] is supported for spilling.- fn spill_previous_if_necessary(&mut self, batch: &RecordBatch) -> Result<()> {- // TODO: support group_ordering for spilling- if !self.group_values.is_empty()- && batch.num_rows() > 0- && matches!(self.group_ordering, GroupOrdering::None)- && !self.spill_state.is_stream_merging- && self.update_memory_reservation().is_err()- {- assert_ne!(self.mode, AggregateMode::Partial);- self.spill()?;- self.clear_shrink(batch);- }- Ok(())- }-/// Emit all intermediate aggregation states, sort them, and store them on disk./// This process helps in reducing memory pressure by allowing the data to be/// read back with streaming merge.@@ -1115,72 +1153,15 @@ impl GroupedHashAggregateStream {}/// Clear memory and shirk capacities to the size of the batch.- fn clear_shrink(&mut self, batch: &RecordBatch) {- self.group_values.clear_shrink(batch);+ fn clear_shrink(&mut self, num_rows: usize) {+ self.group_values.clear_shrink(num_rows);self.current_group_indices.clear();- self.current_group_indices.shrink_to(batch.num_rows());+ self.current_group_indices.shrink_to(num_rows);}/// Clear memory and shirk capacities to zero.fn clear_all(&mut self) {- let s = self.schema();- self.clear_shrink(&RecordBatch::new_empty(s));- }-- /// Emit if the used memory exceeds the target for partial aggregation.- /// Currently only [`GroupOrdering::None`] is supported for early emitting.- /// TODO: support group_ordering for early emitting- ///- /// Returns `Some(ExecutionState)` if the state should be changed, None otherwise.- fn emit_early_if_necessary(&mut self) -> Result<Option<ExecutionState>> {- if self.group_values.len() >= self.batch_size- && matches!(self.group_ordering, GroupOrdering::None)- && self.update_memory_reservation().is_err()- {- assert_eq!(self.mode, AggregateMode::Partial);- let n = self.group_values.len() / self.batch_size * self.batch_size;- if let Some(batch) = self.emit(EmitTo::First(n), false)? {- return Ok(Some(ExecutionState::ProducingOutput(batch)));- };- }- Ok(None)- }-- /// At this point, all the inputs are read and there are some spills.- /// Emit the remaining rows and create a batch.- /// Conduct a streaming merge sort between the batch and spilled data. Since the stream is fully- /// sorted, set `self.group_ordering` to Full, then later we can read with [`EmitTo::First`].- fn update_merged_stream(&mut self) -> Result<()> {- let Some(batch) = self.emit(EmitTo::All, true)? else {- return Ok(());- };- // clear up memory for streaming_merge- self.clear_all();- self.update_memory_reservation()?;- let mut streams: Vec<SendableRecordBatchStream> = vec![];- let expr = self.spill_state.spill_expr.clone();- let schema = batch.schema();- streams.push(Box::pin(RecordBatchStreamAdapter::new(- Arc::clone(&schema),- futures::stream::once(futures::future::lazy(move |_| {- sort_batch(&batch, &expr, None)- })),- )));-- self.spill_state.is_stream_merging = true;- self.input = StreamingMergeBuilder::new()- .with_streams(streams)- .with_schema(schema)- .with_spill_manager(self.spill_state.spill_manager.clone())- .with_sorted_spill_files(std::mem::take(&mut self.spill_state.spills))- .with_expressions(&self.spill_state.spill_expr)- .with_metrics(self.baseline_metrics.clone())- .with_batch_size(self.batch_size)- .with_reservation(self.reservation.new_empty())- .build()?;- self.input_done = false;- self.group_ordering = GroupOrdering::Full(GroupOrderingFull::new());- Ok(())+ self.clear_shrink(0);}/// returns true if there is a soft groups limit and the number of distinct@@ -1192,18 +1173,60 @@ impl GroupedHashAggregateStream {group_values_soft_limit <= self.group_values.len()}- /// common function for signalling end of processing of the input stream+ /// Finalizes reading of the input stream and prepares for producing output values.+ ///+ /// This method is called both when the original input stream and,+ /// in case of disk spilling, the SPM stream have been drained.fn set_input_done_and_produce_output(&mut self) -> Result<()> {self.input_done = true;self.group_ordering.input_done();let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();let timer = elapsed_compute.timer();self.exec_state = if self.spill_state.spills.is_empty() {+ // Input has been entirely processed without spilling to disk.++ // Flush any remaining group values.let batch = self.emit(EmitTo::All, false)?;++ // If there are none, we're done; otherwise switch to emitting thembatch.map_or(ExecutionState::Done, ExecutionState::ProducingOutput)} else {- // If spill files exist, stream-merge them.- self.update_merged_stream()?;+ // Spill any remaining data to disk. There is some performance overhead in+ // writing out this last chunk of data and reading it back. The benefit of+ // doing this is that memory usage for this stream is reduced, and the more+ // sophisticated memory handling in `MultiLevelMergeBuilder` can take over+ // instead.+ // Spilling to disk and reading back also ensures batch size is consistent+ // rather than potentially having one significantly larger last batch.+ self.spill()?;++ // Mark that we're switching to stream merging mode.+ self.spill_state.is_stream_merging = true;++ self.input = StreamingMergeBuilder::new()+ .with_schema(Arc::clone(&self.spill_state.spill_schema))+ .with_spill_manager(self.spill_state.spill_manager.clone())+ .with_sorted_spill_files(std::mem::take(&mut self.spill_state.spills))+ .with_expressions(&self.spill_state.spill_expr)+ .with_metrics(self.baseline_metrics.clone())+ .with_batch_size(self.batch_size)+ .with_reservation(self.reservation.new_empty())+ .build()?;+ self.input_done = false;++ // Reset the group values collectors.+ self.clear_all();++ // We can now use `GroupOrdering::Full` since the spill files are sorted+ // on the grouping columns.+ self.group_ordering = GroupOrdering::Full(GroupOrderingFull::new());++ // Use `OutOfMemoryMode::ReportError` from this point on+ // to ensure we don't spill the spilled data to disk again.+ self.oom_mode = OutOfMemoryMode::ReportError;++ self.update_memory_reservation()?;+ExecutionState::ReadingInput};timer.done();diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex f2336920b3..c94b5a4131 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -342,6 +342,7 @@ impl TestMemoryExec {}self.sort_information = sort_information;+ self.cache = self.compute_properties();Ok(self)}