Apache DataFusion SQL Query Engine
perf: materialize completed groups once in OrderedSingleAggregateStream (#25984)
Jay Zhan7c67e4cf185d
diff --git a/datafusion/physical-plan/benches/ordered_group_values.rs b/datafusion/physical-plan/benches/ordered_group_values.rsindex ae0e44a859..30af725cdd 100644--- a/datafusion/physical-plan/benches/ordered_group_values.rs+++ b/datafusion/physical-plan/benches/ordered_group_values.rs@@ -19,7 +19,8 @@//! existing streaming hash table on identical batches. Include emission and//! cross-batch continuation, not just lookup. Input preparation is untimed.//! The physical-plan benchmark includes aggregation but excludes SQL planning,-//! sorting and I/O; the data already has a proven ordering.+//! sorting and I/O; the data already has a proven ordering. A partially+//! ordered case covers ordered prefixes that complete many groups at once.use std::hint::black_box;use std::sync::Arc;@@ -186,6 +187,8 @@ fn check_case(schema: &SchemaRef, batches: &[Vec<ArrayRef>]) -> (usize, usize) {fn aggregate_plan(schema: &SchemaRef,keys: Vec<Vec<ArrayRef>>,+ sort_columns: &[&str],+ input_order_mode: &InputOrderMode,) -> Arc<dyn ExecutionPlan> {let mut fields = schema.fields().to_vec();fields.push(Arc::new(Field::new("v", DataType::Int64, false)));@@ -197,10 +200,11 @@ fn aggregate_plan(RecordBatch::try_new(Arc::clone(&schema), cols).unwrap()}).collect::<Vec<_>>();- let ordering = LexOrdering::new(vec![- PhysicalSortExpr::new_default(col("a", &schema).unwrap()),- PhysicalSortExpr::new_default(col("b", &schema).unwrap()),- ])+ let ordering = LexOrdering::new(+ sort_columns+ .iter()+ .map(|name| PhysicalSortExpr::new_default(col(name, &schema).unwrap())),+ ).unwrap();let input =TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None).unwrap();@@ -228,7 +232,7 @@ fn aggregate_plan(schema,).unwrap();- assert_eq!(plan.input_order_mode(), &InputOrderMode::Sorted);+ assert_eq!(plan.input_order_mode(), input_order_mode);Arc::new(plan)}@@ -241,7 +245,8 @@ fn aggregation(c: &mut Criterion) {for strings in [false, true] {for run_length in [1, 8, 128, 8192] {let (schema, keys) = inputs(run_length, 8192, strings);- let plan = aggregate_plan(&schema, keys);+ let plan =+ aggregate_plan(&schema, keys, &["a", "b"], &InputOrderMode::Sorted);let name =format!("{}_run{run_length}", if strings { "string" } else { "int" });group.bench_function(name, |b| {@@ -261,5 +266,57 @@ fn aggregation(c: &mut Criterion) {group.finish();}-criterion_group!(benches, grouping, aggregation);+/// Input ordered by `a` only. Each `a` value has 32768 distinct `b` values, so+/// every `a` boundary completes four default batches of groups at once.+fn partially_ordered_aggregation(c: &mut Criterion) {+ const GROUPS_PER_KEY: usize = 32768;+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int32, false),+ Field::new("b", DataType::Int32, false),+ ]));+ let keys = (0..ROWS)+ .step_by(8192)+ .map(|start| {+ let rows = start..(start + 8192).min(ROWS);+ let a: ArrayRef = Arc::new(Int32Array::from_iter_values(+ rows.clone().map(|row| (row / GROUPS_PER_KEY) as i32),+ ));+ let b: ArrayRef = Arc::new(Int32Array::from_iter_values(+ rows.map(|row| (row % GROUPS_PER_KEY) as i32),+ ));+ vec![a, b]+ })+ .collect();+ let plan = aggregate_plan(+ &schema,+ keys,+ &["a"],+ &InputOrderMode::PartiallySorted(vec![0]),+ );+ let runtime = Runtime::new().unwrap();+ let mut group = c.benchmark_group("partially_ordered_aggregate_exec");+ group.sample_size(10);+ group.warm_up_time(Duration::from_millis(250));+ group.measurement_time(Duration::from_secs(1));+ group.bench_function("groups_per_key32768", |b| {+ b.iter(|| {+ black_box(+ runtime+ .block_on(collect(+ Arc::clone(&plan),+ Arc::new(TaskContext::default()),+ ))+ .unwrap(),+ )+ });+ });+ group.finish();+}++criterion_group!(+ benches,+ grouping,+ aggregation,+ partially_ordered_aggregation+);criterion_main!(benches);diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rsindex fd9e92635f..86c052009f 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs@@ -24,7 +24,6 @@ use std::sync::Arc;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;-use datafusion_common::assert_or_internal_err;use datafusion_execution::memory_pool::proxy::VecAllocExt;use datafusion_expr::{AggregateMetrics, EmitTo};@@ -128,9 +127,6 @@ pub(in crate::aggregates) struct OrderedAggregateTable<OrderedAggrMode> {/// to pass through or spill its current state.pub(super) state_schema: SchemaRef,- /// Maximum rows per emitted output batch, from config `batch_size`.- pub(super) batch_size: usize,-/// Grouping and accumulator-specific timing metrics.pub(super) group_by_metrics: GroupByMetrics,@@ -188,17 +184,11 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {input_schema: &SchemaRef,output_schema: SchemaRef,state_schema: SchemaRef,- batch_size: usize,input_order_mode: &InputOrderMode,aggregate_mode: &AggregateMode,filters: Vec<Option<Arc<dyn PhysicalExpr>>>,metrics: OrderedAggregateTableMetrics,) -> Result<Self> {- assert_or_internal_err!(- batch_size > 0,- "OrderedAggregateTable requires config batch_size >= 1"- );-let group_ordering = GroupOrdering::try_new(input_order_mode)?;let group_schema = agg.group_by().group_schema(input_schema)?;let group_values = new_group_values(group_schema, &group_ordering)?;@@ -229,7 +219,6 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {Ok(Self {output_schema,state_schema,- batch_size,group_by_metrics: metrics.group_by,aggregate_argument_metrics: metrics.aggregate_arguments,aggregate_accumulator_metrics: metrics.accumulator,@@ -418,38 +407,8 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {Ok(())}- /// Emits groups allowed by `GroupOrdering`, leaving only the current- /// unfinished ordered-key range buffered.- ///- /// Each aggregation mode chooses a different `materialize_accumulator_fn`- /// according to its semantics. For example, partial aggregation emits- /// partial states to feed the final stage, so it uses- /// [`datafusion_expr::GroupsAccumulator::state`].- pub(super) fn next_output_batch_inner(- &mut self,- materialize_accumulator_fn: MaterializeAccumulatorFn,- accumulator_phase: AccumulatorPhase,- ) -> Result<Option<RecordBatch>> {- if self.buffer.group_values.is_empty() {- return Ok(None);- }-- let Some(emit_to) = self.buffer.group_ordering.emit_to() else {- return Ok(None);- };- let emit_to = match emit_to {- EmitTo::First(n) => EmitTo::First(n.min(self.batch_size)),- EmitTo::All if self.num_groups() > self.batch_size => {- EmitTo::First(self.batch_size)- }- EmitTo::All => EmitTo::All,- };- self.materialize_groups(emit_to, materialize_accumulator_fn, accumulator_phase)- .map(Some)- }-/// Removes the selected groups once and materializes their output columns.- /// The caller chooses the completed prefix and any output-size limit.+ /// The caller chooses the completed prefix.pub(super) fn materialize_groups(&mut self,emit_to: EmitTo,@@ -459,11 +418,9 @@ impl<AggrMode> OrderedAggregateTable<AggrMode> {let accumulator_metrics = Arc::clone(&self.aggregate_accumulator_metrics);let output = self.group_by_metrics.time_emitting(|| {let mut output = self.buffer.group_values.emit(emit_to)?;- // EOF can also emit a prefix when a caller limits its batch size,- // but the completed ordering state no longer tracks group indexes.- if let EmitTo::First(n) = emit_to- && matches!(self.buffer.group_ordering.emit_to(), Some(EmitTo::First(_)))- {+ // `EmitTo::All` is only used after `input_done`, when the ordering+ // state no longer tracks group indexes.+ if let EmitTo::First(n) = emit_to {self.buffer.group_ordering.remove_groups(n);}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rsindex f70fc0bf0d..9cf6497e48 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs@@ -46,7 +46,6 @@ impl OrderedAggregateTable<FinalMarker> {agg: &AggregateExec,input_schema: &SchemaRef,output_schema: SchemaRef,- batch_size: usize,input_order_mode: &InputOrderMode,metrics: OrderedAggregateTableMetrics,) -> Result<Self> {@@ -55,7 +54,6 @@ impl OrderedAggregateTable<FinalMarker> {input_schema,output_schema,Arc::clone(input_schema),- batch_size,input_order_mode,&AggregateMode::Final,vec![None; agg.aggr_expr().len()],diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rsindex 104855c5f0..3756c4b8e7 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs@@ -57,7 +57,6 @@ impl OrderedAggregateTable<PartialMarker> {agg: &AggregateExec,partition: usize,output_schema: SchemaRef,- batch_size: usize,) -> Result<Self> {let input_schema = agg.input().schema();let state_schema = Arc::clone(&output_schema);@@ -67,7 +66,6 @@ impl OrderedAggregateTable<PartialMarker> {&input_schema,output_schema,state_schema,- batch_size,&agg.input_order_mode,&AggregateMode::Partial,agg.filter_expr().to_vec(),diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rsindex dd266a68eb..89859bc036 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/ordered_single_table.rs@@ -44,7 +44,6 @@ impl OrderedAggregateTable<SingleMarker> {partition: usize,output_schema: SchemaRef,state_schema: SchemaRef,- batch_size: usize,) -> Result<Self> {debug_assert!(matches!(agg.mode,@@ -58,7 +57,6 @@ impl OrderedAggregateTable<SingleMarker> {&input_schema,output_schema,state_schema,- batch_size,&agg.input_order_mode,&agg.mode,agg.filter_expr().to_vec(),@@ -80,14 +78,24 @@ impl OrderedAggregateTable<SingleMarker> {)}- /// Emits the next batch of final aggregate values for groups proven complete- /// by the input ordering.- pub(in crate::aggregates) fn next_output_batch(+ /// Materializes final results for all groups proven complete by the input+ /// ordering, leaving the active ordered-key range in the table.+ ///+ /// Returns None if there are no completed groups.+ pub(in crate::aggregates) fn take_completed_result_batch(&mut self,) -> Result<Option<RecordBatch>> {- self.next_output_batch_inner(+ if self.is_empty() {+ return Ok(None);+ }+ let Some(emit_to) = self.group_ordering().emit_to() else {+ return Ok(None);+ };+ self.materialize_groups(+ emit_to,HashAggregateAccumulator::evaluate_to_columns,AccumulatorPhase::Evaluate,)+ .map(Some)}}diff --git a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rsindex d9f5152456..daa5a85323 100644--- a/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_final_stream.rs@@ -189,7 +189,6 @@ impl OrderedFinalAggregateStream {agg,&input_schema,Arc::clone(&schema),- batch_size,input_order_mode,metrics,)?;diff --git a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rsindex 31046ec3d6..83a25620cd 100644--- a/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_partial_stream.rs@@ -161,7 +161,6 @@ impl OrderedPartialAggregateStream {agg,partition,Arc::clone(&schema),- batch_size,)?;let reservation =MemoryConsumer::new(format!("OrderedPartialAggregateStream[{partition}]"))diff --git a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rsindex 834fae3cc6..57744dec57 100644--- a/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs+++ b/datafusion/physical-plan/src/aggregates/ordered_single_stream.rs@@ -61,7 +61,12 @@ use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};/// After each input batch, check whether any groups can be emitted eagerly to/// improve memory efficiency. For example, if the last group key seen is/// `k = 100`, it is safe to emit all groups with keys less than 100 because the-/// input is ordered.+/// input is ordered. Materialize that entire completed prefix once, then emit+/// slices of it before reading more input. See+/// [`OrderedPartialAggregateStream::into_stream`] for why this avoids+/// repeatedly removing small batches of groups from the table.+///+/// [`OrderedPartialAggregateStream::into_stream`]: super::ordered_partial_stream::OrderedPartialAggregateStream::into_stream////// # Memory Pressure and Spilling///@@ -98,6 +103,7 @@ pub(crate) struct OrderedSingleAggregateStream {input: SendableRecordBatchStream,reservation: MemoryReservation,baseline_metrics: BaselineMetrics,+ batch_size: usize,state: Option<OrderedSingleAggregateState>,}@@ -114,8 +120,13 @@ enum OrderedSingleAggregateState {table: OrderedAggregateTable<SingleMarker>,spill_context: Box<AggregateSpill>,},- ProducingOutput {- table: OrderedAggregateTable<SingleMarker>,+ /// Emits one materialized batch in `batch_size` slices, then continues+ /// with `next_state` (`ReadingInput`, or `Done` after input is exhausted).+ Outputting {+ batch: RecordBatch,+ /// Reserved memory of `batch`, released when handing off the last slice.+ batch_memory: usize,+ next_state: Box<OrderedSingleAggregateState>,},PreparingMergeInput {table: OrderedAggregateTable<SingleMarker>,@@ -167,7 +178,6 @@ impl OrderedSingleAggregateStream {partition,Arc::clone(&schema),Arc::clone(&state_schema),- batch_size,)?;let can_spill =@@ -201,6 +211,7 @@ impl OrderedSingleAggregateStream {input,reservation,baseline_metrics,+ batch_size,state: Some(OrderedSingleAggregateState::ReadingInput {table,spill_context,@@ -238,8 +249,38 @@ impl OrderedSingleAggregateStream {}}- /// Consumes one ordered raw input batch, then immediately emits- /// finalized groups if the ordering proves any group is ready.+ /// Reserves `batch` on top of `table_memory` and moves to `Outputting`,+ /// which emits it in `batch_size` slices before continuing with+ /// `next_state`. If the batch cannot be reserved, hands it off whole.+ fn start_outputting(+ &mut self,+ batch: RecordBatch,+ table_memory: usize,+ next_state: OrderedSingleAggregateState,+ ) -> OrderedSingleAggregateStateTransition {+ let batch_memory = batch.get_array_memory_size();+ match self.reservation.try_resize(table_memory + batch_memory) {+ Ok(()) => ControlFlow::Continue(OrderedSingleAggregateState::Outputting {+ batch,+ batch_memory,+ next_state: Box::new(next_state),+ }),+ Err(DataFusionError::ResourcesExhausted(_)) => {+ // Only the retained table needs to remain reserved.+ if let Err(e) = self.reservation.try_resize(table_memory) {+ return Self::break_with_err(e);+ }+ ControlFlow::Break((+ Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),+ next_state,+ ))+ }+ Err(e) => Self::break_with_err(e),+ }+ }++ /// Consumes one ordered raw input batch, then materializes all finalized+ /// groups if the ordering proves any group is ready.////// See comments at `poll_next()` for details.///@@ -320,35 +361,27 @@ impl OrderedSingleAggregateStream {Ok(None)} else {let timer = elapsed_compute.timer();- let result = table.next_output_batch();+ let result = table.take_completed_result_batch();timer.done();result};match result {- // Some finalized groups can be emitted. Yield them, then- // continue aggregating input in the current state.+ // Some finalized groups can be emitted. Yield them in+ // slices, then continue aggregating input.Ok(Some(batch)) => {- if let Err(e) =- self.reservation- .try_resize(Self::reservation_size_for_table(- &table,- spill_context.as_deref(),- ))- {- return Self::break_with_err(e);- }- let next_state = OrderedSingleAggregateState::ReadingInput {- table,- spill_context,- };-- ControlFlow::Break((- Poll::Ready(Some(Ok(- batch.record_output(&self.baseline_metrics)- ))),- next_state,- ))+ let table_memory = Self::reservation_size_for_table(+ &table,+ spill_context.as_deref(),+ );+ self.start_outputting(+ batch,+ table_memory,+ OrderedSingleAggregateState::ReadingInput {+ table,+ spill_context,+ },+ )}// Can't do early emit, continue aggregating.Ok(None) => {@@ -374,9 +407,23 @@ impl OrderedSingleAggregateStream {}_ => {table.input_done();- ControlFlow::Continue(- OrderedSingleAggregateState::ProducingOutput { table },- )+ let elapsed_compute =+ self.baseline_metrics.elapsed_compute().clone();+ let timer = elapsed_compute.timer();+ let result = table.take_completed_result_batch();+ drop(table);+ timer.done();+ match result {+ Ok(Some(batch)) => self.start_outputting(+ batch,+ 0,+ OrderedSingleAggregateState::Done,+ ),+ Ok(None) => {+ ControlFlow::Continue(OrderedSingleAggregateState::Done)+ }+ Err(e) => Self::break_with_err(e),+ }}}}@@ -520,61 +567,47 @@ impl OrderedSingleAggregateStream {}}- /// Emits one batch after input is exhausted.- ///- /// `table.input_done()` has already made every remaining group safe to emit,- /// so this state keeps draining until the table is empty.+ /// Emits the next `batch_size` slice of a materialized batch.////// See comments at `poll_next()` for details.////// Returns the next operator state with control flow decision.- fn handle_producing_output(+ fn handle_outputting(&mut self,original_state: OrderedSingleAggregateState,) -> OrderedSingleAggregateStateTransition {- let OrderedSingleAggregateState::ProducingOutput { table } = original_state+ let OrderedSingleAggregateState::Outputting {+ batch,+ batch_memory,+ next_state,+ } = original_stateelse {return Self::break_with_internal_err(- "Ordered single aggregate stream expected ProducingOutput state",+ "Ordered single aggregate stream expected Outputting state",);};- let mut table = table;- let elapsed_compute = self.baseline_metrics.elapsed_compute().clone();- let timer = elapsed_compute.timer();- let result = table.next_output_batch();- timer.done();-- match result {- Ok(Some(batch)) => {- let next_state = if table.is_empty() {- drop(table);- if let Err(e) = self.reservation.try_resize(0) {- return Self::break_with_err(e);- }- OrderedSingleAggregateState::Done- } else {- if let Err(e) = self.reservation.try_resize(table.memory_size()) {- return Self::break_with_err(e);- }- OrderedSingleAggregateState::ProducingOutput { table }- };-- ControlFlow::Break((- Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),+ if batch.num_rows() > self.batch_size {+ let output = batch.slice(0, self.batch_size);+ let batch = batch.slice(self.batch_size, batch.num_rows() - self.batch_size);+ return ControlFlow::Break((+ Poll::Ready(Some(Ok(output.record_output(&self.baseline_metrics)))),+ OrderedSingleAggregateState::Outputting {+ batch,+ batch_memory,next_state,- ))- }- Err(e) => Self::break_with_err(e),- Ok(None) => {- drop(table);- let next_state = OrderedSingleAggregateState::Done;- if let Err(e) = self.reservation.try_resize(0) {- return Self::break_with_err(e);- }- ControlFlow::Continue(next_state)- }+ },+ ));}++ // The final slice transfers ownership of the buffers to the consumer.+ if let Err(e) = self.reservation.try_shrink(batch_memory) {+ return Self::break_with_err(e);+ }+ ControlFlow::Break((+ Poll::Ready(Some(Ok(batch.record_output(&self.baseline_metrics)))),+ *next_state,+ ))}}@@ -595,14 +628,18 @@ impl Stream for OrderedSingleAggregateStream {////// ReadingInput/// -> ReadingInput- /// Aggregate one input batch. If it fits in memory, optionally yield- /// groups proven complete by the input ordering, then read the next batch.+ /// Aggregate one input batch. If it fits in memory and no groups are+ /// complete, read the next batch./// -> Spilling/// The table cannot reserve enough memory. Move all current states into/// one fully group-key-sorted spill run.- /// -> ProducingOutput- /// Input was exhausted without spilling. Mark every remaining group as- /// complete and produce its final result.+ /// -> Outputting+ /// Either the input ordering proves some groups complete, or input was+ /// exhausted without spilling and every remaining group is complete.+ /// Materialize all of them once into one batch. If the batch cannot be+ /// reserved, yield it whole and go to the state after `Outputting`.+ /// -> Done+ /// Input was exhausted without spilling and no groups remain./// -> PreparingMergeInput/// Input was exhausted after spilling. Spill the last in-memory run and/// construct the ordered input used to merge all spill files.@@ -624,12 +661,15 @@ impl Stream for OrderedSingleAggregateStream {/// -> Done/// The merged spill input was fully aggregated.///- /// ProducingOutput- /// -> ProducingOutput- /// One remaining final aggregate batch was yielded; repeat to continue- /// draining the table.+ /// Outputting+ /// Yield one `batch_size` slice of the materialized batch, keeping the+ /// batch reserved until its last slice is handed off.+ /// -> Outputting+ /// More slices remain.+ /// -> ReadingInput+ /// The batch was fully emitted; resume aggregating input./// -> Done- /// All remaining groups were emitted.+ /// The batch was fully emitted after input was exhausted.////// Any active state/// -> Error@@ -664,8 +704,8 @@ impl Stream for OrderedSingleAggregateStream {state @ OrderedSingleAggregateState::MergingSpills { .. } => {self.handle_merging_spills(cx, state)}- state @ OrderedSingleAggregateState::ProducingOutput { .. } => {- self.handle_producing_output(state)+ state @ OrderedSingleAggregateState::Outputting { .. } => {+ self.handle_outputting(state)}state @ OrderedSingleAggregateState::Error => {self.close_input();@@ -711,3 +751,98 @@ impl RecordBatchStream for OrderedSingleAggregateStream {Arc::clone(&self.schema)}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::aggregates::PhysicalGroupBy;+ use crate::collect;+ use crate::test::TestMemoryExec;+ use arrow::array::Int64Array;+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::test_util::batches_to_sort_string;+ use datafusion_execution::config::SessionConfig;+ use datafusion_execution::memory_pool::{+ GreedyMemoryPool, MemoryPool, PeakRecordingPool,+ };+ use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_functions_aggregate::sum::sum_udaf;+ use datafusion_physical_expr::PhysicalSortExpr;+ use datafusion_physical_expr::aggregate::AggregateExprBuilder;+ use datafusion_physical_expr::expressions::col;+ use datafusion_physical_expr_common::sort_expr::LexOrdering;++ /// Runs `SELECT a, b, SUM(v) GROUP BY a, b` with `batch_size = 4` over+ /// input ordered by `a`. Each `a` boundary completes 43 groups at once.+ async fn run(pool: Arc<dyn MemoryPool>) -> Result<Vec<RecordBatch>> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, false),+ Field::new("v", DataType::Int64, false),+ ]));+ let batches = [0, 0, 1, 1, 2, 2]+ .map(|a| {+ RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int64Array::from(vec![a; 43])),+ Arc::new(Int64Array::from_iter_values(0..43)),+ Arc::new(Int64Array::from_iter_values(0..43)),+ ],+ )+ .unwrap()+ })+ .to_vec();+ let input = TestMemoryExec::try_new(&[batches], Arc::clone(&schema), None)?+ .try_with_sort_information(vec![+ LexOrdering::new([PhysicalSortExpr::new_default(col("a", &schema)?)])+ .unwrap(),+ ])?;+ let aggregate = AggregateExec::try_new(+ AggregateMode::Single,+ PhysicalGroupBy::new_single(vec![+ (col("a", &schema)?, "a".into()),+ (col("b", &schema)?, "b".into()),+ ]),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("sum")+ .build()?,+ )],+ vec![None],+ Arc::new(TestMemoryExec::update_cache(&Arc::new(input))),+ Arc::clone(&schema),+ )?;+ let context = TaskContext::default()+ .with_session_config(+ SessionConfig::new()+ .with_batch_size(4)+ .set_bool("datafusion.execution.enable_migration_aggregate", true),+ )+ .with_runtime(+ RuntimeEnvBuilder::new()+ .with_memory_pool(pool)+ .build_arc()?,+ );+ collect(Arc::new(aggregate), Arc::new(context)).await+ }++ /// One byte below the unlimited peak, the completed groups cannot be+ /// reserved next to the table, so they are handed off without slicing.+ /// The result must not change.+ #[tokio::test]+ async fn completed_groups_under_memory_pressure() -> Result<()> {+ let pool = Arc::new(PeakRecordingPool::new(Arc::new(GreedyMemoryPool::new(+ usize::MAX,+ ))));+ let expected = run(Arc::clone(&pool) as _).await?;+ let limit = pool.peak_reserved() - 1;+ let actual = run(Arc::new(GreedyMemoryPool::new(limit))).await?;+ assert_eq!(+ batches_to_sort_string(&actual),+ batches_to_sort_string(&expected)+ );+ Ok(())+ }+}