Apache DataFusion SQL Query Engine
Avoid repeated `EmitTo::First` in partial hash aggregate output (#23250)
Huang Qiwei7d9f6ea405d5
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rsindex 719fbe93e5..a0d8204180 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs@@ -182,7 +182,7 @@ impl<AggrMode> AggregateHashTable<AggrMode> {acc + state.group_values.size()+ state.batch_group_indices.allocated_size()}- AggregateHashTableState::OutputtingMaterializedFinal(output) => {+ AggregateHashTableState::OutputtingMaterialized(output) => {output.memory_size()}AggregateHashTableState::Done => 0,@@ -214,15 +214,6 @@ impl<AggrMode> AggregateHashTable<AggrMode> {}}-pub(super) fn emit_to_for_batch_size(batch_size: usize, group_count: usize) -> EmitTo {- debug_assert!(batch_size > 0);- if group_count <= batch_size {- EmitTo::All- } else {- EmitTo::First(batch_size)- }-}-/// State and argument information for a single Aggregate////// For example, for `SELECT COUNT(x), SUM(y WHERE z > 10) ...` there would be two@@ -304,24 +295,25 @@ pub(super) enum AggregateHashTableState {Building(AggregateHashTableBuffer),/// Emitting results directly from group keys and aggregate state.Outputting(AggregateHashTableBuffer),- /// Materialize all the output results, and then incrementally output in the `OutputtingMaterializedFinal` state.+ /// Materialize all the output results, and then incrementally output in the `OutputtingMaterialized` state.////// Note this is a temporary solution until the `GroupValues` issue is solved:/// Issue: <https://github.com/apache/datafusion/issues/23178>- OutputtingMaterializedFinal(MaterializedFinalOutput),+ OutputtingMaterialized(MaterializedAggregateOutput),Done,}-/// Fully evaluated final aggregate output and the next row offset to emit.+/// Fully evaluated aggregate output and the next row offset to emit.///-/// Final aggregate evaluation consumes accumulator state, so final output is-/// materialized once and then sliced to honor `batch_size` across output polls.-pub(super) struct MaterializedFinalOutput {+/// Final aggregate evaluation consumes accumulator state, and partial terminal+/// output should not repeatedly renumber group values with `EmitTo::First`.+/// Materialize once and then slice to honor `batch_size` across output polls.+pub(super) struct MaterializedAggregateOutput {batch: RecordBatch,offset: usize,}-impl MaterializedFinalOutput {+impl MaterializedAggregateOutput {pub(super) fn new(batch: RecordBatch) -> Self {Self { batch, offset: 0 }}@@ -496,7 +488,7 @@ mod tests {use super::*;#[test]- fn materialized_final_output_slices_batches_until_exhausted() -> Result<()> {+ fn materialized_aggregate_output_slices_batches_until_exhausted() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("group_col",DataType::Int32,@@ -506,7 +498,7 @@ mod tests {schema,vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))],)?;- let mut output = MaterializedFinalOutput::new(batch);+ let mut output = MaterializedAggregateOutput::new(batch);assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![1, 2]);assert_eq!(int32_values(&output.next_batch(2).unwrap(), 0), vec![3, 4]);diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rsindex c3e4f831c4..bd70d10858 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs@@ -26,7 +26,7 @@ use crate::aggregates::AggregateExec;use super::common::{AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState, FinalMarker,- MaterializedFinalOutput,+ MaterializedAggregateOutput,};/// Methods specific to the aggregate hash table used in the final aggregation stage.@@ -57,8 +57,8 @@ impl AggregateHashTable<FinalMarker> {) -> Result<Option<RecordBatch>> {let output_schema = Arc::clone(&self.output_schema);let batch_size = self.batch_size;- // Take ownership of the output state. Note `emit_next_materialized_batch`- // updates state after it emits a materialized slice.+ // Take ownership of the output state. `emit_next_materialized_batch`+ // restores `self.state` to `OutputtingMaterialized` or `Done`.match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {AggregateHashTableState::Outputting(state) => {if state.group_values.is_empty() {@@ -68,7 +68,7 @@ impl AggregateHashTable<FinalMarker> {let output = self.materialize_final_output(state, output_schema)?;Ok(self.emit_next_materialized_batch(output, batch_size))}- AggregateHashTableState::OutputtingMaterializedFinal(output) => {+ AggregateHashTableState::OutputtingMaterialized(output) => {Ok(self.emit_next_materialized_batch(output, batch_size))}AggregateHashTableState::Done => Ok(None),@@ -82,7 +82,7 @@ impl AggregateHashTable<FinalMarker> {&self,mut state: AggregateHashTableBuffer,output_schema: SchemaRef,- ) -> Result<MaterializedFinalOutput> {+ ) -> Result<MaterializedAggregateOutput> {// Final aggregate evaluation consumes accumulator state. Evaluate all// groups once, then slice the materialized batch on subsequent polls.let emit_to = EmitTo::All;@@ -96,19 +96,19 @@ impl AggregateHashTable<FinalMarker> {let batch = RecordBatch::try_new(output_schema, output)?;debug_assert!(batch.num_rows() > 0);- Ok(MaterializedFinalOutput::new(batch))+ Ok(MaterializedAggregateOutput::new(batch))}fn emit_next_materialized_batch(&mut self,- mut output: MaterializedFinalOutput,+ mut output: MaterializedAggregateOutput,batch_size: usize,) -> Option<RecordBatch> {let batch = output.next_batch(batch_size);if output.is_exhausted() {self.state = AggregateHashTableState::Done;} else {- self.state = AggregateHashTableState::OutputtingMaterializedFinal(output);+ self.state = AggregateHashTableState::OutputtingMaterialized(output);}batch}diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rsindex 9d226aa28b..53b44f5d9a 100644--- a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs@@ -23,6 +23,7 @@ use arrow::array::{ArrayRef, BooleanArray, new_null_array};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};+use datafusion_expr::EmitTo;use crate::aggregates::group_values::new_group_values;use crate::aggregates::order::GroupOrdering;@@ -30,8 +31,8 @@ use crate::aggregates::{AggregateExec, group_id_array, max_duplicate_ordinal};use super::common::{AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState,- EvaluatedAccumulatorArgs, HashAggregateAccumulator, PartialMarker, PartialSkipMarker,- emit_to_for_batch_size,+ EvaluatedAccumulatorArgs, HashAggregateAccumulator, MaterializedAggregateOutput,+ PartialMarker, PartialSkipMarker,};/// Methods specific to the aggregate hash table used in the partial aggregation stage.@@ -62,43 +63,60 @@ impl AggregateHashTable<PartialMarker> {) -> Result<Option<RecordBatch>> {let output_schema = Arc::clone(&self.output_schema);let batch_size = self.batch_size;- match &mut self.state {+ // Take ownership of the output state. `emit_next_materialized_batch`+ // restores `self.state` to `OutputtingMaterialized` or `Done`.+ match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {AggregateHashTableState::Outputting(state) => {if state.group_values.is_empty() {- self.state = AggregateHashTableState::Done;return Ok(None);}- let emit_to =- emit_to_for_batch_size(batch_size, state.group_values.len());- let timer = self.group_by_metrics.emitting_time.timer();- let mut output = state.group_values.emit(emit_to)?;-- for acc in state.accumulators.iter_mut() {- output.extend(acc.state(emit_to)?);- }- let done = state.group_values.is_empty();- drop(timer);-- let batch = RecordBatch::try_new(output_schema, output)?;- debug_assert!(batch.num_rows() > 0);- if done {- self.state = AggregateHashTableState::Done;- }- Ok(Some(batch))+ let output = self.materialize_partial_output(state, output_schema)?;+ Ok(self.emit_next_materialized_batch(output, batch_size))+ }+ AggregateHashTableState::OutputtingMaterialized(output) => {+ Ok(self.emit_next_materialized_batch(output, batch_size))}AggregateHashTableState::Done => Ok(None),AggregateHashTableState::Building(_) => {internal_err!("next_output_batch must be called in the outputting state")}- AggregateHashTableState::OutputtingMaterializedFinal(_) => {- internal_err!(- "partial aggregate output should not materialize final output"- )- }}}+ fn materialize_partial_output(+ &self,+ mut state: AggregateHashTableBuffer,+ output_schema: SchemaRef,+ ) -> Result<MaterializedAggregateOutput> {+ let emit_to = EmitTo::All;+ let timer = self.group_by_metrics.emitting_time.timer();+ let mut output = state.group_values.emit(emit_to)?;++ for acc in state.accumulators.iter_mut() {+ output.extend(acc.state(emit_to)?);+ }+ drop(timer);++ let batch = RecordBatch::try_new(output_schema, output)?;+ debug_assert!(batch.num_rows() > 0);+ Ok(MaterializedAggregateOutput::new(batch))+ }++ fn emit_next_materialized_batch(+ &mut self,+ mut output: MaterializedAggregateOutput,+ batch_size: usize,+ ) -> Option<RecordBatch> {+ let batch = output.next_batch(batch_size);+ if output.is_exhausted() {+ self.state = AggregateHashTableState::Done;+ } else {+ self.state = AggregateHashTableState::OutputtingMaterialized(output);+ }+ batch+ }+pub(in crate::aggregates) fn can_skip_aggregation(&self) -> bool {self.state.building()diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 4f5b893578..07c77c860e 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -2434,8 +2434,8 @@ mod tests {};use arrow::array::{- DictionaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StructArray,- UInt32Array, UInt64Array,+ BooleanArray, DictionaryArray, Float32Array, Float64Array, Int32Array,+ Int64Array, StructArray, UInt32Array, UInt64Array,};use arrow::compute::{SortOptions, concat_batches};use arrow::datatypes::Int32Type;@@ -2445,7 +2445,10 @@ mod tests {use datafusion_execution::memory_pool::FairSpillPool;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};- use datafusion_expr::{AggregateUDF, AggregateUDFImpl, Signature, Volatility};+ use datafusion_expr::{+ Accumulator, AggregateUDF, AggregateUDFImpl, EmitTo, GroupsAccumulator,+ Signature, Volatility,+ };use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf;use datafusion_functions_aggregate::array_agg::array_agg_udaf;use datafusion_functions_aggregate::average::avg_udaf;@@ -3229,6 +3232,76 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn partial_grouped_aggregate_materializes_before_slicing() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("key", DataType::Int32, false),+ Field::new("value", DataType::Int32, false),+ ]));+ let input_batches = vec![RecordBatch::try_new(+ Arc::clone(&schema),+ vec![+ Arc::new(Int32Array::from(vec![1, 2, 3])),+ Arc::new(Int32Array::from(vec![10, 20, 30])),+ ],+ )?];+ let input =+ TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;+ let group_by =+ PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);+ let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(+ AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("no_first_emit(value)")+ .build()?,+ )];+ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ group_by,+ aggregates,+ vec![None],+ input,+ Arc::clone(&schema),+ )?);+ let task_ctx = Arc::new(+ TaskContext::default().with_session_config(+ SessionConfig::new()+ .with_batch_size(2)+ .set_bool("datafusion.execution.enable_migration_aggregate", true)+ .set(+ "datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",+ &ScalarValue::Float64(Some(2.0)),+ ),+ ),+ );++ let stream = aggregate.execute_typed(0, &task_ctx)?;+ assert!(matches!(stream, StreamType::PartialHash(_)));++ let stream: SendableRecordBatchStream = stream.into();+ let batches = collect(stream).await?;+ assert_eq!(+ batches+ .iter()+ .map(RecordBatch::num_rows)+ .collect::<Vec<_>>(),+ vec![2, 1]+ );+ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 3);+ assert_snapshot!(batches_to_sort_string(&batches), @r"+ +-----+-----------------------------++ | key | no_first_emit(value)[count] |+ +-----+-----------------------------++ | 1 | 1 |+ | 2 | 1 |+ | 3 | 1 |+ +-----+-----------------------------++ ");++ Ok(())+ }+#[tokio::test]async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {let schema =@@ -6256,6 +6329,140 @@ mod tests {}}+ #[derive(Debug, PartialEq, Eq, Hash)]+ struct NoFirstEmitUdaf {+ signature: Signature,+ }++ impl NoFirstEmitUdaf {+ fn new() -> Self {+ Self {+ signature: Signature::exact(vec![DataType::Int32], Volatility::Immutable),+ }+ }+ }++ impl AggregateUDFImpl for NoFirstEmitUdaf {+ fn name(&self) -> &str {+ "no_first_emit"+ }++ fn signature(&self) -> &Signature {+ &self.signature+ }++ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {+ Ok(DataType::Int64)+ }++ fn state_fields(&self, args: StateFieldsArgs) -> Result<Vec<FieldRef>> {+ Ok(vec![Arc::new(Field::new(+ format!("{}[count]", args.name),+ DataType::Int64,+ false,+ ))])+ }++ fn accumulator(+ &self,+ _acc_args: AccumulatorArgs,+ ) -> Result<Box<dyn Accumulator>> {+ Ok(Box::new(NoFirstEmitAccumulator))+ }++ fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {+ true+ }++ fn create_groups_accumulator(+ &self,+ _args: AccumulatorArgs,+ ) -> Result<Box<dyn GroupsAccumulator>> {+ Ok(Box::new(NoFirstEmitGroupsAccumulator { counts: vec![] }))+ }+ }++ #[derive(Debug)]+ struct NoFirstEmitAccumulator;++ impl Accumulator for NoFirstEmitAccumulator {+ fn update_batch(&mut self, _values: &[ArrayRef]) -> Result<()> {+ Ok(())+ }++ fn evaluate(&mut self) -> Result<ScalarValue> {+ Ok(ScalarValue::Int64(Some(0)))+ }++ fn size(&self) -> usize {+ size_of_val(self)+ }++ fn state(&mut self) -> Result<Vec<ScalarValue>> {+ Ok(vec![ScalarValue::Int64(Some(0))])+ }++ fn merge_batch(&mut self, _states: &[ArrayRef]) -> Result<()> {+ Ok(())+ }+ }++ #[derive(Debug)]+ struct NoFirstEmitGroupsAccumulator {+ counts: Vec<i64>,+ }++ impl NoFirstEmitGroupsAccumulator {+ fn emit_counts(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {+ match emit_to {+ EmitTo::All => {+ let counts = std::mem::take(&mut self.counts);+ Ok(Arc::new(Int64Array::from(counts)))+ }+ EmitTo::First(_) => internal_err!(+ "partial grouped aggregate output must materialize with EmitTo::All before slicing"+ ),+ }+ }+ }++ impl GroupsAccumulator for NoFirstEmitGroupsAccumulator {+ fn update_batch(+ &mut self,+ _values: &[ArrayRef],+ group_indices: &[usize],+ _opt_filter: Option<&BooleanArray>,+ total_num_groups: usize,+ ) -> Result<()> {+ self.counts.resize(total_num_groups, 0);+ for group_index in group_indices {+ self.counts[*group_index] += 1;+ }+ Ok(())+ }++ fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {+ self.emit_counts(emit_to)+ }++ fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {+ Ok(vec![self.emit_counts(emit_to)?])+ }++ fn merge_batch(+ &mut self,+ _values: &[ArrayRef],+ _group_indices: &[usize],+ _total_num_groups: usize,+ ) -> Result<()> {+ Ok(())+ }++ fn size(&self) -> usize {+ size_of_val(self) + self.counts.capacity() * size_of::<i64>()+ }+ }+/// Test that [`AggregateExec::with_dynamic_filter_expr`] overrides the existing dynamic filter#[test]fn test_with_dynamic_filter() -> Result<()> {