Skip to content

Apache DataFusion SQL Query Engine

refactor(hash-aggr): Use `EmitTo` to output (#23055)

Yongting You681ba9bc7a45
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
new file mode 100644
index 0000000000..90039e70a6
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs
@@ -0,0 +1,442 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, AsArray, new_null_array};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, internal_err};
+use datafusion_execution::memory_pool::proxy::VecAllocExt;
+use datafusion_expr::{EmitTo, GroupsAccumulator};
+use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
+
+use crate::PhysicalExpr;
+use crate::aggregates::group_values::{GroupByMetrics, GroupValues, new_group_values};
+use crate::aggregates::order::GroupOrdering;
+use crate::aggregates::row_hash::create_group_accumulator;
+use crate::aggregates::{
+ AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
+};
+
+/// Marker for raw rows -> partial state aggregation.
+pub(in crate::aggregates) struct PartialMarker;
+/// Marker for raw rows -> partial state conversion without aggregation.
+pub(in crate::aggregates) struct PartialSkipMarker;
+/// Marker for partial state -> final value aggregation.
+pub(in crate::aggregates) struct FinalMarker;
+
+/// Grouped hash table shared by the partial and final paths.
+///
+/// While building, it consumes input batches and updates group / accumulator
+/// state. While outputting, it incrementally drains that state into output
+/// batches.
+///
+/// # Logical and Physical Model
+///
+/// Logically, this is a hash table that maps { group keys -> accumulator states }
+/// For example, `AVG(v) GROUP BY k` stores one entry per `k`, where each
+/// entry owns the `sum(v)` and `count(v)` state needed to compute the final
+/// average.
+///
+/// Physically, the group keys and accumulators are backed by [`GroupValues`] and
+/// [`GroupsAccumulator`]. Both use columnar storage so aggregation can stay
+/// vectorized.
+///
+/// # Marker Type
+/// `AggrMode` selects the aggregate semantics.
+///
+/// e.g. `AggregateHashTable::<PartialMarker>::new(...)` creates an aggregate hash table
+/// for the partial hash aggregate stage, the input schema is raw rows and output
+/// schema is intermediate states.
+///
+/// It is a zero-sized compile-time marker, so each stage keeps its update logic
+/// in a separate impl block, to make the behavior difference explicit.
+pub(in crate::aggregates) struct AggregateHashTable<AggrMode> {
+ /// Grouping and accumulator-specific timing metrics.
+ pub(super) group_by_metrics: GroupByMetrics,
+
+ /// Raw input schema, used to evaluate expressions and synthesize empty
+ /// grouping-set rows.
+ pub(super) input_schema: SchemaRef,
+
+ /// Output schema: group columns followed by aggregate state or final values.
+ pub(super) output_schema: SchemaRef,
+
+ /// Maximum rows per emitted output batch, from config `batch_size`.
+ pub(super) batch_size: usize,
+
+ /// Lifecycle-specific state: building stage / outputting stage.
+ pub(super) state: AggregateHashTableState,
+
+ pub(super) _mode: PhantomData<AggrMode>,
+}
+
+/// Methods shared by all aggregate hash table modes.
+impl<AggrMode> AggregateHashTable<AggrMode> {
+ pub(super) fn new_with_filters(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ filters: Vec<Option<Arc<dyn PhysicalExpr>>>,
+ ) -> Result<Self> {
+ if batch_size == 0 {
+ return internal_err!("AggregateHashTable requires config batch_size >= 1");
+ }
+
+ let input_schema = agg.input().schema();
+ let aggregate_arguments = aggregate_expressions(
+ &agg.aggr_expr,
+ &agg.mode,
+ agg.group_by.num_group_exprs(),
+ )?;
+ let accumulators: Vec<_> = agg
+ .aggr_expr
+ .iter()
+ .zip(aggregate_arguments)
+ .zip(filters)
+ .map(|((agg_expr, arguments), filter)| {
+ let accumulator = create_group_accumulator(agg_expr)?;
+ Ok(HashAggregateAccumulator::new(
+ Arc::clone(agg_expr),
+ arguments,
+ filter,
+ accumulator,
+ ))
+ })
+ .collect::<Result<_>>()?;
+
+ let group_schema = agg.group_by.group_schema(&input_schema)?;
+ let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
+
+ Ok(Self {
+ group_by_metrics: GroupByMetrics::new(&agg.metrics, partition),
+ input_schema,
+ output_schema,
+ batch_size,
+ state: AggregateHashTableState::Building(AggregateHashTableBuffer {
+ group_by: Arc::clone(&agg.group_by),
+ group_values,
+ batch_group_indices: Default::default(),
+ accumulators,
+ }),
+ _mode: PhantomData,
+ })
+ }
+
+ /// See comments in [`EvaluatedAggregateBatch`]
+ pub(super) fn evaluate_batch(
+ &self,
+ batch: &RecordBatch,
+ ) -> Result<EvaluatedAggregateBatch> {
+ let state = self.state.building();
+ let timer = self.group_by_metrics.time_calculating_group_ids.timer();
+ // outer vec: one per each grouping set
+ // inner vec: all group by exprs for the current grouping set
+ let grouping_set_args = evaluate_group_by(&state.group_by, batch)?;
+ drop(timer);
+
+ let timer = self.group_by_metrics.aggregate_arguments_time.timer();
+ // The evaluated args for each accumulator
+ let accumulator_args = self
+ .state
+ .building()
+ .accumulators
+ .iter()
+ .map(|acc| acc.evaluate_acc_args(batch))
+ .collect::<Result<Vec<_>>>()?;
+ drop(timer);
+
+ Ok(EvaluatedAggregateBatch {
+ grouping_set_args,
+ accumulator_args,
+ })
+ }
+
+ pub(in crate::aggregates) fn memory_size(&self) -> usize {
+ match &self.state {
+ AggregateHashTableState::Building(state)
+ | AggregateHashTableState::Outputting(state) => {
+ let acc = state
+ .accumulators
+ .iter()
+ .map(|acc| acc.accumulator.size())
+ .sum::<usize>();
+
+ acc + state.group_values.size()
+ + state.batch_group_indices.allocated_size()
+ }
+ AggregateHashTableState::Done => 0,
+ }
+ }
+
+ /// Returns the number of distinct groups accumulated so far.
+ pub(in crate::aggregates) fn building_group_count(&self) -> usize {
+ self.state.building().group_values.len()
+ }
+
+ pub(in crate::aggregates) fn is_building(&self) -> bool {
+ matches!(self.state, AggregateHashTableState::Building(_))
+ }
+
+ pub(in crate::aggregates) fn is_done(&self) -> bool {
+ matches!(self.state, AggregateHashTableState::Done)
+ }
+
+ pub(super) fn start_outputting(&mut self) {
+ let AggregateHashTableState::Building(mut state) =
+ std::mem::replace(&mut self.state, AggregateHashTableState::Done)
+ else {
+ unreachable!("hash aggregate table is not building")
+ };
+
+ state.batch_group_indices = Vec::new();
+ self.state = AggregateHashTableState::Outputting(state);
+ }
+}
+
+pub(super) fn emit_to_for_batch_size(batch_size: usize, group_count: usize) -> EmitTo {
+ debug_assert!(batch_size > 0);
+ if group_count <= batch_size {
+ EmitTo::All
+ } else {
+ EmitTo::First(batch_size)
+ }
+}
+
+/// State and argument information for a single Aggregate
+///
+/// For example, for `SELECT COUNT(x), SUM(y WHERE z > 10) ...` there would be two
+/// `HashAggregateAccumulator`, one each for `COUNT(x)` and `SUM(y WHERE z > 10)`
+pub(super) struct HashAggregateAccumulator {
+ /// Aggregate expression used to create a fresh accumulator for related
+ /// hash tables, such as the partial-skip table.
+ aggregate_expr: Arc<AggregateFunctionExpr>,
+
+ /// Arguments to pass to this accumulator.
+ ///
+ /// Example: `CORR(x, y)` stores two expressions here, while `SUM(x)` stores one.
+ arguments: Vec<Arc<dyn PhysicalExpr>>,
+
+ /// Optional `FILTER` expression for this accumulator.
+ ///
+ /// Example: `SUM(x) FILTER (WHERE x > 10)` stores the `x > 10` predicate.
+ filter: Option<Arc<dyn PhysicalExpr>>,
+
+ /// Accumulator state for all groups for one aggregate expression.
+ accumulator: Box<dyn GroupsAccumulator>,
+}
+
+/// Evaluated aggregate arguments and filter for one input batch.
+///
+/// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
+/// and `x > 0`.
+///
+/// These arrays can be passed directly to [`GroupsAccumulator`].
+pub(super) struct EvaluatedAccumulatorArgs {
+ /// Evaluated argument arrays. Some aggregate functions take multiple arguments.
+ pub(super) arguments: Vec<ArrayRef>,
+ /// Evaluated filter array, `Some` if the aggregate has a `FILTER` expression.
+ pub(super) filter: Option<ArrayRef>,
+}
+
+/// Evaluated all group by keys and accumulator args.
+///
+/// e.g., `select k+1, sum(v*v) from t group by (k+1)`, this function evaluates
+/// `k+1`, `v*v`
+pub(super) struct EvaluatedAggregateBatch {
+ /// One entry per grouping set; each entry contains all evaluated group key
+ /// arrays for the current input batch.
+ pub(super) grouping_set_args: Vec<Vec<ArrayRef>>,
+
+ /// Evaluated arguments and filters, one entry per aggregate expression.
+ pub(super) accumulator_args: Vec<EvaluatedAccumulatorArgs>,
+}
+
+/// Buffer for the aggregate hash table's group keys and accumulator states.
+///
+/// It accumulates input during aggregation and emits final results during the
+/// outputting stage.
+///
+/// [`GroupValues`] stores the physical group-key layout, while
+/// [`GroupsAccumulator`] stores per-group aggregate state.
+pub(super) struct AggregateHashTableBuffer {
+ /// GROUP BY expressions evaluated for each input batch.
+ pub(super) group_by: Arc<PhysicalGroupBy>,
+
+ /// Interned group keys. Accumulator state is stored separately by group index.
+ pub(super) group_values: Box<dyn GroupValues>,
+
+ /// Group index for each row in the current input batch.
+ ///
+ /// Each value indexes into `group_values`, and the same index is used by every
+ /// accumulator to update that group's aggregate state.
+ pub(super) batch_group_indices: Vec<usize>,
+
+ /// One item per aggregate expression.
+ ///
+ /// Example: `COUNT(x), SUM(y)` creates two items. Each item owns the input
+ /// expressions, optional filter, and accumulator state for all groups.
+ pub(super) accumulators: Vec<HashAggregateAccumulator>,
+}
+
+pub(super) enum AggregateHashTableState {
+ Building(AggregateHashTableBuffer),
+ Outputting(AggregateHashTableBuffer),
+ Done,
+}
+
+impl HashAggregateAccumulator {
+ fn new(
+ aggregate_expr: Arc<AggregateFunctionExpr>,
+ arguments: Vec<Arc<dyn PhysicalExpr>>,
+ filter: Option<Arc<dyn PhysicalExpr>>,
+ accumulator: Box<dyn GroupsAccumulator>,
+ ) -> Self {
+ Self {
+ aggregate_expr,
+ arguments,
+ filter,
+ accumulator,
+ }
+ }
+
+ /// Construct a new accumulator with the same definition, but with empty internal
+ /// state buffers (empty [`GroupsAccumulator`]).
+ pub(super) fn empty_like(&self) -> Result<Self> {
+ let accumulator = create_group_accumulator(&self.aggregate_expr)?;
+ Ok(Self::new(
+ Arc::clone(&self.aggregate_expr),
+ self.arguments.clone(),
+ self.filter.clone(),
+ accumulator,
+ ))
+ }
+
+ /// Evaluate aggregate arguments and filter for one input batch.
+ ///
+ /// For example, `AVG(x + 1) FILTER (WHERE x > 0)` evaluates both `x + 1`
+ /// and `x > 0`.
+ ///
+ /// These arrays can be passed directly to [`GroupsAccumulator`] next.
+ fn evaluate_acc_args(&self, batch: &RecordBatch) -> Result<EvaluatedAccumulatorArgs> {
+ let arguments = self
+ .arguments
+ .iter()
+ .map(|expr| {
+ expr.evaluate(batch)
+ .and_then(|value| value.into_array(batch.num_rows()))
+ })
+ .collect::<Result<_>>()?;
+
+ let filter = self
+ .filter
+ .as_ref()
+ .map(|filter| {
+ filter
+ .evaluate(batch)
+ .and_then(|value| value.into_array(batch.num_rows()))
+ })
+ .transpose()?;
+
+ Ok(EvaluatedAccumulatorArgs { arguments, filter })
+ }
+
+ pub(super) fn update_batch(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<()> {
+ let filter = values.filter.as_ref().map(|filter| filter.as_boolean());
+ self.accumulator.update_batch(
+ &values.arguments,
+ group_indices,
+ filter,
+ total_num_groups,
+ )
+ }
+
+ pub(super) fn merge_batch(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<()> {
+ debug_assert!(values.filter.is_none());
+ self.accumulator
+ .merge_batch(&values.arguments, group_indices, total_num_groups)
+ }
+
+ /// Evaluating final aggregate results according to `EmitTo`, and reset inner
+ /// states. (e.g. after `evaluate(EmitTo::All)`, it returns all accumulated groups
+ /// , and clear the inner buffers)
+ pub(super) fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
+ self.accumulator.evaluate(emit_to)
+ }
+
+ /// Evaluating partial aggregate results according to `EmitTo`, and reset inner
+ /// states. (e.g. after `state(EmitTo::All)`, it returns all accumulated groups
+ /// , and clear the inner buffers)
+ pub(super) fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
+ self.accumulator.state(emit_to)
+ }
+
+ pub(super) fn supports_convert_to_state(&self) -> bool {
+ self.accumulator.supports_convert_to_state()
+ }
+
+ pub(super) fn convert_to_state(
+ &mut self,
+ values: &EvaluatedAccumulatorArgs,
+ ) -> Result<Vec<ArrayRef>> {
+ let opt_filter = values.filter.as_ref().map(|filter| filter.as_boolean());
+ self.accumulator
+ .convert_to_state(&values.arguments, opt_filter)
+ }
+
+ pub(super) fn null_arguments(
+ &self,
+ input_schema: &SchemaRef,
+ ) -> Result<Vec<ArrayRef>> {
+ self.arguments
+ .iter()
+ .map(|expr| {
+ let data_type = expr.data_type(input_schema)?;
+ Ok(new_null_array(&data_type, 1))
+ })
+ .collect()
+ }
+}
+
+impl AggregateHashTableState {
+ pub(super) fn building(&self) -> &AggregateHashTableBuffer {
+ let Self::Building(state) = self else {
+ unreachable!("hash aggregate table is not building")
+ };
+ state
+ }
+
+ pub(super) fn building_mut(&mut self) -> &mut AggregateHashTableBuffer {
+ let Self::Building(state) = self else {
+ unreachable!("hash aggregate table is not building")
+ };
+ state
+ }
+}
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
new file mode 100644
index 0000000000..415694d8c2
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::Arc;
+
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, internal_err};
+
+use crate::aggregates::AggregateExec;
+
+use super::common::{
+ AggregateHashTable, AggregateHashTableState, FinalMarker, emit_to_for_batch_size,
+};
+
+/// Methods specific to the aggregate hash table used in the final aggregation stage.
+impl AggregateHashTable<FinalMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ output_schema,
+ batch_size,
+ vec![None; agg.aggr_expr.len()],
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and final aggregate values.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ let output_schema = Arc::clone(&self.output_schema);
+ let batch_size = self.batch_size;
+ match &mut self.state {
+ AggregateHashTableState::Outputting(state) => {
+ if state.group_values.is_empty() {
+ self.state = AggregateHashTableState::Done;
+ return Ok(None);
+ }
+
+ let emit_to =
+ emit_to_for_batch_size(batch_size, state.group_values.len());
+ let timer = self.group_by_metrics.emitting_time.timer();
+ let mut output = state.group_values.emit(emit_to)?;
+
+ for acc in state.accumulators.iter_mut() {
+ output.push(acc.evaluate(emit_to)?);
+ }
+ let done = state.group_values.is_empty();
+ drop(timer);
+
+ let batch = RecordBatch::try_new(output_schema, output)?;
+ debug_assert!(batch.num_rows() > 0);
+ if done {
+ self.state = AggregateHashTableState::Done;
+ }
+ Ok(Some(batch))
+ }
+ AggregateHashTableState::Done => Ok(None),
+ AggregateHashTableState::Building(_) => {
+ internal_err!("next_output_batch must be called in the outputting state")
+ }
+ }
+ }
+
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+ let state = self.state.building_mut();
+
+ let timer = self.group_by_metrics.aggregation_time.timer();
+ for group_values in &evaluated_batch.grouping_set_args {
+ state
+ .group_values
+ .intern(group_values, &mut state.batch_group_indices)?;
+ let group_indices = &state.batch_group_indices;
+ let total_num_groups = state.group_values.len();
+
+ for (acc, values) in state
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ acc.merge_batch(values, group_indices, total_num_groups)?;
+ }
+ }
+ drop(timer);
+
+ Ok(())
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.start_outputting();
+ Ok(())
+ }
+}
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs
new file mode 100644
index 0000000000..eb152f4128
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/mod.rs
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod common;
+mod final_table;
+mod partial_table;
+
+pub(super) use common::{
+ AggregateHashTable, FinalMarker, PartialMarker, PartialSkipMarker,
+};
diff --git a/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
new file mode 100644
index 0000000000..fd3cf801cf
--- /dev/null
+++ b/datafusion/physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, BooleanArray, new_null_array};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
+
+use crate::aggregates::group_values::new_group_values;
+use crate::aggregates::order::GroupOrdering;
+use crate::aggregates::{AggregateExec, group_id_array, max_duplicate_ordinal};
+
+use super::common::{
+ AggregateHashTable, AggregateHashTableBuffer, AggregateHashTableState,
+ EvaluatedAccumulatorArgs, HashAggregateAccumulator, PartialMarker, PartialSkipMarker,
+ emit_to_for_batch_size,
+};
+
+/// Methods specific to the aggregate hash table used in the partial aggregation stage.
+impl AggregateHashTable<PartialMarker> {
+ pub(in crate::aggregates) fn new(
+ agg: &AggregateExec,
+ partition: usize,
+ output_schema: SchemaRef,
+ batch_size: usize,
+ ) -> Result<Self> {
+ Self::new_with_filters(
+ agg,
+ partition,
+ output_schema,
+ batch_size,
+ agg.filter_expr.iter().cloned().collect(),
+ )
+ }
+
+ /// Emits the next batch of aggregated group keys and aggregate states.
+ ///
+ /// The output batch size is determined by `self.batch_size`.
+ ///
+ /// Returns `Some(batch)` for each emitted batch, `None` when output is
+ /// exhausted, and an internal error if polled in the `Building` state.
+ pub(in crate::aggregates) fn next_output_batch(
+ &mut self,
+ ) -> Result<Option<RecordBatch>> {
+ let output_schema = Arc::clone(&self.output_schema);
+ let batch_size = self.batch_size;
+ match &mut self.state {
+ AggregateHashTableState::Outputting(state) => {
+ if state.group_values.is_empty() {
+ self.state = AggregateHashTableState::Done;
+ return Ok(None);
+ }
+
+ let emit_to =
+ emit_to_for_batch_size(batch_size, state.group_values.len());
+ let timer = self.group_by_metrics.emitting_time.timer();
+ let mut output = state.group_values.emit(emit_to)?;
+
+ for acc in state.accumulators.iter_mut() {
+ output.extend(acc.state(emit_to)?);
+ }
+ let done = state.group_values.is_empty();
+ drop(timer);
+
+ let batch = RecordBatch::try_new(output_schema, output)?;
+ debug_assert!(batch.num_rows() > 0);
+ if done {
+ self.state = AggregateHashTableState::Done;
+ }
+ Ok(Some(batch))
+ }
+ AggregateHashTableState::Done => Ok(None),
+ AggregateHashTableState::Building(_) => {
+ internal_err!("next_output_batch must be called in the outputting state")
+ }
+ }
+ }
+
+ pub(in crate::aggregates) fn can_skip_aggregation(&self) -> bool {
+ self.state
+ .building()
+ .accumulators
+ .iter()
+ .all(|acc| acc.supports_convert_to_state())
+ }
+
+ /// In skip-partial-aggregation optimization, when a decision has been made to skip
+ /// partial stage, build a typed hash table only for aggregation state conversion
+ /// row-by-row.
+ pub(in crate::aggregates) fn partial_skip_table(
+ &self,
+ ) -> Result<AggregateHashTable<PartialSkipMarker>> {
+ let state = self.state.building();
+ let group_schema = state.group_by.group_schema(&self.input_schema)?;
+ let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
+ let accumulators = state
+ .accumulators
+ .iter()
+ .map(HashAggregateAccumulator::empty_like)
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(AggregateHashTable {
+ group_by_metrics: self.group_by_metrics.clone(),
+ input_schema: Arc::clone(&self.input_schema),
+ output_schema: Arc::clone(&self.output_schema),
+ batch_size: self.batch_size,
+ state: AggregateHashTableState::Building(AggregateHashTableBuffer {
+ group_by: Arc::clone(&state.group_by),
+ group_values,
+ batch_group_indices: Default::default(),
+ accumulators,
+ }),
+ _mode: PhantomData,
+ })
+ }
+
+ pub(in crate::aggregates) fn aggregate_batch(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<()> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+ let state = self.state.building_mut();
+
+ let _timer = self.group_by_metrics.aggregation_time.timer();
+ for group_values in &evaluated_batch.grouping_set_args {
+ state
+ .group_values
+ .intern(group_values, &mut state.batch_group_indices)?;
+ let group_indices = &state.batch_group_indices;
+ let total_num_groups = state.group_values.len();
+
+ for (acc, values) in state
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ acc.update_batch(values, group_indices, total_num_groups)?;
+ }
+ }
+
+ Ok(())
+ }
+
+ pub(in crate::aggregates) fn start_output(&mut self) -> Result<()> {
+ self.init_empty_grouping_sets()?;
+ self.start_outputting();
+ Ok(())
+ }
+
+ /// Creates the required empty grouping-set rows when the input is empty.
+ ///
+ /// For example, this query must still produce one grand-total group even if
+ /// `t` has no rows:
+ ///
+ /// ```sql
+ /// SELECT COUNT(v)
+ /// FROM t
+ /// GROUP BY GROUPING SETS (());
+ /// ```
+ ///
+ /// The synthetic row is filtered out before accumulator update so aggregates
+ /// see the same state they would see for an empty input, rather than a real
+ /// null-valued row.
+ fn init_empty_grouping_sets(&mut self) -> Result<()> {
+ let state = self.state.building_mut();
+ if !state.group_by.has_grouping_set() || !state.group_values.is_empty() {
+ return Ok(());
+ }
+
+ let max_ordinal = max_duplicate_ordinal(state.group_by.groups());
+ let mut ordinals: HashMap<&[bool], usize> = HashMap::new();
+ let group_schema = state.group_by.group_schema(&self.input_schema)?;
+ let n_expr = state.group_by.expr().len();
+ let mut any_interned = false;
+
+ for group in state.group_by.groups() {
+ let ordinal = {
+ let entry = ordinals.entry(group.as_slice()).or_insert(0);
+ let ordinal = *entry;
+ *entry += 1;
+ ordinal
+ };
+
+ if !group.iter().all(|&is_null| is_null) {
+ continue;
+ }
+
+ let mut cols: Vec<ArrayRef> = group_schema
+ .fields()
+ .iter()
+ .take(n_expr)
+ .map(|field| new_null_array(field.data_type(), 1))
+ .collect();
+ cols.push(group_id_array(group, ordinal, max_ordinal, 1)?);
+
+ state
+ .group_values
+ .intern(&cols, &mut state.batch_group_indices)?;
+ any_interned = true;
+ }
+
+ if any_interned {
+ let total_groups = state.group_values.len();
+ let false_filter = BooleanArray::from(vec![false]);
+ for acc in state.accumulators.iter_mut() {
+ let null_args = acc.null_arguments(&self.input_schema)?;
+ let values = EvaluatedAccumulatorArgs {
+ arguments: null_args,
+ filter: Some(Arc::new(false_filter.clone())),
+ };
+ acc.update_batch(&values, &[0], total_groups)?;
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl AggregateHashTable<PartialSkipMarker> {
+ pub(in crate::aggregates) fn convert_batch_to_state(
+ &mut self,
+ batch: &RecordBatch,
+ ) -> Result<RecordBatch> {
+ let evaluated_batch = self.evaluate_batch(batch)?;
+
+ assert_eq_or_internal_err!(
+ evaluated_batch.grouping_set_args.len(),
+ 1,
+ "group_values expected to have single element"
+ );
+ let mut output = evaluated_batch
+ .grouping_set_args
+ .into_iter()
+ .next()
+ .unwrap_or_default();
+
+ let state = self.state.building_mut();
+ for (acc, values) in state
+ .accumulators
+ .iter_mut()
+ .zip(evaluated_batch.accumulator_args.iter())
+ {
+ output.extend(acc.convert_to_state(values)?);
+ }
+
+ Ok(RecordBatch::try_new(
+ Arc::clone(&self.output_schema),
+ output,
+ )?)
+ }
+}
diff --git a/datafusion/physical-plan/src/aggregates/hash_aggregate.rs b/datafusion/physical-plan/src/aggregates/hash_aggregate.rs
index 59ee09912f..4c8756c0e8 100644
--- a/datafusion/physical-plan/src/aggregates/hash_aggregate.rs
+++ b/datafusion/physical-plan/src/aggregates/hash_aggregate.rs
@@ -37,7 +37,9 @@ use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
use futures::stream::{Stream, StreamExt};
use super::AggregateExec;
-use super::hash_table::{AggregateHashTable, Final, Partial, PartialSkip};
+use super::aggregate_hash_table::{
+ AggregateHashTable, FinalMarker, PartialMarker, PartialSkipMarker,
+};
use super::skip_partial::SkipAggregationProbe;
use crate::metrics::{
BaselineMetrics, MetricBuilder, MetricCategory, RecordOutput, SpillMetrics,
@@ -131,25 +133,25 @@ pub(crate) struct PartialHashAggregateStream {
group_values_soft_limit: Option<usize>,
/// Tracks the high-level stream lifecycle. The hash table owns the lower-level
- /// state for materializing and slicing output batches.
+ /// state for emitting output batches.
state: Option<PartialHashAggregateState>,
}
/// States for partial hash aggregation processing.
enum PartialHashAggregateState {
ReadingInput {
- hash_table: AggregateHashTable<Partial>,
+ hash_table: AggregateHashTable<PartialMarker>,
},
ProducingOutput {
- hash_table: AggregateHashTable<Partial>,
+ hash_table: AggregateHashTable<PartialMarker>,
/// If `None`, partial skip was never triggered and this state will
/// finish in `Done`. If `Some`, partial skip has triggered and the
/// stream will move to `SkippingAggregation` after these accumulated
/// groups are emitted.
- skip_hash_table: Option<AggregateHashTable<PartialSkip>>,
+ skip_hash_table: Option<AggregateHashTable<PartialSkipMarker>>,
},
SkippingAggregation {
- hash_table: AggregateHashTable<PartialSkip>,
+ hash_table: AggregateHashTable<PartialSkipMarker>,
},
Done,
}
@@ -161,7 +163,7 @@ type PartialHashAggregateStateTransition = ControlFlow<
>;
impl PartialHashAggregateState {
- fn hash_table(&self) -> &AggregateHashTable<Partial> {
+ fn hash_table(&self) -> &AggregateHashTable<PartialMarker> {
match self {
Self::ReadingInput { hash_table }
| Self::ProducingOutput { hash_table, .. } => hash_table,
@@ -171,7 +173,7 @@ impl PartialHashAggregateState {
}
}
- fn hash_table_mut(&mut self) -> &mut AggregateHashTable<Partial> {
+ fn hash_table_mut(&mut self) -> &mut AggregateHashTable<PartialMarker> {
match self {
Self::ReadingInput { hash_table }
| Self::ProducingOutput { hash_table, .. } => hash_table,
@@ -203,7 +205,7 @@ pub(crate) struct FinalHashAggregateStream {
group_values_soft_limit: Option<usize>,
/// Tracks the high-level stream lifecycle. The hash table owns the lower-level
- /// state for materializing and slicing output batches.
+ /// state for emitting output batches.
state: Option<FinalHashAggregateState>,
}
@@ -212,10 +214,10 @@ pub(crate) struct FinalHashAggregateStream {
// the future.
enum FinalHashAggregateState {
ReadingInput {
- hash_table: AggregateHashTable<Final>,
+ hash_table: AggregateHashTable<FinalMarker>,
},
ProducingOutput {
- hash_table: AggregateHashTable<Final>,
+ hash_table: AggregateHashTable<FinalMarker>,
},
Done,
}
@@ -227,7 +229,7 @@ type FinalHashAggregateStateTransition = ControlFlow<
>;
impl FinalHashAggregateState {
- fn hash_table(&self) -> &AggregateHashTable<Final> {
+ fn hash_table(&self) -> &AggregateHashTable<FinalMarker> {
match self {
Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {
hash_table
@@ -236,7 +238,7 @@ impl FinalHashAggregateState {
}
}
- fn hash_table_mut(&mut self) -> &mut AggregateHashTable<Final> {
+ fn hash_table_mut(&mut self) -> &mut AggregateHashTable<FinalMarker> {
match self {
Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {
hash_table
@@ -245,7 +247,7 @@ impl FinalHashAggregateState {
}
}
- fn into_hash_table(self) -> AggregateHashTable<Final> {
+ fn into_hash_table(self) -> AggregateHashTable<FinalMarker> {
match self {
Self::ReadingInput { hash_table } | Self::ProducingOutput { hash_table } => {
hash_table
@@ -285,7 +287,7 @@ impl PartialHashAggregateStream {
.with_type(metrics::MetricType::Summary)
.ratio_metrics("reduction_factor", partition);
- let hash_table = AggregateHashTable::<Partial>::new(
+ let hash_table = AggregateHashTable::<PartialMarker>::new(
agg,
partition,
Arc::clone(&schema),
@@ -332,7 +334,10 @@ impl PartialHashAggregateStream {
}
/// See comments in [`Self::group_values_soft_limit`] for details.
- fn hit_soft_group_limit(&self, hash_table: &AggregateHashTable<Partial>) -> bool {
+ fn hit_soft_group_limit(
+ &self,
+ hash_table: &AggregateHashTable<PartialMarker>,
+ ) -> bool {
self.group_values_soft_limit
.is_some_and(|limit| limit <= hash_table.building_group_count())
}
@@ -354,7 +359,7 @@ impl PartialHashAggregateStream {
fn start_output(
&mut self,
- hash_table: &mut AggregateHashTable<Partial>,
+ hash_table: &mut AggregateHashTable<PartialMarker>,
close_input: bool,
) -> Result<()> {
if close_input {
@@ -757,7 +762,7 @@ impl FinalHashAggregateStream {
// Preserve the existing aggregate metric surface for this plan node.
let _spill_metrics = SpillMetrics::new(&agg.metrics, partition);
- let hash_table = AggregateHashTable::<Final>::new(
+ let hash_table = AggregateHashTable::<FinalMarker>::new(
agg,
partition,
Arc::clone(&schema),
@@ -779,12 +784,15 @@ impl FinalHashAggregateStream {
}
/// See comments in [`Self::group_values_soft_limit`] for details.
- fn hit_soft_group_limit(&self, hash_table: &AggregateHashTable<Final>) -> bool {
+ fn hit_soft_group_limit(&self, hash_table: &AggregateHashTable<FinalMarker>) -> bool {
self.group_values_soft_limit
.is_some_and(|limit| limit <= hash_table.building_group_count())
}
- fn start_output(&mut self, hash_table: &mut AggregateHashTable<Final>) -> Result<()> {
+ fn start_output(
+ &mut self,
+ hash_table: &mut AggregateHashTable<FinalMarker>,
+ ) -> Result<()> {
let input_schema = self.input.schema();
self.input = Box::pin(EmptyRecordBatchStream::new(input_schema));
hash_table.start_output()
diff --git a/datafusion/physical-plan/src/aggregates/hash_table.rs b/datafusion/physical-plan/src/aggregates/hash_table.rs
deleted file mode 100644
index e6b2fa22c1..0000000000
--- a/datafusion/physical-plan/src/aggregates/hash_table.rs
+++ /dev/null
@@ -1,704 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::collections::HashMap;
-use std::marker::PhantomData;
-use std::sync::Arc;
-
-use arrow::array::{ArrayRef, AsArray, BooleanArray, new_null_array};
-use arrow::datatypes::SchemaRef;
-use arrow::record_batch::RecordBatch;
-use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
-use datafusion_execution::memory_pool::proxy::VecAllocExt;
-use datafusion_expr::{EmitTo, GroupsAccumulator};
-use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
-
-use super::group_values::{GroupByMetrics, GroupValues, new_group_values};
-use super::order::GroupOrdering;
-use super::row_hash::create_group_accumulator;
-use super::{
- AggregateExec, PhysicalGroupBy, aggregate_expressions, evaluate_group_by,
- group_id_array, max_duplicate_ordinal,
-};
-use crate::PhysicalExpr;
-
-/// Marker for raw rows -> partial state aggregation.
-pub(super) struct Partial;
-/// Marker for raw rows -> partial state conversion without aggregation.
-pub(super) struct PartialSkip;
-/// Marker for partial state -> final value aggregation.
-pub(super) struct Final;
-
-/// Grouped hash table shared by the partial and final paths.
-///
-/// While building, it consumes input batches and updates group / accumulator
-/// state. While outputting, it incrementally output the materialized batches.
-///
-/// # Marker Type
-/// `AggrMode` selects the aggregate semantics.
-///
-/// e.g. `AggregateHashTable::<Partial>::new(...)` creates an aggregate hash table
-/// for the partial hash aggregate stage, the input schema is raw rows and output
-/// schema is intermediate states.
-///
-/// It is a zero-sized compile-time marker, so each stage keeps its update logic
-/// in a separate impl block, to make the behavior difference explicit.
-pub(super) struct AggregateHashTable<AggrMode> {
- /// Grouping and accumulator-specific timing metrics.
- group_by_metrics: GroupByMetrics,
-
- /// Raw input schema, used to evaluate expressions and synthesize empty
- /// grouping-set rows.
- input_schema: SchemaRef,
-
- /// Output schema: group columns followed by aggregate state or final values.
- output_schema: SchemaRef,
-
- /// Maximum rows per emitted output batch.
- batch_size: usize,
-
- /// Lifecycle-specific state: building stage / outputting stage
- state: AggregateHashTableState,
-
- _mode: PhantomData<AggrMode>,
-}
-
-struct HashAggregateAccumulator {
- /// Aggregate expression used to create a fresh accumulator for related
- /// hash tables, such as the partial-skip table.
- aggregate_expr: Arc<AggregateFunctionExpr>,
-
- /// Arguments to pass to this accumulator.
- ///
- /// Example: `CORR(x, y)` stores two expressions here, while `SUM(x)` stores one.
- arguments: Vec<Arc<dyn PhysicalExpr>>,
-
- /// Optional `FILTER` expression for this accumulator.
- ///
- /// Example: `SUM(x) FILTER (WHERE x > 10)` stores the `x > 10` predicate.
- filter: Option<Arc<dyn PhysicalExpr>>,
-
- /// Accumulator state for all groups for one aggregate expression.
- accumulator: Box<dyn GroupsAccumulator>,
-}
-
-struct EvaluatedHashAggregateAccumulator {
- arguments: Vec<ArrayRef>,
- filter: Option<ArrayRef>,
-}
-
-/// Evaluated all group by keys and accumulator args.
-///
-/// e.g., `select k+1, sum(v*v) from t group by (k+1)`, this function evaluates
-/// `k+1`, `v*v`
-struct EvaluatedAggregateBatch {
- /// One entry per grouping set; each entry contains all evaluated group key
- /// arrays for the current input batch.
- grouping_set_args: Vec<Vec<ArrayRef>>,
-
- /// Evaluated arguments and filters, one entry per aggregate expression.
- accumulator_args: Vec<EvaluatedHashAggregateAccumulator>,
-}
-
-/// Hash table state while grouped aggregation is consuming input.
-///
-/// This owns the coupled state for:
-/// - evaluating group keys,
-/// - interning each distinct group,
-/// - mapping each input row to its group index,
-/// - evaluating aggregate inputs,
-/// - updating per-group accumulator state.
-struct BuildingHashTableState {
- /// GROUP BY expressions evaluated for each input batch.
- group_by: Arc<PhysicalGroupBy>,
-
- /// Interned group keys. Accumulator state is stored separately by group index.
- group_values: Box<dyn GroupValues>,
-
- /// Group index for each row in the current input batch.
- ///
- /// Each value indexes into `group_values`, and the same index is used by every
- /// accumulator to update that group's aggregate state.
- batch_group_indices: Vec<usize>,
-
- /// One item per aggregate expression.
- ///
- /// Example: `COUNT(x), SUM(y)` creates two items. Each item owns the input
- /// expressions, optional filter, and accumulator state for all groups.
- accumulators: Vec<HashAggregateAccumulator>,
-}
-
-enum AggregateHashTableState {
- Building(BuildingHashTableState),
- Outputting {
- output_batch: Option<RecordBatch>,
- output_batch_offset: usize,
- },
- Done,
-}
-
-impl HashAggregateAccumulator {
- fn new(
- aggregate_expr: Arc<AggregateFunctionExpr>,
- arguments: Vec<Arc<dyn PhysicalExpr>>,
- filter: Option<Arc<dyn PhysicalExpr>>,
- accumulator: Box<dyn GroupsAccumulator>,
- ) -> Self {
- Self {
- aggregate_expr,
- arguments,
- filter,
- accumulator,
- }
- }
-
- fn empty_like(&self) -> Result<Self> {
- let accumulator = create_group_accumulator(&self.aggregate_expr)?;
- Ok(Self::new(
- Arc::clone(&self.aggregate_expr),
- self.arguments.clone(),
- self.filter.clone(),
- accumulator,
- ))
- }
-
- fn evaluate(&self, batch: &RecordBatch) -> Result<EvaluatedHashAggregateAccumulator> {
- let arguments = self
- .arguments
- .iter()
- .map(|expr| {
- expr.evaluate(batch)
- .and_then(|value| value.into_array(batch.num_rows()))
- })
- .collect::<Result<_>>()?;
-
- let filter = self
- .filter
- .as_ref()
- .map(|filter| {
- filter
- .evaluate(batch)
- .and_then(|value| value.into_array(batch.num_rows()))
- })
- .transpose()?;
-
- Ok(EvaluatedHashAggregateAccumulator { arguments, filter })
- }
-
- fn update_batch(
- &mut self,
- values: &EvaluatedHashAggregateAccumulator,
- group_indices: &[usize],
- total_num_groups: usize,
- ) -> Result<()> {
- let filter = values.filter.as_ref().map(|filter| filter.as_boolean());
- self.accumulator.update_batch(
- &values.arguments,
- group_indices,
- filter,
- total_num_groups,
- )
- }
-
- fn merge_batch(
- &mut self,
- values: &EvaluatedHashAggregateAccumulator,
- group_indices: &[usize],
- total_num_groups: usize,
- ) -> Result<()> {
- debug_assert!(values.filter.is_none());
- self.accumulator
- .merge_batch(&values.arguments, group_indices, total_num_groups)
- }
-
- fn evaluate_final(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
- self.accumulator.evaluate(emit_to)
- }
-
- fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
- self.accumulator.state(emit_to)
- }
-
- fn supports_convert_to_state(&self) -> bool {
- self.accumulator.supports_convert_to_state()
- }
-
- fn convert_to_state(
- &mut self,
- values: &EvaluatedHashAggregateAccumulator,
- ) -> Result<Vec<ArrayRef>> {
- let opt_filter = values.filter.as_ref().map(|filter| filter.as_boolean());
- self.accumulator
- .convert_to_state(&values.arguments, opt_filter)
- }
-
- fn null_arguments(&self, input_schema: &SchemaRef) -> Result<Vec<ArrayRef>> {
- self.arguments
- .iter()
- .map(|expr| {
- let data_type = expr.data_type(input_schema)?;
- Ok(new_null_array(&data_type, 1))
- })
- .collect()
- }
-}
-
-impl AggregateHashTableState {
- fn building(&self) -> &BuildingHashTableState {
- let Self::Building(state) = self else {
- unreachable!("hash aggregate table is not building")
- };
- state
- }
-
- fn building_mut(&mut self) -> &mut BuildingHashTableState {
- let Self::Building(state) = self else {
- unreachable!("hash aggregate table is not building")
- };
- state
- }
-}
-
-impl<Mode> AggregateHashTable<Mode> {
- fn new_with_filters(
- agg: &AggregateExec,
- partition: usize,
- output_schema: SchemaRef,
- batch_size: usize,
- filters: Vec<Option<Arc<dyn PhysicalExpr>>>,
- ) -> Result<Self> {
- let input_schema = agg.input().schema();
- let aggregate_arguments = aggregate_expressions(
- &agg.aggr_expr,
- &agg.mode,
- agg.group_by.num_group_exprs(),
- )?;
- let accumulators: Vec<_> = agg
- .aggr_expr
- .iter()
- .zip(aggregate_arguments)
- .zip(filters)
- .map(|((agg_expr, arguments), filter)| {
- let accumulator = create_group_accumulator(agg_expr)?;
- Ok(HashAggregateAccumulator::new(
- Arc::clone(agg_expr),
- arguments,
- filter,
- accumulator,
- ))
- })
- .collect::<Result<_>>()?;
-
- let group_schema = agg.group_by.group_schema(&input_schema)?;
- let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
-
- Ok(Self {
- group_by_metrics: GroupByMetrics::new(&agg.metrics, partition),
- input_schema,
- output_schema,
- batch_size,
- state: AggregateHashTableState::Building(BuildingHashTableState {
- group_by: Arc::clone(&agg.group_by),
- group_values,
- batch_group_indices: Default::default(),
- accumulators,
- }),
- _mode: PhantomData,
- })
- }
-
- /// See comments in [`EvaluatedAggregateBatch`]
- fn evaluate_batch(&self, batch: &RecordBatch) -> Result<EvaluatedAggregateBatch> {
- let state = self.state.building();
- let timer = self.group_by_metrics.time_calculating_group_ids.timer();
- // outer vec: one per each grouping set
- // inner vec: all group by exprs for the current grouping set
- let grouping_set_args = evaluate_group_by(&state.group_by, batch)?;
- drop(timer);
-
- let timer = self.group_by_metrics.aggregate_arguments_time.timer();
- // The evaluated args for each accumulator
- let accumulator_args = self
- .state
- .building()
- .accumulators
- .iter()
- .map(|acc| acc.evaluate(batch))
- .collect::<Result<Vec<_>>>()?;
- drop(timer);
-
- Ok(EvaluatedAggregateBatch {
- grouping_set_args,
- accumulator_args,
- })
- }
-
- pub(super) fn memory_size(&self) -> usize {
- match &self.state {
- AggregateHashTableState::Building(state) => {
- let acc = state
- .accumulators
- .iter()
- .map(|acc| acc.accumulator.size())
- .sum::<usize>();
-
- acc + state.group_values.size()
- + state.batch_group_indices.allocated_size()
- }
- AggregateHashTableState::Outputting { output_batch, .. } => {
- output_batch_memory_size(output_batch)
- }
- AggregateHashTableState::Done => 0,
- }
- }
-
- /// How many distinct groups has been accumulated now.
- pub(super) fn building_group_count(&self) -> usize {
- self.state.building().group_values.len()
- }
-
- pub(super) fn is_building(&self) -> bool {
- matches!(self.state, AggregateHashTableState::Building(_))
- }
-
- pub(super) fn is_done(&self) -> bool {
- matches!(self.state, AggregateHashTableState::Done)
- }
-
- fn set_output_batch(&mut self, output_batch: Option<RecordBatch>) {
- self.state = AggregateHashTableState::Outputting {
- output_batch,
- output_batch_offset: 0,
- };
- }
-
- pub(super) fn next_output_batch(&mut self) -> Result<Option<RecordBatch>> {
- match std::mem::replace(&mut self.state, AggregateHashTableState::Done) {
- AggregateHashTableState::Outputting {
- output_batch,
- mut output_batch_offset,
- } => {
- let Some(batch) = output_batch.as_ref() else {
- return Ok(None);
- };
-
- let num_rows = batch.num_rows();
- if output_batch_offset >= num_rows {
- return Ok(None);
- }
-
- debug_assert!(self.batch_size > 0);
- let output_len =
- self.batch_size.max(1).min(num_rows - output_batch_offset);
- let output = batch.slice(output_batch_offset, output_len);
- output_batch_offset += output_len;
-
- if output_batch_offset == num_rows {
- self.state = AggregateHashTableState::Done;
- } else {
- self.state = AggregateHashTableState::Outputting {
- output_batch,
- output_batch_offset,
- };
- }
-
- debug_assert!(output.num_rows() > 0);
- debug_assert!(output.num_rows() <= self.batch_size.max(1));
- Ok(Some(output))
- }
- _ => {
- self.state = AggregateHashTableState::Done;
- internal_err!("next_output_batch must be called in the outputting state")
- }
- }
- }
-}
-
-impl AggregateHashTable<Partial> {
- pub(super) fn new(
- agg: &AggregateExec,
- partition: usize,
- output_schema: SchemaRef,
- batch_size: usize,
- ) -> Result<Self> {
- Self::new_with_filters(
- agg,
- partition,
- output_schema,
- batch_size,
- agg.filter_expr.iter().cloned().collect(),
- )
- }
-
- pub(super) fn can_skip_aggregation(&self) -> bool {
- self.state
- .building()
- .accumulators
- .iter()
- .all(|acc| acc.supports_convert_to_state())
- }
-
- /// In skip-partial-aggregation optimization, when a decision has made to skip
- /// partial stage, build a typed hash table only for aggregation state conversion
- /// row-by-row.
- pub(super) fn partial_skip_table(&self) -> Result<AggregateHashTable<PartialSkip>> {
- let state = self.state.building();
- let group_schema = state.group_by.group_schema(&self.input_schema)?;
- let group_values = new_group_values(group_schema, &GroupOrdering::None)?;
- let accumulators = state
- .accumulators
- .iter()
- .map(HashAggregateAccumulator::empty_like)
- .collect::<Result<Vec<_>>>()?;
-
- Ok(AggregateHashTable {
- group_by_metrics: self.group_by_metrics.clone(),
- input_schema: Arc::clone(&self.input_schema),
- output_schema: Arc::clone(&self.output_schema),
- batch_size: self.batch_size,
- state: AggregateHashTableState::Building(BuildingHashTableState {
- group_by: Arc::clone(&state.group_by),
- group_values,
- batch_group_indices: Default::default(),
- accumulators,
- }),
- _mode: PhantomData,
- })
- }
-
- pub(super) fn aggregate_batch(&mut self, batch: &RecordBatch) -> Result<()> {
- let evaluated_batch = self.evaluate_batch(batch)?;
- let state = self.state.building_mut();
-
- let timer = self.group_by_metrics.aggregation_time.timer();
- for group_values in &evaluated_batch.grouping_set_args {
- state
- .group_values
- .intern(group_values, &mut state.batch_group_indices)?;
- let group_indices = &state.batch_group_indices;
- let total_num_groups = state.group_values.len();
-
- for (acc, values) in state
- .accumulators
- .iter_mut()
- .zip(evaluated_batch.accumulator_args.iter())
- {
- acc.update_batch(values, group_indices, total_num_groups)?;
- }
- }
- drop(timer);
-
- Ok(())
- }
-
- pub(super) fn start_output(&mut self) -> Result<()> {
- self.init_empty_grouping_sets()?;
- let state = self.state.building_mut();
-
- let output_batch = if state.group_values.is_empty() {
- None
- } else {
- let timer = self.group_by_metrics.emitting_time.timer();
- let mut output = state.group_values.emit(EmitTo::All)?;
-
- for acc in state.accumulators.iter_mut() {
- output.extend(acc.state(EmitTo::All)?);
- }
-
- let batch = RecordBatch::try_new(Arc::clone(&self.output_schema), output)?;
- debug_assert!(batch.num_rows() > 0);
- drop(timer);
- Some(batch)
- };
-
- self.set_output_batch(output_batch);
- Ok(())
- }
-
- /// Creates the required empty grouping-set rows when the input is empty.
- ///
- /// For example, this query must still produce one grand-total group even if
- /// `t` has no rows:
- ///
- /// ```sql
- /// SELECT COUNT(v)
- /// FROM t
- /// GROUP BY GROUPING SETS (());
- /// ```
- ///
- /// The synthetic row is filtered out before accumulator update so aggregates
- /// see the same state they would see for an empty input, rather than a real
- /// null-valued row.
- fn init_empty_grouping_sets(&mut self) -> Result<()> {
- let state = self.state.building_mut();
- if !state.group_by.has_grouping_set() || !state.group_values.is_empty() {
- return Ok(());
- }
-
- let max_ordinal = max_duplicate_ordinal(state.group_by.groups());
- let mut ordinals: HashMap<&[bool], usize> = HashMap::new();
- let group_schema = state.group_by.group_schema(&self.input_schema)?;
- let n_expr = state.group_by.expr().len();
- let mut any_interned = false;
-
- for group in state.group_by.groups() {
- let ordinal = {
- let entry = ordinals.entry(group.as_slice()).or_insert(0);
- let ordinal = *entry;
- *entry += 1;
- ordinal
- };
-
- if !group.iter().all(|&is_null| is_null) {
- continue;
- }
-
- let mut cols: Vec<ArrayRef> = group_schema
- .fields()
- .iter()
- .take(n_expr)
- .map(|field| new_null_array(field.data_type(), 1))
- .collect();
- cols.push(group_id_array(group, ordinal, max_ordinal, 1)?);
-
- state
- .group_values
- .intern(&cols, &mut state.batch_group_indices)?;
- any_interned = true;
- }
-
- if any_interned {
- let total_groups = state.group_values.len();
- let false_filter = BooleanArray::from(vec![false]);
- for acc in state.accumulators.iter_mut() {
- let null_args = acc.null_arguments(&self.input_schema)?;
- let values = EvaluatedHashAggregateAccumulator {
- arguments: null_args,
- filter: Some(Arc::new(false_filter.clone())),
- };
- acc.update_batch(&values, &[0], total_groups)?;
- }
- }
-
- Ok(())
- }
-}
-
-impl AggregateHashTable<PartialSkip> {
- pub(super) fn convert_batch_to_state(
- &mut self,
- batch: &RecordBatch,
- ) -> Result<RecordBatch> {
- let evaluated_batch = self.evaluate_batch(batch)?;
-
- assert_eq_or_internal_err!(
- evaluated_batch.grouping_set_args.len(),
- 1,
- "group_values expected to have single element"
- );
- let mut output = evaluated_batch
- .grouping_set_args
- .into_iter()
- .next()
- .unwrap_or_default();
-
- let state = self.state.building_mut();
- for (acc, values) in state
- .accumulators
- .iter_mut()
- .zip(evaluated_batch.accumulator_args.iter())
- {
- output.extend(acc.convert_to_state(values)?);
- }
-
- Ok(RecordBatch::try_new(
- Arc::clone(&self.output_schema),
- output,
- )?)
- }
-}
-
-impl AggregateHashTable<Final> {
- pub(super) fn new(
- agg: &AggregateExec,
- partition: usize,
- output_schema: SchemaRef,
- batch_size: usize,
- ) -> Result<Self> {
- Self::new_with_filters(
- agg,
- partition,
- output_schema,
- batch_size,
- vec![None; agg.aggr_expr.len()],
- )
- }
-
- pub(super) fn aggregate_batch(&mut self, batch: &RecordBatch) -> Result<()> {
- let evaluated_batch = self.evaluate_batch(batch)?;
- let state = self.state.building_mut();
-
- let timer = self.group_by_metrics.aggregation_time.timer();
- for group_values in &evaluated_batch.grouping_set_args {
- state
- .group_values
- .intern(group_values, &mut state.batch_group_indices)?;
- let group_indices = &state.batch_group_indices;
- let total_num_groups = state.group_values.len();
-
- for (acc, values) in state
- .accumulators
- .iter_mut()
- .zip(evaluated_batch.accumulator_args.iter())
- {
- acc.merge_batch(values, group_indices, total_num_groups)?;
- }
- }
- drop(timer);
-
- Ok(())
- }
-
- pub(super) fn start_output(&mut self) -> Result<()> {
- let state = self.state.building_mut();
- let output_batch = if state.group_values.is_empty() {
- None
- } else {
- let timer = self.group_by_metrics.emitting_time.timer();
- let mut output = state.group_values.emit(EmitTo::All)?;
-
- for acc in state.accumulators.iter_mut() {
- output.push(acc.evaluate_final(EmitTo::All)?);
- }
-
- let batch = RecordBatch::try_new(Arc::clone(&self.output_schema), output)?;
- debug_assert!(batch.num_rows() > 0);
- drop(timer);
- Some(batch)
- };
-
- self.set_output_batch(output_batch);
- Ok(())
- }
-}
-
-fn output_batch_memory_size(output_batch: &Option<RecordBatch>) -> usize {
- output_batch
- .as_ref()
- .map(RecordBatch::get_array_memory_size)
- .unwrap_or_default()
-}
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index e1c598e02d..08468bffc0 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -72,9 +72,9 @@ use itertools::Itertools;
use topk::hash_table::is_supported_hash_key_type;
use topk::heap::is_supported_heap_type;
+mod aggregate_hash_table;
pub mod group_values;
mod hash_aggregate;
-mod hash_table;
mod no_grouping;
pub mod order;
mod row_hash;