Apache DataFusion SQL Query Engine
perf: add a GroupsAccumulator for array_agg(DISTINCT) (#26357)
Qi Zhu290e26347b40
diff --git a/datafusion/core/tests/sql/aggregates/nested_nullability.rs b/datafusion/core/tests/sql/aggregates/nested_nullability.rsindex 3677bf4281..ceb98263f8 100644--- a/datafusion/core/tests/sql/aggregates/nested_nullability.rs+++ b/datafusion/core/tests/sql/aggregates/nested_nullability.rs@@ -235,7 +235,8 @@ async fn array_agg_struct_from_stricter_batches_with_spilling() -> Result<()> {#[tokio::test]async fn array_agg_distinct_struct_from_stricter_batches_with_spilling() -> Result<()> {AggregateBatchesTest::new()- .with_num_rows(10_000)+ // Enough groups that even the compact native accumulator exceeds the limit+ .with_num_rows(100_000)// One partition keeps the native aggregate's memory pressure deterministic..with_target_partitions(1).without_single_distinct_to_groupby()diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rsindex 4fcb2b32f2..2d92222561 100644--- a/datafusion/functions-aggregate/benches/array_agg.rs+++ b/datafusion/functions-aggregate/benches/array_agg.rs@@ -25,10 +25,11 @@ use arrow::array::{use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, Schema};use criterion::{BatchSize, Criterion, criterion_group, criterion_main};use datafusion_expr::function::AccumulatorArgs;-use datafusion_expr::{Accumulator, AggregateUDFImpl};+use datafusion_expr::{Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator};use datafusion_functions_aggregate::array_agg::{ArrayAgg, ArrayAggAccumulator, DistinctArrayAggAccumulator,};+use datafusion_functions_aggregate_common::aggregate::groups_accumulator::GroupsAccumulatorAdapter;use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};use datafusion_physical_expr_common::physical_expr::PhysicalExpr;@@ -323,6 +324,113 @@ fn distinct_array_agg_benchmark(c: &mut Criterion) {);}+/// `ARRAY_AGG(DISTINCT x) ... GROUP BY g` over one 8 192-row Utf8 batch, with+/// the groups accumulator against the previous per-group path+/// (`GroupsAccumulatorAdapter` over one `DistinctArrayAggAccumulator` per group),+/// across group counts, distinct cardinalities and partial emission.+fn distinct_array_agg_groups_benchmark(c: &mut Criterion) {+ let schema = Schema::new(vec![Field::new("value", DataType::Utf8, true)]);+ let value_expr = col("value", &schema).unwrap();+ let value_field = value_expr.return_field(&schema).unwrap();+ let return_field: FieldRef = Field::new(+ "array_agg",+ DataType::List(Field::new_list_field(DataType::Utf8, true).into()),+ true,+ )+ .into();+ let args = AccumulatorArgs {+ return_field,+ schema: &schema,+ expr_fields: std::slice::from_ref(&value_field),+ ignore_nulls: false,+ order_bys: &[],+ is_reversed: false,+ name: "array_agg(DISTINCT value)",+ is_distinct: true,+ exprs: std::slice::from_ref(&value_expr),+ };+ let array_agg = ArrayAgg::default();++ let inputs = [+ (+ "low cardinality (~25 distinct)",+ Arc::new(create_string_array_low_cardinality(8192, 0.0)) as ArrayRef,+ ),+ (+ "high cardinality (~7800 distinct)",+ Arc::new(create_string_array_high_cardinality(8192, 0.05, 0.0)) as ArrayRef,+ ),+ ];+ for num_groups in [100, 1000, 8192] {+ let mut rng = StdRng::seed_from_u64(42);+ let group_indices = (0..8192)+ .map(|_| rng.random_range(0..num_groups))+ .collect::<Vec<usize>>();+ for (input_name, values) in &inputs {+ // Emits every group at once, or in eight `EmitTo::First` chunks, which+ // makes the accumulator rebuild its state after each partial emission.+ for emit_chunks in [1, 8] {+ let run = |acc: &mut dyn GroupsAccumulator| {+ acc.update_batch(+ std::slice::from_ref(values),+ &group_indices,+ None,+ num_groups,+ )+ .unwrap();+ if emit_chunks == 1 {+ return vec![acc.evaluate(EmitTo::All).unwrap()];+ }+ let chunk = num_groups.div_ceil(emit_chunks);+ let mut remaining = num_groups;+ let mut emitted = Vec::with_capacity(emit_chunks);+ while remaining > 0 {+ let n = chunk.min(remaining);+ emitted.push(acc.evaluate(EmitTo::First(n)).unwrap());+ remaining -= n;+ }+ emitted+ };+ let emit = if emit_chunks == 1 {+ "emit all".to_string()+ } else {+ format!("emit in {emit_chunks} chunks")+ };+ c.bench_function(+ &format!(+ "distinct_array_agg groups {num_groups} groups, {input_name}, {emit}"+ ),+ |b| {+ b.iter(|| {+ let mut acc =+ array_agg.create_groups_accumulator(args.clone()).unwrap();+ run(acc.as_mut())+ })+ },+ );+ c.bench_function(+ &format!(+ "distinct_array_agg adapter {num_groups} groups, {input_name}, {emit}"+ ),+ |b| {+ b.iter(|| {+ let factory = || {+ Ok(Box::new(DistinctArrayAggAccumulator::try_new(+ &DataType::Utf8,+ None,+ false,+ )?) as Box<dyn Accumulator>)+ };+ let mut acc = GroupsAccumulatorAdapter::new(factory);+ run(&mut acc)+ })+ },+ );+ }+ }+ }+}+/// Precomputes the schema, physical expressions, sort expression, and aggregate/// metadata so each benchmark iteration can focus on accumulator operations.struct OrderedArrayAggBenchFixture {@@ -545,6 +653,7 @@ criterion_group!(benches,array_agg_benchmark,distinct_array_agg_benchmark,+ distinct_array_agg_groups_benchmark,ordered_array_agg_benchmark,ordered_array_agg_wide_utf8_benchmark);diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rsindex f81a60c222..9c27b8b249 100644--- a/datafusion/functions-aggregate/src/array_agg.rs+++ b/datafusion/functions-aggregate/src/array_agg.rs@@ -18,6 +18,7 @@//! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`]use std::collections::VecDeque;+use std::hash::BuildHasher;use std::mem::{size_of, size_of_val, take};use std::ops::Range;use std::sync::Arc;@@ -27,7 +28,7 @@ use arrow::array::{UInt32Array, make_array, new_empty_array,};use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};-use arrow::compute::{SortOptions, cast, filter};+use arrow::compute::{SortOptions, cast, filter, take as take_array};use arrow::datatypes::{DataType, Field, FieldRef, Fields};use arrow::row::{OwnedRow, Row, RowConverter, Rows, SortField};use arrow_select::dictionary::garbage_collect_any_dictionary;@@ -35,10 +36,12 @@ use arrow_select::dictionary::garbage_collect_any_dictionary;use datafusion_common::cast::as_list_array;use datafusion_common::hash_utils::{RandomState, create_hashes};use datafusion_common::scalar::copy_array_data;+use datafusion_common::utils::memory::estimate_memory_size;use datafusion_common::utils::proxy::HashTableAllocExt;use datafusion_common::utils::{SingleRowListArrayBuilder, offset_span};use datafusion_common::{- Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err,+ HashMap, HashSet, Result, ScalarValue, assert_eq_or_internal_err, exec_err,+ internal_datafusion_err, internal_err,};use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};use datafusion_expr::utils::format_state_name;@@ -177,40 +180,9 @@ impl AggregateUDFImpl for ArrayAgg {let ignore_nulls = acc_args.ignore_nulls && field.is_nullable();if acc_args.is_distinct {- // Limitation similar to Postgres. The aggregation function can only mix- // DISTINCT and ORDER BY if all the expressions in the ORDER BY appear- // also in the arguments of the function. This implies that if the- // aggregation function only accepts one argument, only one argument- // can be used in the ORDER BY, For example:- //- // ARRAY_AGG(DISTINCT col)- //- // can only be mixed with an ORDER BY if the order expression is "col".- //- // ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid- // ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid- // ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid- // ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid- let sort_option = match acc_args.order_bys {- // This accumulator sorts the values itself, so a reversed- // aggregate must still sort them in the requested order.- [single] if single.expr.eq(&acc_args.exprs[0]) => {- Some(if acc_args.is_reversed {- !single.options- } else {- single.options- })- }- [] => None,- _ => {- return exec_err!(- "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list"- );- }- };return Ok(Box::new(DistinctArrayAggAccumulator::try_new(data_type,- sort_option,+ distinct_sort_options(&acc_args)?,ignore_nulls,)?));}@@ -243,7 +215,11 @@ impl AggregateUDFImpl for ArrayAgg {}fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {- !args.is_distinct && args.order_bys.is_empty()+ if args.is_distinct {+ distinct_sort_options(&args).is_ok()+ } else {+ args.order_bys.is_empty()+ }}fn create_groups_accumulator(@@ -253,6 +229,13 @@ impl AggregateUDFImpl for ArrayAgg {let field = &args.expr_fields[0];let data_type = field.data_type().clone();let ignore_nulls = args.ignore_nulls && field.is_nullable();+ if args.is_distinct {+ return Ok(Box::new(DistinctArrayAggGroupsAccumulator::new(+ data_type,+ distinct_sort_options(&args)?,+ ignore_nulls,+ )));+ }Ok(Box::new(ArrayAggGroupsAccumulator::new(data_type,ignore_nulls,@@ -347,6 +330,42 @@ impl ArrayAggAccumulator {}}+/// Returns the sort options for `ARRAY_AGG(DISTINCT ...)`, or an error if the+/// ORDER BY cannot be combined with DISTINCT.+///+/// Limitation similar to Postgres. The aggregation function can only mix+/// DISTINCT and ORDER BY if all the expressions in the ORDER BY appear+/// also in the arguments of the function. This implies that if the+/// aggregation function only accepts one argument, only one argument+/// can be used in the ORDER BY, For example:+///+/// ARRAY_AGG(DISTINCT col)+///+/// can only be mixed with an ORDER BY if the order expression is "col".+///+/// ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid+/// ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid+/// ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid+/// ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid+///+/// The DISTINCT accumulators sort the values themselves, so a reversed+/// aggregate must still sort them in the requested order.+fn distinct_sort_options(acc_args: &AccumulatorArgs) -> Result<Option<SortOptions>> {+ match acc_args.order_bys {+ [single] if single.expr.eq(&acc_args.exprs[0]) => {+ Ok(Some(if acc_args.is_reversed {+ !single.options+ } else {+ single.options+ }))+ }+ [] => Ok(None),+ _ => exec_err!(+ "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list"+ ),+ }+}+/// Returns the mask of non-null rows in `values` when `ignore_nulls` is set/// and some row is null.fn non_null_mask(values: &dyn Array, ignore_nulls: bool) -> Option<BooleanArray> {@@ -803,27 +822,12 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {) -> Result<Vec<ArrayRef>> {assert_eq!(values.len(), 1, "one argument to convert_to_state");- let input = &values[0];-- // Each row becomes a 1-element list: offsets are [0, 1, 2, ..., n].- let offsets = OffsetBuffer::from_repeated_length(1, input.len());-- // Filtered rows become null list entries, which merge_batch will skip.- let filter_nulls = opt_filter.map(filter_to_nulls);-- // With ignore_nulls, null values also become null list entries. Without- // ignore_nulls, null values stay as [NULL] so merge_batch retains them.- let nulls = if self.ignore_nulls {- let logical = input.logical_nulls();- NullBuffer::union(filter_nulls.as_ref(), logical.as_ref())- } else {- filter_nulls- };-- let field = Arc::new(Field::new_list_field(self.datatype.clone(), true));- let list_array = ListArray::new(field, offsets, Arc::clone(input), nulls);-- Ok(vec![Arc::new(list_array)])+ Ok(vec![single_value_lists(+ &values[0],+ opt_filter,+ self.ignore_nulls,+ &self.datatype,+ )])}fn size(&self) -> usize {self.batches@@ -840,6 +844,419 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {}}+/// Converts each input row into a one-element list, the state shape of+/// `ARRAY_AGG`.+///+/// Filtered rows become null list entries, which `merge_batch` skips. With+/// `ignore_nulls`, null values also become null list entries; without it they+/// stay as `[NULL]` so `merge_batch` retains them.+fn single_value_lists(+ input: &ArrayRef,+ opt_filter: Option<&BooleanArray>,+ ignore_nulls: bool,+ datatype: &DataType,+) -> ArrayRef {+ // Offsets are [0, 1, 2, ..., n].+ let offsets = OffsetBuffer::from_repeated_length(1, input.len());+ let filter_nulls = opt_filter.map(filter_to_nulls);+ let nulls = if ignore_nulls {+ let logical = input.logical_nulls();+ NullBuffer::union(filter_nulls.as_ref(), logical.as_ref())+ } else {+ filter_nulls+ };+ let field = Arc::new(Field::new_list_field(datatype.clone(), true));+ Arc::new(ListArray::new(field, offsets, Arc::clone(input), nulls))+}++/// [`GroupsAccumulator`] for `ARRAY_AGG(DISTINCT x)`, optionally with+/// `ORDER BY x`.+///+/// Distinct values are encoded once with a [`RowConverter`] and stored in a+/// single [`Rows`] buffer shared by all groups, so a value that appears in many+/// groups is held once. Each group records the values it has seen as+/// `(group_index, value_index)` pairs, deduplicated in one hash set rather than+/// one set per group.+#[derive(Debug)]+struct DistinctArrayAggGroupsAccumulator {+ datatype: DataType,+ sort_options: Option<SortOptions>,+ ignore_nulls: bool,+ /// Encodes values to rows. Created on the first non-empty input, once the+ /// runtime type is known.+ converter: Option<RowConverter>,+ /// Every distinct value seen so far, indexed by value index.+ values: Option<Rows>,+ /// `(hash, value_index)` for every row in `values`.+ value_map: HashTable<(u64, u32)>,+ /// Heap size of `value_map` in bytes.+ value_map_size: usize,+ /// Distinct `(group_index, value_index)` pairs, in first-seen order.+ pairs: Vec<(u32, u32)>,+ /// Deduplicates `pairs`.+ pair_set: HashSet<(u32, u32), RandomState>,+ /// Total number of groups tracked.+ num_groups: usize,+ random_state: RandomState,+ /// Time spent deduplicating values, reported as the DISTINCT submetric.+ distinct_metric: Option<Arc<dyn AggregateMetric>>,+}++impl DistinctArrayAggGroupsAccumulator {+ fn new(+ datatype: DataType,+ sort_options: Option<SortOptions>,+ ignore_nulls: bool,+ ) -> Self {+ Self {+ datatype,+ sort_options,+ ignore_nulls,+ converter: None,+ values: None,+ value_map: HashTable::new(),+ value_map_size: 0,+ pairs: Vec::new(),+ pair_set: HashSet::default(),+ num_groups: 0,+ random_state: RandomState::default(),+ distinct_metric: None,+ }+ }++ fn converter(&mut self, data_type: &DataType) -> Result<&RowConverter> {+ if self.converter.is_none() {+ let sort_field = match self.sort_options {+ Some(opts) => SortField::new_with_options(data_type.clone(), opts),+ None => SortField::new(data_type.clone()),+ };+ let converter = RowConverter::new(vec![sort_field])?;+ self.values = Some(converter.empty_rows(0, 0));+ self.converter = Some(converter);+ }+ self.converter+ .as_ref()+ .ok_or_else(|| internal_datafusion_err!("row converter must be set"))+ }++ /// Like [`Self::insert`], but first gathers the selected rows when they+ /// are only part of `input`, so rejected values are never encoded.+ fn insert_selected(+ &mut self,+ input: &ArrayRef,+ rows: Vec<(usize, usize)>,+ ) -> Result<()> {+ if rows.len() == input.len() {+ return self.insert(input, rows);+ }+ let indices = UInt32Array::from_iter_values(+ rows.iter().map(|&(_, row_index)| row_index as u32),+ );+ let selected = take_array(input.as_ref(), &indices, None)?;+ self.insert(+ &selected,+ rows.into_iter()+ .enumerate()+ .map(|(position, (group_index, _))| (group_index, position)),+ )+ }++ /// Records that `group_index` contains each value of `input` selected by+ /// `rows`, given as `(group_index, row_in_input)` pairs.+ fn insert(+ &mut self,+ input: &ArrayRef,+ rows: impl IntoIterator<Item = (usize, usize)>,+ ) -> Result<()> {+ let encoded = self+ .converter(input.data_type())?+ .convert_columns(std::slice::from_ref(input))?;+ let Self {+ values,+ value_map,+ value_map_size,+ pairs,+ pair_set,+ random_state,+ ..+ } = self;+ let values = values+ .as_mut()+ .ok_or_else(|| internal_datafusion_err!("value rows must be set"))?;++ for (group_index, row_index) in rows {+ let row = encoded.row(row_index);+ let hash = random_state.hash_one(row.as_ref());+ let value_index = match value_map.find(hash, |&(h, index)| {+ h == hash && values.row(index as usize) == row+ }) {+ Some(&(_, index)) => index,+ None => {+ let index = values.num_rows() as u32;+ values.push(row);+ value_map.insert_accounted(+ (hash, index),+ |&(h, _)| h,+ value_map_size,+ );+ index+ }+ };+ let pair = (group_index as u32, value_index);+ if pair_set.insert(pair) {+ pairs.push(pair);+ }+ }+ Ok(())+ }++ /// Drops the state of the first `emit_groups` groups and renumbers the+ /// rest so they start from 0. Values only referenced by emitted groups are+ /// dropped too.+ fn retain_after(&mut self, emit_groups: usize) {+ let emit_groups = emit_groups as u32;+ let old_pairs = take(&mut self.pairs);+ let old_values = self.values.take();+ self.value_map = HashTable::new();+ self.value_map_size = 0;+ self.pair_set = HashSet::default();+ self.num_groups -= emit_groups as usize;++ let (Some(converter), Some(old_values)) = (&self.converter, old_values) else {+ return;+ };+ let mut values = converter.empty_rows(0, 0);+ let mut remap: HashMap<u32, u32> = HashMap::new();+ let mut pairs = Vec::new();+ for (group_index, value_index) in old_pairs {+ if group_index < emit_groups {+ continue;+ }+ let new_index = *remap.entry(value_index).or_insert_with(|| {+ let row = old_values.row(value_index as usize);+ let index = values.num_rows() as u32;+ values.push(row);+ let hash = self.random_state.hash_one(row.as_ref());+ self.value_map.insert_accounted(+ (hash, index),+ |&(h, _)| h,+ &mut self.value_map_size,+ );+ index+ });+ let pair = (group_index - emit_groups, new_index);+ self.pair_set.insert(pair);+ pairs.push(pair);+ }+ self.pairs = pairs;+ self.values = Some(values);+ }+}++impl GroupsAccumulator for DistinctArrayAggGroupsAccumulator {+ fn update_batch(+ &mut self,+ values: &[ArrayRef],+ group_indices: &[usize],+ opt_filter: Option<&BooleanArray>,+ total_num_groups: usize,+ ) -> Result<()> {+ // With `ORDER BY x` the ordering column is passed too. It is the+ // argument itself, so only the first column is needed.+ let Some(input) = values.first() else {+ return internal_err!("array_agg(DISTINCT) expects an argument");+ };+ self.num_groups = self.num_groups.max(total_num_groups);++ let nulls = if self.ignore_nulls {+ input.logical_nulls()+ } else {+ None+ };+ let rows = group_indices+ .iter()+ .enumerate()+ .filter(|&(row_index, _)| {+ let filtered_out = opt_filter.is_some_and(|filter| {+ filter.is_null(row_index) || !filter.value(row_index)+ });+ let skipped_null =+ nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index));+ !filtered_out && !skipped_null+ })+ .map(|(row_index, &group_index)| (group_index, row_index))+ .collect::<Vec<_>>();+ if rows.is_empty() {+ return Ok(());+ }+ let mut metric_recorder =+ AggregateMetricRecorder::new(self.distinct_metric.clone());+ let _timer = metric_recorder.timer();+ self.insert_selected(input, rows)+ }++ /// Produces a `ListArray` ordered by group index. Groups without values+ /// are null, matching [`DistinctArrayAggAccumulator`].+ fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {+ let emit_groups = match emit_to {+ EmitTo::All => self.num_groups,+ EmitTo::First(n) => n,+ };++ // Counting sort of the pairs into group order.+ let mut counts = vec![0u32; emit_groups];+ for &(group_index, _) in &self.pairs {+ if let Some(count) = counts.get_mut(group_index as usize) {+ *count += 1;+ }+ }+ let mut offsets = Vec::<i32>::with_capacity(emit_groups + 1);+ offsets.push(0);+ let mut nulls = NullBufferBuilder::new(emit_groups);+ let mut write_positions = Vec::with_capacity(emit_groups);+ let mut total = 0u32;+ for &count in &counts {+ if count == 0 {+ nulls.append_null();+ } else {+ nulls.append_non_null();+ }+ write_positions.push(total);+ total += count;+ offsets.push(total as i32);+ }+ let mut ordered = vec![0u32; total as usize];+ for &(group_index, value_index) in &self.pairs {+ if let Some(position) = write_positions.get_mut(group_index as usize) {+ ordered[*position as usize] = value_index;+ *position += 1;+ }+ }++ let flat_values = match (&self.converter, &self.values) {+ (Some(converter), Some(values)) if total > 0 => {+ if self.sort_options.is_some() {+ // The sort direction is encoded in the row bytes, so+ // comparing rows gives the requested order.+ for window in offsets.windows(2) {+ ordered[window[0] as usize..window[1] as usize].sort_unstable_by(+ |&a, &b| values.row(a as usize).cmp(&values.row(b as usize)),+ );+ }+ }+ let decoded = converter+ .convert_rows(ordered.iter().map(|&i| values.row(i as usize)))?+ .swap_remove(0);+ // `convert_rows` always returns the physical (non-dictionary)+ // type, so cast back when the declared type has a dictionary.+ if decoded.data_type() != &self.datatype+ && datatype_contains_dictionary(&self.datatype)+ {+ cast(decoded.as_ref(), &self.datatype)?+ } else {+ decoded+ }+ }+ _ => new_empty_array(&self.datatype),+ };++ match emit_to {+ EmitTo::All => {+ let converter = self.converter.take();+ let distinct_metric = self.distinct_metric.take();+ *self = Self::new(+ self.datatype.clone(),+ self.sort_options,+ self.ignore_nulls,+ );+ self.distinct_metric = distinct_metric;+ // Keep the converter: the runtime type does not change.+ if let Some(converter) = converter {+ self.values = Some(converter.empty_rows(0, 0));+ self.converter = Some(converter);+ }+ }+ EmitTo::First(_) => self.retain_after(emit_groups),+ }++ let field = Arc::new(Field::new_list_field(self.datatype.clone(), true));+ Ok(Arc::new(ListArray::new(+ field,+ OffsetBuffer::new(ScalarBuffer::from(offsets)),+ flat_values,+ nulls.finish(),+ )))+ }++ fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {+ Ok(vec![self.evaluate(emit_to)?])+ }++ fn merge_batch(+ &mut self,+ values: &[ArrayRef],+ group_indices: &[usize],+ total_num_groups: usize,+ ) -> Result<()> {+ assert_eq_or_internal_err!(values.len(), 1, "one argument to merge_batch");+ let lists = values[0].as_list::<i32>();+ self.num_groups = self.num_groups.max(total_num_groups);++ // A sliced list keeps its whole child array, so encode only the span+ // its offsets cover.+ let offsets = lists.offsets();+ let (span_start, span_len) = offset_span(offsets);+ let child = lists.values().slice(span_start, span_len);+ let rows = group_indices+ .iter()+ .enumerate()+ .filter(|&(list_index, _)| lists.is_valid(list_index))+ .flat_map(|(list_index, &group_index)| {+ let start = offsets[list_index] as usize - span_start;+ let end = offsets[list_index + 1] as usize - span_start;+ (start..end).map(move |position| (group_index, position))+ })+ .collect::<Vec<_>>();+ if rows.is_empty() {+ return Ok(());+ }+ let mut metric_recorder =+ AggregateMetricRecorder::new(self.distinct_metric.clone());+ let _timer = metric_recorder.timer();+ self.insert_selected(&child, rows)+ }++ fn convert_to_state(+ &self,+ values: &[ArrayRef],+ opt_filter: Option<&BooleanArray>,+ ) -> Result<Vec<ArrayRef>> {+ let Some(input) = values.first() else {+ return internal_err!("array_agg(DISTINCT) expects an argument");+ };+ Ok(vec![single_value_lists(+ input,+ opt_filter,+ self.ignore_nulls,+ &self.datatype,+ )])+ }++ fn set_metrics(&mut self, metrics: Arc<dyn AggregateMetrics>) {+ self.distinct_metric = Some(metrics.metric("distinct"));+ }++ fn size(&self) -> usize {+ size_of_val(self)+ + self.converter.as_ref().map_or(0, RowConverter::size)+ + self.values.as_ref().map_or(0, Rows::size)+ + self.value_map_size+ + self.pairs.capacity() * size_of::<(u32, u32)>()+ + estimate_memory_size::<(u32, u32)>(self.pair_set.capacity(), 0)+ .unwrap_or_default()+ }+}+/// Resources that are allocated lazily on the first `update_batch` call,/// once the concrete runtime Arrow type is known.///@@ -1791,8 +2208,8 @@ impl Accumulator for OrderSensitiveArrayAggAccumulator {#[cfg(test)]mod tests {use super::*;- use arrow::array::{Int32Builder, ListBuilder, StringBuilder};- use arrow::datatypes::Schema;+ use arrow::array::{DictionaryArray, Int32Builder, ListBuilder, StringBuilder};+ use arrow::datatypes::{Int32Type, Schema};use datafusion_common::cast::as_generic_string_array;use datafusion_common::internal_err;use datafusion_physical_expr::PhysicalExpr;@@ -3417,13 +3834,291 @@ mod tests {}fn eval_i32_lists(- acc: &mut ArrayAggGroupsAccumulator,+ acc: &mut dyn GroupsAccumulator,emit_to: EmitTo,) -> Result<Vec<Option<Vec<Option<i32>>>>> {let result = acc.evaluate(emit_to)?;Ok(list_array_to_i32_vecs(result.as_list::<i32>()))}+ fn distinct_groups_accumulator(+ ignore_nulls: bool,+ ) -> DistinctArrayAggGroupsAccumulator {+ DistinctArrayAggGroupsAccumulator::new(DataType::Int32, None, ignore_nulls)+ }++ #[test]+ fn distinct_groups_accumulator_dedups_per_group() -> Result<()> {+ let mut acc = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 1, 2, 1, 2, 3]));+ acc.update_batch(&[values], &[0, 0, 0, 1, 1, 1], None, 2)?;+ let values: ArrayRef = Arc::new(Int32Array::from(vec![2, 3, 3]));+ acc.update_batch(&[values], &[0, 1, 1], None, 2)?;++ // Values shared by both groups are stored once.+ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(3));++ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(1), Some(2)]));+ assert_eq!(vals[1], Some(vec![Some(1), Some(2), Some(3)]));+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_nulls() -> Result<()> {+ let values: ArrayRef =+ Arc::new(Int32Array::from(vec![None, Some(1), None, None]));+ let groups = [0, 0, 0, 1];++ // Without IGNORE NULLS a null is one more distinct value.+ let mut acc = distinct_groups_accumulator(false);+ acc.update_batch(&[Arc::clone(&values)], &groups, None, 3)?;+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![None, Some(1)]));+ assert_eq!(vals[1], Some(vec![None]));+ // A group with no rows is null.+ assert_eq!(vals[2], None);++ // With IGNORE NULLS a group of only nulls is null.+ let mut acc = distinct_groups_accumulator(true);+ acc.update_batch(&[values], &groups, None, 2)?;+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(1)]));+ assert_eq!(vals[1], None);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_opt_filter() -> Result<()> {+ let mut acc = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 2, 3]));+ let filter = BooleanArray::from(vec![Some(true), Some(false), Some(true), None]);+ acc.update_batch(&[values], &[0, 0, 0, 1], Some(&filter), 2)?;++ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(1), Some(2)]));+ assert_eq!(vals[1], None);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_sparse_filter() -> Result<()> {+ // Only the selected rows are gathered and encoded, so their positions+ // move; check each value still lands in its own group.+ let mut acc = distinct_groups_accumulator(true);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![+ Some(5),+ Some(6),+ Some(7),+ None,+ Some(8),+ ]));+ let filter = BooleanArray::from(vec![false, true, false, true, true]);+ acc.update_batch(&[values], &[0, 1, 0, 2, 2], Some(&filter), 3)?;++ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2));+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals, vec![None, Some(vec![Some(6)]), Some(vec![Some(8)])]);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_merges_sliced_state() -> Result<()> {+ let mut partial = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 4]));+ partial.update_batch(&[values], &[0, 1, 2, 3, 3], None, 4)?;+ let state = partial.state(EmitTo::All)?;++ // Partial aggregation emits slices of one state array.+ let mut acc = distinct_groups_accumulator(false);+ acc.merge_batch(&[state[0].slice(2, 2)], &[0, 1], 2)?;+ acc.merge_batch(&[state[0].slice(0, 2)], &[1, 0], 2)?;++ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(3), Some(2)]));+ assert_eq!(vals[1], Some(vec![Some(4), Some(1)]));+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_records_metric() -> Result<()> {+ let metric_updates = Arc::new(AtomicUsize::new(0));+ let mut acc = distinct_groups_accumulator(false);+ acc.set_metrics(Arc::new(CountingMetrics(Arc::clone(&metric_updates))));++ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));+ acc.update_batch(&[Arc::clone(&values)], &[0, 1], None, 2)?;+ let state = acc.state(EmitTo::All)?;+ acc.merge_batch(&state, &[0, 1], 2)?;+ assert_eq!(metric_updates.load(Ordering::Relaxed), 2);++ // Emitting everything resets the state but keeps the metric.+ acc.evaluate(EmitTo::All)?;+ acc.update_batch(&[values], &[0, 1], None, 2)?;+ assert_eq!(metric_updates.load(Ordering::Relaxed), 3);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_order_by() -> Result<()> {+ let descending = SortOptions {+ descending: true,+ nulls_first: false,+ };+ let mut acc = DistinctArrayAggGroupsAccumulator::new(+ DataType::Int32,+ Some(descending),+ false,+ );+ let values: ArrayRef = Arc::new(Int32Array::from(vec![+ Some(2),+ None,+ Some(5),+ Some(2),+ Some(1),+ ]));+ acc.update_batch(&[values], &[0, 0, 0, 0, 1], None, 2)?;++ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(5), Some(2), None]));+ assert_eq!(vals[1], Some(vec![Some(1)]));+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_emit_first() -> Result<()> {+ let mut acc = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 10, 20, 30, 20]));+ acc.update_batch(&[values], &[0, 0, 1, 2, 2], None, 3)?;++ let vals = eval_i32_lists(&mut acc, EmitTo::First(2))?;+ assert_eq!(vals, vec![Some(vec![Some(10)]), Some(vec![Some(20)])]);++ // 10 was only referenced by an emitted group, so it is dropped.+ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2));++ // The remaining group is renumbered to 0 and keeps deduplicating.+ let values: ArrayRef = Arc::new(Int32Array::from(vec![20, 40]));+ acc.update_batch(&[values], &[0, 0], None, 1)?;+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals, vec![Some(vec![Some(30), Some(20), Some(40)])]);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_state_merge_roundtrip() -> Result<()> {+ let mut acc1 = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));+ acc1.update_batch(&[values], &[0, 1], None, 2)?;++ let mut acc2 = distinct_groups_accumulator(false);+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 3, 2]));+ acc2.update_batch(&[values], &[0, 0, 1], None, 2)?;++ let state = acc2.state(EmitTo::All)?;+ acc1.merge_batch(&state, &[0, 1], 2)?;++ let vals = eval_i32_lists(&mut acc1, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(1), Some(3)]));+ assert_eq!(vals[1], Some(vec![Some(2)]));+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_convert_to_state_merge() -> Result<()> {+ let partial = distinct_groups_accumulator(true);+ let values: ArrayRef =+ Arc::new(Int32Array::from(vec![Some(7), None, Some(7), Some(8)]));+ let filter = BooleanArray::from(vec![true, true, true, false]);+ let state = partial.convert_to_state(&[values], Some(&filter))?;++ let mut acc = distinct_groups_accumulator(true);+ acc.merge_batch(&state, &[0, 0, 0, 1], 2)?;+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;+ assert_eq!(vals[0], Some(vec![Some(7)]));+ assert_eq!(vals[1], None);+ Ok(())+ }++ #[test]+ fn distinct_groups_accumulator_keeps_dictionary_type() -> Result<()> {+ let datatype =+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));+ let mut acc =+ DistinctArrayAggGroupsAccumulator::new(datatype.clone(), None, false);+ let values: ArrayRef = Arc::new(+ vec!["a", "b", "a", "a"]+ .into_iter()+ .collect::<DictionaryArray<Int32Type>>(),+ );+ acc.update_batch(&[values], &[0, 0, 0, 1], None, 2)?;++ let result = acc.evaluate(EmitTo::All)?;+ let lists = result.as_list::<i32>();+ assert_eq!(lists.values().data_type(), &datatype);+ let flat = cast(lists.values().as_ref(), &DataType::Utf8)?;+ let flat = flat.as_string::<i32>().iter().collect::<Vec<_>>();+ assert_eq!(flat, vec![Some("a"), Some("b"), Some("a")]);+ assert_eq!(lists.value_offsets(), &[0, 2, 3]);+ Ok(())+ }++ /// Compares the groups accumulator with one [`DistinctArrayAggAccumulator`]+ /// per group over batches with repeated values, nulls and a filter.+ #[test]+ fn distinct_groups_accumulator_matches_row_accumulator() -> Result<()> {+ let num_groups = 7;+ let mut acc = distinct_groups_accumulator(false);+ let mut expected = (0..num_groups)+ .map(|_| DistinctArrayAggAccumulator::try_new(&DataType::Int32, None, false))+ .collect::<Result<Vec<_>>>()?;++ for batch in 0..5 {+ let len = 50;+ let values = (0..len)+ .map(|i| ((i * 7 + batch * 3) % 11 != 0).then_some((i * 13 + batch) % 9))+ .collect::<Vec<Option<i32>>>();+ let groups = (0..len)+ .map(|i| ((i * 5 + batch) % num_groups as i32) as usize)+ .collect::<Vec<_>>();+ let filter = (0..len).map(|i| (i + batch) % 4 != 0).collect::<Vec<_>>();++ let array: ArrayRef = Arc::new(Int32Array::from(values.clone()));+ acc.update_batch(+ &[array],+ &groups,+ Some(&BooleanArray::from(filter.clone())),+ num_groups,+ )?;+ for ((value, group), keep) in values.into_iter().zip(groups).zip(filter) {+ if keep {+ let value: ArrayRef = Arc::new(Int32Array::from(vec![value]));+ expected[group].update_batch(&[value])?;+ }+ }+ }++ let sorted = |list: Option<Vec<Option<i32>>>| {+ list.map(|mut values| {+ values.sort();+ values+ })+ };+ let actual = eval_i32_lists(&mut acc, EmitTo::All)?;+ for (group, acc) in expected.iter_mut().enumerate() {+ let ScalarValue::List(list) = acc.evaluate()? else {+ return internal_err!("expected a list");+ };+ let expected_values = list_array_to_i32_vecs(&list).swap_remove(0);+ assert_eq!(+ sorted(actual[group].clone()),+ sorted(expected_values),+ "group {group}"+ );+ }+ Ok(())+ }+#[test]fn groups_accumulator_multiple_batches() -> Result<()> {let mut acc = ArrayAggGroupsAccumulator::new(DataType::Int32, false);diff --git a/datafusion/functions-aggregate/tests/state_compat.rs b/datafusion/functions-aggregate/tests/state_compat.rsindex e1915da9cf..5e6f794d82 100644--- a/datafusion/functions-aggregate/tests/state_compat.rs+++ b/datafusion/functions-aggregate/tests/state_compat.rs@@ -27,8 +27,8 @@ use std::collections::BTreeSet;use std::panic::{AssertUnwindSafe, catch_unwind};use std::sync::Arc;-use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, UInt32Array};-use arrow::compute::{cast, concat, take};+use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, ListArray, UInt32Array};+use arrow::compute::{cast, concat, sort, take};use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit};use arrow::record_batch::{RecordBatch, RecordBatchOptions};use datafusion_common::{DataFusionError, Result, ScalarValue};@@ -518,7 +518,10 @@ fn take_all(arrays: &[ArrayRef], indices: &UInt32Array) -> Result<Vec<ArrayRef>>/// Runs every state exchange for one case.fn check_case(expr: &AggregateFunctionExpr, case: &Case) -> Errors {- let mut errors = Errors::default();+ let mut errors = Errors {+ ignore_list_order: case.distinct && !case.ordered,+ ..Default::default()+ };let input = match Input::new(expr, case) {Ok(input) => input,Err(e) => {@@ -693,7 +696,12 @@ fn guard<T>(f: impl FnOnce() -> Result<T>) -> Result<T> {/// Failures for one case, deduplicated by message across groups.#[derive(Default)]-struct Errors(Vec<(String, Vec<usize>)>);+struct Errors {+ failures: Vec<(String, Vec<usize>)>,+ /// Compare list results as multisets. `array_agg(DISTINCT x)` without+ /// ORDER BY does not define the order of its elements.+ ignore_list_order: bool,+}impl Errors {fn push(&mut self, msg: &str) {@@ -708,14 +716,14 @@ impl Errors {// Only keep the first line, dropping e.g. the "please file a bug" text// of internal errors.let msg = msg.lines().next().unwrap_or_default().to_string();- match self.0.iter_mut().find(|(m, _)| *m == msg) {+ match self.failures.iter_mut().find(|(m, _)| *m == msg) {Some((_, groups)) => groups.extend(group),- None => self.0.push((msg, group.into_iter().collect())),+ None => self.failures.push((msg, group.into_iter().collect())),}}fn lines(&self) -> Vec<String> {- self.0+ self.failures.iter().map(|(msg, groups)| {if groups.is_empty() {@@ -771,7 +779,7 @@ impl Errors {) {match got {Err(e) => self.push_group(&format!("{what}: {e}"), group),- Ok(got) if !scalars_match(&got, want) => {+ Ok(got) if !scalars_match(&got, want, self.ignore_list_order) => {self.push_group(&format!("{what}: got {got:?}, expected {want:?}"), group)}Ok(_) => {}@@ -780,8 +788,15 @@ impl Errors {}/// Exact equality, except that floats are compared with a relative tolerance-/// because merging in a different order can change rounding.-fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool {+/// because merging in a different order can change rounding, and lists are+/// compared as multisets when `ignore_list_order` is set.+fn scalars_match(a: &ScalarValue, b: &ScalarValue, ignore_list_order: bool) -> bool {+ if ignore_list_order+ && let (ScalarValue::List(x), ScalarValue::List(y)) = (a, b)+ && let (Some(x), Some(y)) = (sorted_list_values(x), sorted_list_values(y))+ {+ return x.as_ref() == y.as_ref();+ }match (a, b) {(ScalarValue::Float64(Some(x)), ScalarValue::Float64(Some(y))) => {floats_match(*x, *y)@@ -793,6 +808,15 @@ fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool {}}+/// The values of a single-row list, sorted, or `None` if the row is null or the+/// values cannot be sorted.+fn sorted_list_values(list: &ListArray) -> Option<ArrayRef> {+ if list.len() != 1 || list.is_null(0) {+ return None;+ }+ sort(list.value(0).as_ref(), None).ok()+}+fn floats_match(x: f64, y: f64) -> bool {if x.is_nan() || y.is_nan() {return x.is_nan() && y.is_nan();diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rsindex 89131b38bc..1638ba5455 100644--- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs+++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs@@ -1500,7 +1500,10 @@ mod tests {use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_expr::{AggregateMetric, AggregateMetrics};use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf};- use datafusion_physical_expr::aggregate::AggregateExprBuilder;+ use datafusion_physical_expr::GroupsAccumulatorAdapter;+ use datafusion_physical_expr::aggregate::{+ AggregateExprBuilder, AggregateFunctionExpr,+ };use datafusion_physical_expr::expressions::col;use std::sync::atomic::{AtomicUsize, Ordering};use std::time::Duration;@@ -1527,6 +1530,18 @@ mod tests {}}+ /// `array_agg(DISTINCT)` has a native groups accumulator, so build the+ /// per-group adapter explicitly to cover its metric handling.+ fn distinct_adapter(+ aggregate_expr: Arc<AggregateFunctionExpr>,+ metrics: Arc<dyn AggregateMetrics>,+ ) -> GroupsAccumulatorAdapter {+ let mut adapter =+ GroupsAccumulatorAdapter::new(move || aggregate_expr.create_accumulator());+ adapter.set_metrics(metrics);+ adapter+ }+#[test]fn legacy_grouped_distinct_installs_metrics_for_conversion_and_update() -> Result<()>{@@ -1544,13 +1559,13 @@ mod tests {);let metric_resolutions = Arc::new(AtomicUsize::new(0));let metric_durations = Arc::new(AtomicUsize::new(0));- let mut accumulator = create_group_accumulator(- &aggregate_expr,+ let mut accumulator = distinct_adapter(+ aggregate_expr,Arc::new(CountingMetrics {resolutions: Arc::clone(&metric_resolutions),durations: Arc::clone(&metric_durations),}),- )?;+ );let values: ArrayRef = Arc::new(UInt32Array::from(vec![1, 2, 3]));accumulator.convert_to_state(&[Arc::clone(&values)], None)?;@@ -1579,13 +1594,13 @@ mod tests {);let metric_resolutions = Arc::new(AtomicUsize::new(0));let metric_durations = Arc::new(AtomicUsize::new(0));- let mut accumulator = create_group_accumulator(- &aggregate_expr,+ let mut accumulator = distinct_adapter(+ aggregate_expr,Arc::new(CountingMetrics {resolutions: Arc::clone(&metric_resolutions),durations: Arc::clone(&metric_durations),}),- )?;+ );let mut state = ListBuilder::new(UInt32Builder::new());for value in [1, 2, 3] {diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.sltindex 8342380e31..89b7bcf3f5 100644--- a/datafusion/sqllogictest/test_files/array_agg.slt+++ b/datafusion/sqllogictest/test_files/array_agg.slt@@ -678,3 +678,40 @@ List(Dictionary(Int32, Utf8))statement okdrop table array_agg_distinct_dict_table;++# array_agg(DISTINCT) per group, mixed with other aggregates and a FILTER+statement ok+create table array_agg_distinct_groups(g int, v varchar) as values+ (1, 'b'), (1, 'a'), (1, 'b'), (1, null),+ (2, 'a'), (2, 'a'),+ (3, null),+ (4, 'c');++query II??+select+ g,+ count(*),+ array_sort(array_agg(DISTINCT v) FILTER (WHERE v <> 'c')),+ array_sort(array_agg(DISTINCT v))+from array_agg_distinct_groups+group by g+order by g;+----+1 4 [a, b] [NULL, a, b]+2 2 [a] [a]+3 1 NULL [NULL]+4 1 NULL [c]++query I?+select g, array_agg(DISTINCT v ORDER BY v DESC) ignore nulls+from array_agg_distinct_groups+group by g+order by g;+----+1 [b, a]+2 [a]+3 NULL+4 [c]++statement ok+drop table array_agg_distinct_groups;