Skip to content

Apache DataFusion SQL Query Engine

perf: add a GroupsAccumulator for array_agg(DISTINCT) (#26357)

Qi Zhu290e26347b40
diff --git a/datafusion/core/tests/sql/aggregates/nested_nullability.rs b/datafusion/core/tests/sql/aggregates/nested_nullability.rs
index 3677bf4281..ceb98263f8 100644
--- a/datafusion/core/tests/sql/aggregates/nested_nullability.rs
+++ b/datafusion/core/tests/sql/aggregates/nested_nullability.rs
@@ -235,7 +235,8 @@ async fn array_agg_struct_from_stricter_batches_with_spilling() -> Result<()> {
#[tokio::test]
async fn array_agg_distinct_struct_from_stricter_batches_with_spilling() -> Result<()> {
AggregateBatchesTest::new()
- .with_num_rows(10_000)
+ // Enough groups that even the compact native accumulator exceeds the limit
+ .with_num_rows(100_000)
// One partition keeps the native aggregate's memory pressure deterministic.
.with_target_partitions(1)
.without_single_distinct_to_groupby()
diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs
index 4fcb2b32f2..2d92222561 100644
--- a/datafusion/functions-aggregate/benches/array_agg.rs
+++ b/datafusion/functions-aggregate/benches/array_agg.rs
@@ -25,10 +25,11 @@ use arrow::array::{
use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, Schema};
use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
use datafusion_expr::function::AccumulatorArgs;
-use datafusion_expr::{Accumulator, AggregateUDFImpl};
+use datafusion_expr::{Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator};
use datafusion_functions_aggregate::array_agg::{
ArrayAgg, ArrayAggAccumulator, DistinctArrayAggAccumulator,
};
+use datafusion_functions_aggregate_common::aggregate::groups_accumulator::GroupsAccumulatorAdapter;
use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
@@ -323,6 +324,113 @@ fn distinct_array_agg_benchmark(c: &mut Criterion) {
);
}
+/// `ARRAY_AGG(DISTINCT x) ... GROUP BY g` over one 8 192-row Utf8 batch, with
+/// the groups accumulator against the previous per-group path
+/// (`GroupsAccumulatorAdapter` over one `DistinctArrayAggAccumulator` per group),
+/// across group counts, distinct cardinalities and partial emission.
+fn distinct_array_agg_groups_benchmark(c: &mut Criterion) {
+ let schema = Schema::new(vec![Field::new("value", DataType::Utf8, true)]);
+ let value_expr = col("value", &schema).unwrap();
+ let value_field = value_expr.return_field(&schema).unwrap();
+ let return_field: FieldRef = Field::new(
+ "array_agg",
+ DataType::List(Field::new_list_field(DataType::Utf8, true).into()),
+ true,
+ )
+ .into();
+ let args = AccumulatorArgs {
+ return_field,
+ schema: &schema,
+ expr_fields: std::slice::from_ref(&value_field),
+ ignore_nulls: false,
+ order_bys: &[],
+ is_reversed: false,
+ name: "array_agg(DISTINCT value)",
+ is_distinct: true,
+ exprs: std::slice::from_ref(&value_expr),
+ };
+ let array_agg = ArrayAgg::default();
+
+ let inputs = [
+ (
+ "low cardinality (~25 distinct)",
+ Arc::new(create_string_array_low_cardinality(8192, 0.0)) as ArrayRef,
+ ),
+ (
+ "high cardinality (~7800 distinct)",
+ Arc::new(create_string_array_high_cardinality(8192, 0.05, 0.0)) as ArrayRef,
+ ),
+ ];
+ for num_groups in [100, 1000, 8192] {
+ let mut rng = StdRng::seed_from_u64(42);
+ let group_indices = (0..8192)
+ .map(|_| rng.random_range(0..num_groups))
+ .collect::<Vec<usize>>();
+ for (input_name, values) in &inputs {
+ // Emits every group at once, or in eight `EmitTo::First` chunks, which
+ // makes the accumulator rebuild its state after each partial emission.
+ for emit_chunks in [1, 8] {
+ let run = |acc: &mut dyn GroupsAccumulator| {
+ acc.update_batch(
+ std::slice::from_ref(values),
+ &group_indices,
+ None,
+ num_groups,
+ )
+ .unwrap();
+ if emit_chunks == 1 {
+ return vec![acc.evaluate(EmitTo::All).unwrap()];
+ }
+ let chunk = num_groups.div_ceil(emit_chunks);
+ let mut remaining = num_groups;
+ let mut emitted = Vec::with_capacity(emit_chunks);
+ while remaining > 0 {
+ let n = chunk.min(remaining);
+ emitted.push(acc.evaluate(EmitTo::First(n)).unwrap());
+ remaining -= n;
+ }
+ emitted
+ };
+ let emit = if emit_chunks == 1 {
+ "emit all".to_string()
+ } else {
+ format!("emit in {emit_chunks} chunks")
+ };
+ c.bench_function(
+ &format!(
+ "distinct_array_agg groups {num_groups} groups, {input_name}, {emit}"
+ ),
+ |b| {
+ b.iter(|| {
+ let mut acc =
+ array_agg.create_groups_accumulator(args.clone()).unwrap();
+ run(acc.as_mut())
+ })
+ },
+ );
+ c.bench_function(
+ &format!(
+ "distinct_array_agg adapter {num_groups} groups, {input_name}, {emit}"
+ ),
+ |b| {
+ b.iter(|| {
+ let factory = || {
+ Ok(Box::new(DistinctArrayAggAccumulator::try_new(
+ &DataType::Utf8,
+ None,
+ false,
+ )?) as Box<dyn Accumulator>)
+ };
+ let mut acc = GroupsAccumulatorAdapter::new(factory);
+ run(&mut acc)
+ })
+ },
+ );
+ }
+ }
+ }
+}
+
/// Precomputes the schema, physical expressions, sort expression, and aggregate
/// metadata so each benchmark iteration can focus on accumulator operations.
struct OrderedArrayAggBenchFixture {
@@ -545,6 +653,7 @@ criterion_group!(
benches,
array_agg_benchmark,
distinct_array_agg_benchmark,
+ distinct_array_agg_groups_benchmark,
ordered_array_agg_benchmark,
ordered_array_agg_wide_utf8_benchmark
);
diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs
index f81a60c222..9c27b8b249 100644
--- a/datafusion/functions-aggregate/src/array_agg.rs
+++ b/datafusion/functions-aggregate/src/array_agg.rs
@@ -18,6 +18,7 @@
//! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`]
use std::collections::VecDeque;
+use std::hash::BuildHasher;
use std::mem::{size_of, size_of_val, take};
use std::ops::Range;
use std::sync::Arc;
@@ -27,7 +28,7 @@ use arrow::array::{
UInt32Array, make_array, new_empty_array,
};
use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
-use arrow::compute::{SortOptions, cast, filter};
+use arrow::compute::{SortOptions, cast, filter, take as take_array};
use arrow::datatypes::{DataType, Field, FieldRef, Fields};
use arrow::row::{OwnedRow, Row, RowConverter, Rows, SortField};
use arrow_select::dictionary::garbage_collect_any_dictionary;
@@ -35,10 +36,12 @@ use arrow_select::dictionary::garbage_collect_any_dictionary;
use datafusion_common::cast::as_list_array;
use datafusion_common::hash_utils::{RandomState, create_hashes};
use datafusion_common::scalar::copy_array_data;
+use datafusion_common::utils::memory::estimate_memory_size;
use datafusion_common::utils::proxy::HashTableAllocExt;
use datafusion_common::utils::{SingleRowListArrayBuilder, offset_span};
use datafusion_common::{
- Result, ScalarValue, assert_eq_or_internal_err, exec_err, internal_err,
+ HashMap, HashSet, Result, ScalarValue, assert_eq_or_internal_err, exec_err,
+ internal_datafusion_err, internal_err,
};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::format_state_name;
@@ -177,40 +180,9 @@ impl AggregateUDFImpl for ArrayAgg {
let ignore_nulls = acc_args.ignore_nulls && field.is_nullable();
if acc_args.is_distinct {
- // Limitation similar to Postgres. The aggregation function can only mix
- // DISTINCT and ORDER BY if all the expressions in the ORDER BY appear
- // also in the arguments of the function. This implies that if the
- // aggregation function only accepts one argument, only one argument
- // can be used in the ORDER BY, For example:
- //
- // ARRAY_AGG(DISTINCT col)
- //
- // can only be mixed with an ORDER BY if the order expression is "col".
- //
- // ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid
- // ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid
- // ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid
- // ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid
- let sort_option = match acc_args.order_bys {
- // This accumulator sorts the values itself, so a reversed
- // aggregate must still sort them in the requested order.
- [single] if single.expr.eq(&acc_args.exprs[0]) => {
- Some(if acc_args.is_reversed {
- !single.options
- } else {
- single.options
- })
- }
- [] => None,
- _ => {
- return exec_err!(
- "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list"
- );
- }
- };
return Ok(Box::new(DistinctArrayAggAccumulator::try_new(
data_type,
- sort_option,
+ distinct_sort_options(&acc_args)?,
ignore_nulls,
)?));
}
@@ -243,7 +215,11 @@ impl AggregateUDFImpl for ArrayAgg {
}
fn groups_accumulator_supported(&self, args: AccumulatorArgs) -> bool {
- !args.is_distinct && args.order_bys.is_empty()
+ if args.is_distinct {
+ distinct_sort_options(&args).is_ok()
+ } else {
+ args.order_bys.is_empty()
+ }
}
fn create_groups_accumulator(
@@ -253,6 +229,13 @@ impl AggregateUDFImpl for ArrayAgg {
let field = &args.expr_fields[0];
let data_type = field.data_type().clone();
let ignore_nulls = args.ignore_nulls && field.is_nullable();
+ if args.is_distinct {
+ return Ok(Box::new(DistinctArrayAggGroupsAccumulator::new(
+ data_type,
+ distinct_sort_options(&args)?,
+ ignore_nulls,
+ )));
+ }
Ok(Box::new(ArrayAggGroupsAccumulator::new(
data_type,
ignore_nulls,
@@ -347,6 +330,42 @@ impl ArrayAggAccumulator {
}
}
+/// Returns the sort options for `ARRAY_AGG(DISTINCT ...)`, or an error if the
+/// ORDER BY cannot be combined with DISTINCT.
+///
+/// Limitation similar to Postgres. The aggregation function can only mix
+/// DISTINCT and ORDER BY if all the expressions in the ORDER BY appear
+/// also in the arguments of the function. This implies that if the
+/// aggregation function only accepts one argument, only one argument
+/// can be used in the ORDER BY, For example:
+///
+/// ARRAY_AGG(DISTINCT col)
+///
+/// can only be mixed with an ORDER BY if the order expression is "col".
+///
+/// ARRAY_AGG(DISTINCT col ORDER BY col) <- Valid
+/// ARRAY_AGG(DISTINCT concat(col, '') ORDER BY concat(col, '')) <- Valid
+/// ARRAY_AGG(DISTINCT col ORDER BY other_col) <- Invalid
+/// ARRAY_AGG(DISTINCT col ORDER BY concat(col, '')) <- Invalid
+///
+/// The DISTINCT accumulators sort the values themselves, so a reversed
+/// aggregate must still sort them in the requested order.
+fn distinct_sort_options(acc_args: &AccumulatorArgs) -> Result<Option<SortOptions>> {
+ match acc_args.order_bys {
+ [single] if single.expr.eq(&acc_args.exprs[0]) => {
+ Ok(Some(if acc_args.is_reversed {
+ !single.options
+ } else {
+ single.options
+ }))
+ }
+ [] => Ok(None),
+ _ => exec_err!(
+ "In an aggregate with DISTINCT, ORDER BY expressions must appear in argument list"
+ ),
+ }
+}
+
/// Returns the mask of non-null rows in `values` when `ignore_nulls` is set
/// and some row is null.
fn non_null_mask(values: &dyn Array, ignore_nulls: bool) -> Option<BooleanArray> {
@@ -803,27 +822,12 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {
) -> Result<Vec<ArrayRef>> {
assert_eq!(values.len(), 1, "one argument to convert_to_state");
- let input = &values[0];
-
- // Each row becomes a 1-element list: offsets are [0, 1, 2, ..., n].
- let offsets = OffsetBuffer::from_repeated_length(1, input.len());
-
- // Filtered rows become null list entries, which merge_batch will skip.
- let filter_nulls = opt_filter.map(filter_to_nulls);
-
- // With ignore_nulls, null values also become null list entries. Without
- // ignore_nulls, null values stay as [NULL] so merge_batch retains them.
- let nulls = if self.ignore_nulls {
- let logical = input.logical_nulls();
- NullBuffer::union(filter_nulls.as_ref(), logical.as_ref())
- } else {
- filter_nulls
- };
-
- let field = Arc::new(Field::new_list_field(self.datatype.clone(), true));
- let list_array = ListArray::new(field, offsets, Arc::clone(input), nulls);
-
- Ok(vec![Arc::new(list_array)])
+ Ok(vec![single_value_lists(
+ &values[0],
+ opt_filter,
+ self.ignore_nulls,
+ &self.datatype,
+ )])
}
fn size(&self) -> usize {
self.batches
@@ -840,6 +844,419 @@ impl GroupsAccumulator for ArrayAggGroupsAccumulator {
}
}
+/// Converts each input row into a one-element list, the state shape of
+/// `ARRAY_AGG`.
+///
+/// Filtered rows become null list entries, which `merge_batch` skips. With
+/// `ignore_nulls`, null values also become null list entries; without it they
+/// stay as `[NULL]` so `merge_batch` retains them.
+fn single_value_lists(
+ input: &ArrayRef,
+ opt_filter: Option<&BooleanArray>,
+ ignore_nulls: bool,
+ datatype: &DataType,
+) -> ArrayRef {
+ // Offsets are [0, 1, 2, ..., n].
+ let offsets = OffsetBuffer::from_repeated_length(1, input.len());
+ let filter_nulls = opt_filter.map(filter_to_nulls);
+ let nulls = if ignore_nulls {
+ let logical = input.logical_nulls();
+ NullBuffer::union(filter_nulls.as_ref(), logical.as_ref())
+ } else {
+ filter_nulls
+ };
+ let field = Arc::new(Field::new_list_field(datatype.clone(), true));
+ Arc::new(ListArray::new(field, offsets, Arc::clone(input), nulls))
+}
+
+/// [`GroupsAccumulator`] for `ARRAY_AGG(DISTINCT x)`, optionally with
+/// `ORDER BY x`.
+///
+/// Distinct values are encoded once with a [`RowConverter`] and stored in a
+/// single [`Rows`] buffer shared by all groups, so a value that appears in many
+/// groups is held once. Each group records the values it has seen as
+/// `(group_index, value_index)` pairs, deduplicated in one hash set rather than
+/// one set per group.
+#[derive(Debug)]
+struct DistinctArrayAggGroupsAccumulator {
+ datatype: DataType,
+ sort_options: Option<SortOptions>,
+ ignore_nulls: bool,
+ /// Encodes values to rows. Created on the first non-empty input, once the
+ /// runtime type is known.
+ converter: Option<RowConverter>,
+ /// Every distinct value seen so far, indexed by value index.
+ values: Option<Rows>,
+ /// `(hash, value_index)` for every row in `values`.
+ value_map: HashTable<(u64, u32)>,
+ /// Heap size of `value_map` in bytes.
+ value_map_size: usize,
+ /// Distinct `(group_index, value_index)` pairs, in first-seen order.
+ pairs: Vec<(u32, u32)>,
+ /// Deduplicates `pairs`.
+ pair_set: HashSet<(u32, u32), RandomState>,
+ /// Total number of groups tracked.
+ num_groups: usize,
+ random_state: RandomState,
+ /// Time spent deduplicating values, reported as the DISTINCT submetric.
+ distinct_metric: Option<Arc<dyn AggregateMetric>>,
+}
+
+impl DistinctArrayAggGroupsAccumulator {
+ fn new(
+ datatype: DataType,
+ sort_options: Option<SortOptions>,
+ ignore_nulls: bool,
+ ) -> Self {
+ Self {
+ datatype,
+ sort_options,
+ ignore_nulls,
+ converter: None,
+ values: None,
+ value_map: HashTable::new(),
+ value_map_size: 0,
+ pairs: Vec::new(),
+ pair_set: HashSet::default(),
+ num_groups: 0,
+ random_state: RandomState::default(),
+ distinct_metric: None,
+ }
+ }
+
+ fn converter(&mut self, data_type: &DataType) -> Result<&RowConverter> {
+ if self.converter.is_none() {
+ let sort_field = match self.sort_options {
+ Some(opts) => SortField::new_with_options(data_type.clone(), opts),
+ None => SortField::new(data_type.clone()),
+ };
+ let converter = RowConverter::new(vec![sort_field])?;
+ self.values = Some(converter.empty_rows(0, 0));
+ self.converter = Some(converter);
+ }
+ self.converter
+ .as_ref()
+ .ok_or_else(|| internal_datafusion_err!("row converter must be set"))
+ }
+
+ /// Like [`Self::insert`], but first gathers the selected rows when they
+ /// are only part of `input`, so rejected values are never encoded.
+ fn insert_selected(
+ &mut self,
+ input: &ArrayRef,
+ rows: Vec<(usize, usize)>,
+ ) -> Result<()> {
+ if rows.len() == input.len() {
+ return self.insert(input, rows);
+ }
+ let indices = UInt32Array::from_iter_values(
+ rows.iter().map(|&(_, row_index)| row_index as u32),
+ );
+ let selected = take_array(input.as_ref(), &indices, None)?;
+ self.insert(
+ &selected,
+ rows.into_iter()
+ .enumerate()
+ .map(|(position, (group_index, _))| (group_index, position)),
+ )
+ }
+
+ /// Records that `group_index` contains each value of `input` selected by
+ /// `rows`, given as `(group_index, row_in_input)` pairs.
+ fn insert(
+ &mut self,
+ input: &ArrayRef,
+ rows: impl IntoIterator<Item = (usize, usize)>,
+ ) -> Result<()> {
+ let encoded = self
+ .converter(input.data_type())?
+ .convert_columns(std::slice::from_ref(input))?;
+ let Self {
+ values,
+ value_map,
+ value_map_size,
+ pairs,
+ pair_set,
+ random_state,
+ ..
+ } = self;
+ let values = values
+ .as_mut()
+ .ok_or_else(|| internal_datafusion_err!("value rows must be set"))?;
+
+ for (group_index, row_index) in rows {
+ let row = encoded.row(row_index);
+ let hash = random_state.hash_one(row.as_ref());
+ let value_index = match value_map.find(hash, |&(h, index)| {
+ h == hash && values.row(index as usize) == row
+ }) {
+ Some(&(_, index)) => index,
+ None => {
+ let index = values.num_rows() as u32;
+ values.push(row);
+ value_map.insert_accounted(
+ (hash, index),
+ |&(h, _)| h,
+ value_map_size,
+ );
+ index
+ }
+ };
+ let pair = (group_index as u32, value_index);
+ if pair_set.insert(pair) {
+ pairs.push(pair);
+ }
+ }
+ Ok(())
+ }
+
+ /// Drops the state of the first `emit_groups` groups and renumbers the
+ /// rest so they start from 0. Values only referenced by emitted groups are
+ /// dropped too.
+ fn retain_after(&mut self, emit_groups: usize) {
+ let emit_groups = emit_groups as u32;
+ let old_pairs = take(&mut self.pairs);
+ let old_values = self.values.take();
+ self.value_map = HashTable::new();
+ self.value_map_size = 0;
+ self.pair_set = HashSet::default();
+ self.num_groups -= emit_groups as usize;
+
+ let (Some(converter), Some(old_values)) = (&self.converter, old_values) else {
+ return;
+ };
+ let mut values = converter.empty_rows(0, 0);
+ let mut remap: HashMap<u32, u32> = HashMap::new();
+ let mut pairs = Vec::new();
+ for (group_index, value_index) in old_pairs {
+ if group_index < emit_groups {
+ continue;
+ }
+ let new_index = *remap.entry(value_index).or_insert_with(|| {
+ let row = old_values.row(value_index as usize);
+ let index = values.num_rows() as u32;
+ values.push(row);
+ let hash = self.random_state.hash_one(row.as_ref());
+ self.value_map.insert_accounted(
+ (hash, index),
+ |&(h, _)| h,
+ &mut self.value_map_size,
+ );
+ index
+ });
+ let pair = (group_index - emit_groups, new_index);
+ self.pair_set.insert(pair);
+ pairs.push(pair);
+ }
+ self.pairs = pairs;
+ self.values = Some(values);
+ }
+}
+
+impl GroupsAccumulator for DistinctArrayAggGroupsAccumulator {
+ fn update_batch(
+ &mut self,
+ values: &[ArrayRef],
+ group_indices: &[usize],
+ opt_filter: Option<&BooleanArray>,
+ total_num_groups: usize,
+ ) -> Result<()> {
+ // With `ORDER BY x` the ordering column is passed too. It is the
+ // argument itself, so only the first column is needed.
+ let Some(input) = values.first() else {
+ return internal_err!("array_agg(DISTINCT) expects an argument");
+ };
+ self.num_groups = self.num_groups.max(total_num_groups);
+
+ let nulls = if self.ignore_nulls {
+ input.logical_nulls()
+ } else {
+ None
+ };
+ let rows = group_indices
+ .iter()
+ .enumerate()
+ .filter(|&(row_index, _)| {
+ let filtered_out = opt_filter.is_some_and(|filter| {
+ filter.is_null(row_index) || !filter.value(row_index)
+ });
+ let skipped_null =
+ nulls.as_ref().is_some_and(|nulls| nulls.is_null(row_index));
+ !filtered_out && !skipped_null
+ })
+ .map(|(row_index, &group_index)| (group_index, row_index))
+ .collect::<Vec<_>>();
+ if rows.is_empty() {
+ return Ok(());
+ }
+ let mut metric_recorder =
+ AggregateMetricRecorder::new(self.distinct_metric.clone());
+ let _timer = metric_recorder.timer();
+ self.insert_selected(input, rows)
+ }
+
+ /// Produces a `ListArray` ordered by group index. Groups without values
+ /// are null, matching [`DistinctArrayAggAccumulator`].
+ fn evaluate(&mut self, emit_to: EmitTo) -> Result<ArrayRef> {
+ let emit_groups = match emit_to {
+ EmitTo::All => self.num_groups,
+ EmitTo::First(n) => n,
+ };
+
+ // Counting sort of the pairs into group order.
+ let mut counts = vec![0u32; emit_groups];
+ for &(group_index, _) in &self.pairs {
+ if let Some(count) = counts.get_mut(group_index as usize) {
+ *count += 1;
+ }
+ }
+ let mut offsets = Vec::<i32>::with_capacity(emit_groups + 1);
+ offsets.push(0);
+ let mut nulls = NullBufferBuilder::new(emit_groups);
+ let mut write_positions = Vec::with_capacity(emit_groups);
+ let mut total = 0u32;
+ for &count in &counts {
+ if count == 0 {
+ nulls.append_null();
+ } else {
+ nulls.append_non_null();
+ }
+ write_positions.push(total);
+ total += count;
+ offsets.push(total as i32);
+ }
+ let mut ordered = vec![0u32; total as usize];
+ for &(group_index, value_index) in &self.pairs {
+ if let Some(position) = write_positions.get_mut(group_index as usize) {
+ ordered[*position as usize] = value_index;
+ *position += 1;
+ }
+ }
+
+ let flat_values = match (&self.converter, &self.values) {
+ (Some(converter), Some(values)) if total > 0 => {
+ if self.sort_options.is_some() {
+ // The sort direction is encoded in the row bytes, so
+ // comparing rows gives the requested order.
+ for window in offsets.windows(2) {
+ ordered[window[0] as usize..window[1] as usize].sort_unstable_by(
+ |&a, &b| values.row(a as usize).cmp(&values.row(b as usize)),
+ );
+ }
+ }
+ let decoded = converter
+ .convert_rows(ordered.iter().map(|&i| values.row(i as usize)))?
+ .swap_remove(0);
+ // `convert_rows` always returns the physical (non-dictionary)
+ // type, so cast back when the declared type has a dictionary.
+ if decoded.data_type() != &self.datatype
+ && datatype_contains_dictionary(&self.datatype)
+ {
+ cast(decoded.as_ref(), &self.datatype)?
+ } else {
+ decoded
+ }
+ }
+ _ => new_empty_array(&self.datatype),
+ };
+
+ match emit_to {
+ EmitTo::All => {
+ let converter = self.converter.take();
+ let distinct_metric = self.distinct_metric.take();
+ *self = Self::new(
+ self.datatype.clone(),
+ self.sort_options,
+ self.ignore_nulls,
+ );
+ self.distinct_metric = distinct_metric;
+ // Keep the converter: the runtime type does not change.
+ if let Some(converter) = converter {
+ self.values = Some(converter.empty_rows(0, 0));
+ self.converter = Some(converter);
+ }
+ }
+ EmitTo::First(_) => self.retain_after(emit_groups),
+ }
+
+ let field = Arc::new(Field::new_list_field(self.datatype.clone(), true));
+ Ok(Arc::new(ListArray::new(
+ field,
+ OffsetBuffer::new(ScalarBuffer::from(offsets)),
+ flat_values,
+ nulls.finish(),
+ )))
+ }
+
+ fn state(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>> {
+ Ok(vec![self.evaluate(emit_to)?])
+ }
+
+ fn merge_batch(
+ &mut self,
+ values: &[ArrayRef],
+ group_indices: &[usize],
+ total_num_groups: usize,
+ ) -> Result<()> {
+ assert_eq_or_internal_err!(values.len(), 1, "one argument to merge_batch");
+ let lists = values[0].as_list::<i32>();
+ self.num_groups = self.num_groups.max(total_num_groups);
+
+ // A sliced list keeps its whole child array, so encode only the span
+ // its offsets cover.
+ let offsets = lists.offsets();
+ let (span_start, span_len) = offset_span(offsets);
+ let child = lists.values().slice(span_start, span_len);
+ let rows = group_indices
+ .iter()
+ .enumerate()
+ .filter(|&(list_index, _)| lists.is_valid(list_index))
+ .flat_map(|(list_index, &group_index)| {
+ let start = offsets[list_index] as usize - span_start;
+ let end = offsets[list_index + 1] as usize - span_start;
+ (start..end).map(move |position| (group_index, position))
+ })
+ .collect::<Vec<_>>();
+ if rows.is_empty() {
+ return Ok(());
+ }
+ let mut metric_recorder =
+ AggregateMetricRecorder::new(self.distinct_metric.clone());
+ let _timer = metric_recorder.timer();
+ self.insert_selected(&child, rows)
+ }
+
+ fn convert_to_state(
+ &self,
+ values: &[ArrayRef],
+ opt_filter: Option<&BooleanArray>,
+ ) -> Result<Vec<ArrayRef>> {
+ let Some(input) = values.first() else {
+ return internal_err!("array_agg(DISTINCT) expects an argument");
+ };
+ Ok(vec![single_value_lists(
+ input,
+ opt_filter,
+ self.ignore_nulls,
+ &self.datatype,
+ )])
+ }
+
+ fn set_metrics(&mut self, metrics: Arc<dyn AggregateMetrics>) {
+ self.distinct_metric = Some(metrics.metric("distinct"));
+ }
+
+ fn size(&self) -> usize {
+ size_of_val(self)
+ + self.converter.as_ref().map_or(0, RowConverter::size)
+ + self.values.as_ref().map_or(0, Rows::size)
+ + self.value_map_size
+ + self.pairs.capacity() * size_of::<(u32, u32)>()
+ + estimate_memory_size::<(u32, u32)>(self.pair_set.capacity(), 0)
+ .unwrap_or_default()
+ }
+}
+
/// Resources that are allocated lazily on the first `update_batch` call,
/// once the concrete runtime Arrow type is known.
///
@@ -1791,8 +2208,8 @@ impl Accumulator for OrderSensitiveArrayAggAccumulator {
#[cfg(test)]
mod tests {
use super::*;
- use arrow::array::{Int32Builder, ListBuilder, StringBuilder};
- use arrow::datatypes::Schema;
+ use arrow::array::{DictionaryArray, Int32Builder, ListBuilder, StringBuilder};
+ use arrow::datatypes::{Int32Type, Schema};
use datafusion_common::cast::as_generic_string_array;
use datafusion_common::internal_err;
use datafusion_physical_expr::PhysicalExpr;
@@ -3417,13 +3834,291 @@ mod tests {
}
fn eval_i32_lists(
- acc: &mut ArrayAggGroupsAccumulator,
+ acc: &mut dyn GroupsAccumulator,
emit_to: EmitTo,
) -> Result<Vec<Option<Vec<Option<i32>>>>> {
let result = acc.evaluate(emit_to)?;
Ok(list_array_to_i32_vecs(result.as_list::<i32>()))
}
+ fn distinct_groups_accumulator(
+ ignore_nulls: bool,
+ ) -> DistinctArrayAggGroupsAccumulator {
+ DistinctArrayAggGroupsAccumulator::new(DataType::Int32, None, ignore_nulls)
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_dedups_per_group() -> Result<()> {
+ let mut acc = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 1, 2, 1, 2, 3]));
+ acc.update_batch(&[values], &[0, 0, 0, 1, 1, 1], None, 2)?;
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![2, 3, 3]));
+ acc.update_batch(&[values], &[0, 1, 1], None, 2)?;
+
+ // Values shared by both groups are stored once.
+ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(3));
+
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(1), Some(2)]));
+ assert_eq!(vals[1], Some(vec![Some(1), Some(2), Some(3)]));
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_nulls() -> Result<()> {
+ let values: ArrayRef =
+ Arc::new(Int32Array::from(vec![None, Some(1), None, None]));
+ let groups = [0, 0, 0, 1];
+
+ // Without IGNORE NULLS a null is one more distinct value.
+ let mut acc = distinct_groups_accumulator(false);
+ acc.update_batch(&[Arc::clone(&values)], &groups, None, 3)?;
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![None, Some(1)]));
+ assert_eq!(vals[1], Some(vec![None]));
+ // A group with no rows is null.
+ assert_eq!(vals[2], None);
+
+ // With IGNORE NULLS a group of only nulls is null.
+ let mut acc = distinct_groups_accumulator(true);
+ acc.update_batch(&[values], &groups, None, 2)?;
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(1)]));
+ assert_eq!(vals[1], None);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_opt_filter() -> Result<()> {
+ let mut acc = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 2, 3]));
+ let filter = BooleanArray::from(vec![Some(true), Some(false), Some(true), None]);
+ acc.update_batch(&[values], &[0, 0, 0, 1], Some(&filter), 2)?;
+
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(1), Some(2)]));
+ assert_eq!(vals[1], None);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_sparse_filter() -> Result<()> {
+ // Only the selected rows are gathered and encoded, so their positions
+ // move; check each value still lands in its own group.
+ let mut acc = distinct_groups_accumulator(true);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![
+ Some(5),
+ Some(6),
+ Some(7),
+ None,
+ Some(8),
+ ]));
+ let filter = BooleanArray::from(vec![false, true, false, true, true]);
+ acc.update_batch(&[values], &[0, 1, 0, 2, 2], Some(&filter), 3)?;
+
+ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2));
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals, vec![None, Some(vec![Some(6)]), Some(vec![Some(8)])]);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_merges_sliced_state() -> Result<()> {
+ let mut partial = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 4]));
+ partial.update_batch(&[values], &[0, 1, 2, 3, 3], None, 4)?;
+ let state = partial.state(EmitTo::All)?;
+
+ // Partial aggregation emits slices of one state array.
+ let mut acc = distinct_groups_accumulator(false);
+ acc.merge_batch(&[state[0].slice(2, 2)], &[0, 1], 2)?;
+ acc.merge_batch(&[state[0].slice(0, 2)], &[1, 0], 2)?;
+
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(3), Some(2)]));
+ assert_eq!(vals[1], Some(vec![Some(4), Some(1)]));
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_records_metric() -> Result<()> {
+ let metric_updates = Arc::new(AtomicUsize::new(0));
+ let mut acc = distinct_groups_accumulator(false);
+ acc.set_metrics(Arc::new(CountingMetrics(Arc::clone(&metric_updates))));
+
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
+ acc.update_batch(&[Arc::clone(&values)], &[0, 1], None, 2)?;
+ let state = acc.state(EmitTo::All)?;
+ acc.merge_batch(&state, &[0, 1], 2)?;
+ assert_eq!(metric_updates.load(Ordering::Relaxed), 2);
+
+ // Emitting everything resets the state but keeps the metric.
+ acc.evaluate(EmitTo::All)?;
+ acc.update_batch(&[values], &[0, 1], None, 2)?;
+ assert_eq!(metric_updates.load(Ordering::Relaxed), 3);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_order_by() -> Result<()> {
+ let descending = SortOptions {
+ descending: true,
+ nulls_first: false,
+ };
+ let mut acc = DistinctArrayAggGroupsAccumulator::new(
+ DataType::Int32,
+ Some(descending),
+ false,
+ );
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![
+ Some(2),
+ None,
+ Some(5),
+ Some(2),
+ Some(1),
+ ]));
+ acc.update_batch(&[values], &[0, 0, 0, 0, 1], None, 2)?;
+
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(5), Some(2), None]));
+ assert_eq!(vals[1], Some(vec![Some(1)]));
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_emit_first() -> Result<()> {
+ let mut acc = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 10, 20, 30, 20]));
+ acc.update_batch(&[values], &[0, 0, 1, 2, 2], None, 3)?;
+
+ let vals = eval_i32_lists(&mut acc, EmitTo::First(2))?;
+ assert_eq!(vals, vec![Some(vec![Some(10)]), Some(vec![Some(20)])]);
+
+ // 10 was only referenced by an emitted group, so it is dropped.
+ assert_eq!(acc.values.as_ref().map(Rows::num_rows), Some(2));
+
+ // The remaining group is renumbered to 0 and keeps deduplicating.
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![20, 40]));
+ acc.update_batch(&[values], &[0, 0], None, 1)?;
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals, vec![Some(vec![Some(30), Some(20), Some(40)])]);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_state_merge_roundtrip() -> Result<()> {
+ let mut acc1 = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2]));
+ acc1.update_batch(&[values], &[0, 1], None, 2)?;
+
+ let mut acc2 = distinct_groups_accumulator(false);
+ let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 3, 2]));
+ acc2.update_batch(&[values], &[0, 0, 1], None, 2)?;
+
+ let state = acc2.state(EmitTo::All)?;
+ acc1.merge_batch(&state, &[0, 1], 2)?;
+
+ let vals = eval_i32_lists(&mut acc1, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(1), Some(3)]));
+ assert_eq!(vals[1], Some(vec![Some(2)]));
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_convert_to_state_merge() -> Result<()> {
+ let partial = distinct_groups_accumulator(true);
+ let values: ArrayRef =
+ Arc::new(Int32Array::from(vec![Some(7), None, Some(7), Some(8)]));
+ let filter = BooleanArray::from(vec![true, true, true, false]);
+ let state = partial.convert_to_state(&[values], Some(&filter))?;
+
+ let mut acc = distinct_groups_accumulator(true);
+ acc.merge_batch(&state, &[0, 0, 0, 1], 2)?;
+ let vals = eval_i32_lists(&mut acc, EmitTo::All)?;
+ assert_eq!(vals[0], Some(vec![Some(7)]));
+ assert_eq!(vals[1], None);
+ Ok(())
+ }
+
+ #[test]
+ fn distinct_groups_accumulator_keeps_dictionary_type() -> Result<()> {
+ let datatype =
+ DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
+ let mut acc =
+ DistinctArrayAggGroupsAccumulator::new(datatype.clone(), None, false);
+ let values: ArrayRef = Arc::new(
+ vec!["a", "b", "a", "a"]
+ .into_iter()
+ .collect::<DictionaryArray<Int32Type>>(),
+ );
+ acc.update_batch(&[values], &[0, 0, 0, 1], None, 2)?;
+
+ let result = acc.evaluate(EmitTo::All)?;
+ let lists = result.as_list::<i32>();
+ assert_eq!(lists.values().data_type(), &datatype);
+ let flat = cast(lists.values().as_ref(), &DataType::Utf8)?;
+ let flat = flat.as_string::<i32>().iter().collect::<Vec<_>>();
+ assert_eq!(flat, vec![Some("a"), Some("b"), Some("a")]);
+ assert_eq!(lists.value_offsets(), &[0, 2, 3]);
+ Ok(())
+ }
+
+ /// Compares the groups accumulator with one [`DistinctArrayAggAccumulator`]
+ /// per group over batches with repeated values, nulls and a filter.
+ #[test]
+ fn distinct_groups_accumulator_matches_row_accumulator() -> Result<()> {
+ let num_groups = 7;
+ let mut acc = distinct_groups_accumulator(false);
+ let mut expected = (0..num_groups)
+ .map(|_| DistinctArrayAggAccumulator::try_new(&DataType::Int32, None, false))
+ .collect::<Result<Vec<_>>>()?;
+
+ for batch in 0..5 {
+ let len = 50;
+ let values = (0..len)
+ .map(|i| ((i * 7 + batch * 3) % 11 != 0).then_some((i * 13 + batch) % 9))
+ .collect::<Vec<Option<i32>>>();
+ let groups = (0..len)
+ .map(|i| ((i * 5 + batch) % num_groups as i32) as usize)
+ .collect::<Vec<_>>();
+ let filter = (0..len).map(|i| (i + batch) % 4 != 0).collect::<Vec<_>>();
+
+ let array: ArrayRef = Arc::new(Int32Array::from(values.clone()));
+ acc.update_batch(
+ &[array],
+ &groups,
+ Some(&BooleanArray::from(filter.clone())),
+ num_groups,
+ )?;
+ for ((value, group), keep) in values.into_iter().zip(groups).zip(filter) {
+ if keep {
+ let value: ArrayRef = Arc::new(Int32Array::from(vec![value]));
+ expected[group].update_batch(&[value])?;
+ }
+ }
+ }
+
+ let sorted = |list: Option<Vec<Option<i32>>>| {
+ list.map(|mut values| {
+ values.sort();
+ values
+ })
+ };
+ let actual = eval_i32_lists(&mut acc, EmitTo::All)?;
+ for (group, acc) in expected.iter_mut().enumerate() {
+ let ScalarValue::List(list) = acc.evaluate()? else {
+ return internal_err!("expected a list");
+ };
+ let expected_values = list_array_to_i32_vecs(&list).swap_remove(0);
+ assert_eq!(
+ sorted(actual[group].clone()),
+ sorted(expected_values),
+ "group {group}"
+ );
+ }
+ Ok(())
+ }
+
#[test]
fn groups_accumulator_multiple_batches() -> Result<()> {
let mut acc = ArrayAggGroupsAccumulator::new(DataType::Int32, false);
diff --git a/datafusion/functions-aggregate/tests/state_compat.rs b/datafusion/functions-aggregate/tests/state_compat.rs
index e1915da9cf..5e6f794d82 100644
--- a/datafusion/functions-aggregate/tests/state_compat.rs
+++ b/datafusion/functions-aggregate/tests/state_compat.rs
@@ -27,8 +27,8 @@ use std::collections::BTreeSet;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::sync::Arc;
-use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, UInt32Array};
-use arrow::compute::{cast, concat, take};
+use arrow::array::{Array, ArrayRef, Float64Array, Int64Array, ListArray, UInt32Array};
+use arrow::compute::{cast, concat, sort, take};
use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit};
use arrow::record_batch::{RecordBatch, RecordBatchOptions};
use datafusion_common::{DataFusionError, Result, ScalarValue};
@@ -518,7 +518,10 @@ fn take_all(arrays: &[ArrayRef], indices: &UInt32Array) -> Result<Vec<ArrayRef>>
/// Runs every state exchange for one case.
fn check_case(expr: &AggregateFunctionExpr, case: &Case) -> Errors {
- let mut errors = Errors::default();
+ let mut errors = Errors {
+ ignore_list_order: case.distinct && !case.ordered,
+ ..Default::default()
+ };
let input = match Input::new(expr, case) {
Ok(input) => input,
Err(e) => {
@@ -693,7 +696,12 @@ fn guard<T>(f: impl FnOnce() -> Result<T>) -> Result<T> {
/// Failures for one case, deduplicated by message across groups.
#[derive(Default)]
-struct Errors(Vec<(String, Vec<usize>)>);
+struct Errors {
+ failures: Vec<(String, Vec<usize>)>,
+ /// Compare list results as multisets. `array_agg(DISTINCT x)` without
+ /// ORDER BY does not define the order of its elements.
+ ignore_list_order: bool,
+}
impl Errors {
fn push(&mut self, msg: &str) {
@@ -708,14 +716,14 @@ impl Errors {
// Only keep the first line, dropping e.g. the "please file a bug" text
// of internal errors.
let msg = msg.lines().next().unwrap_or_default().to_string();
- match self.0.iter_mut().find(|(m, _)| *m == msg) {
+ match self.failures.iter_mut().find(|(m, _)| *m == msg) {
Some((_, groups)) => groups.extend(group),
- None => self.0.push((msg, group.into_iter().collect())),
+ None => self.failures.push((msg, group.into_iter().collect())),
}
}
fn lines(&self) -> Vec<String> {
- self.0
+ self.failures
.iter()
.map(|(msg, groups)| {
if groups.is_empty() {
@@ -771,7 +779,7 @@ impl Errors {
) {
match got {
Err(e) => self.push_group(&format!("{what}: {e}"), group),
- Ok(got) if !scalars_match(&got, want) => {
+ Ok(got) if !scalars_match(&got, want, self.ignore_list_order) => {
self.push_group(&format!("{what}: got {got:?}, expected {want:?}"), group)
}
Ok(_) => {}
@@ -780,8 +788,15 @@ impl Errors {
}
/// Exact equality, except that floats are compared with a relative tolerance
-/// because merging in a different order can change rounding.
-fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool {
+/// because merging in a different order can change rounding, and lists are
+/// compared as multisets when `ignore_list_order` is set.
+fn scalars_match(a: &ScalarValue, b: &ScalarValue, ignore_list_order: bool) -> bool {
+ if ignore_list_order
+ && let (ScalarValue::List(x), ScalarValue::List(y)) = (a, b)
+ && let (Some(x), Some(y)) = (sorted_list_values(x), sorted_list_values(y))
+ {
+ return x.as_ref() == y.as_ref();
+ }
match (a, b) {
(ScalarValue::Float64(Some(x)), ScalarValue::Float64(Some(y))) => {
floats_match(*x, *y)
@@ -793,6 +808,15 @@ fn scalars_match(a: &ScalarValue, b: &ScalarValue) -> bool {
}
}
+/// The values of a single-row list, sorted, or `None` if the row is null or the
+/// values cannot be sorted.
+fn sorted_list_values(list: &ListArray) -> Option<ArrayRef> {
+ if list.len() != 1 || list.is_null(0) {
+ return None;
+ }
+ sort(list.value(0).as_ref(), None).ok()
+}
+
fn floats_match(x: f64, y: f64) -> bool {
if x.is_nan() || y.is_nan() {
return x.is_nan() && y.is_nan();
diff --git a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
index 89131b38bc..1638ba5455 100644
--- a/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
+++ b/datafusion/physical-plan/src/aggregates/grouped_hash_stream.rs
@@ -1500,7 +1500,10 @@ mod tests {
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_expr::{AggregateMetric, AggregateMetrics};
use datafusion_functions_aggregate::{array_agg::array_agg_udaf, count::count_udaf};
- use datafusion_physical_expr::aggregate::AggregateExprBuilder;
+ use datafusion_physical_expr::GroupsAccumulatorAdapter;
+ use datafusion_physical_expr::aggregate::{
+ AggregateExprBuilder, AggregateFunctionExpr,
+ };
use datafusion_physical_expr::expressions::col;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
@@ -1527,6 +1530,18 @@ mod tests {
}
}
+ /// `array_agg(DISTINCT)` has a native groups accumulator, so build the
+ /// per-group adapter explicitly to cover its metric handling.
+ fn distinct_adapter(
+ aggregate_expr: Arc<AggregateFunctionExpr>,
+ metrics: Arc<dyn AggregateMetrics>,
+ ) -> GroupsAccumulatorAdapter {
+ let mut adapter =
+ GroupsAccumulatorAdapter::new(move || aggregate_expr.create_accumulator());
+ adapter.set_metrics(metrics);
+ adapter
+ }
+
#[test]
fn legacy_grouped_distinct_installs_metrics_for_conversion_and_update() -> Result<()>
{
@@ -1544,13 +1559,13 @@ mod tests {
);
let metric_resolutions = Arc::new(AtomicUsize::new(0));
let metric_durations = Arc::new(AtomicUsize::new(0));
- let mut accumulator = create_group_accumulator(
- &aggregate_expr,
+ let mut accumulator = distinct_adapter(
+ aggregate_expr,
Arc::new(CountingMetrics {
resolutions: Arc::clone(&metric_resolutions),
durations: Arc::clone(&metric_durations),
}),
- )?;
+ );
let values: ArrayRef = Arc::new(UInt32Array::from(vec![1, 2, 3]));
accumulator.convert_to_state(&[Arc::clone(&values)], None)?;
@@ -1579,13 +1594,13 @@ mod tests {
);
let metric_resolutions = Arc::new(AtomicUsize::new(0));
let metric_durations = Arc::new(AtomicUsize::new(0));
- let mut accumulator = create_group_accumulator(
- &aggregate_expr,
+ let mut accumulator = distinct_adapter(
+ aggregate_expr,
Arc::new(CountingMetrics {
resolutions: Arc::clone(&metric_resolutions),
durations: Arc::clone(&metric_durations),
}),
- )?;
+ );
let mut state = ListBuilder::new(UInt32Builder::new());
for value in [1, 2, 3] {
diff --git a/datafusion/sqllogictest/test_files/array_agg.slt b/datafusion/sqllogictest/test_files/array_agg.slt
index 8342380e31..89b7bcf3f5 100644
--- a/datafusion/sqllogictest/test_files/array_agg.slt
+++ b/datafusion/sqllogictest/test_files/array_agg.slt
@@ -678,3 +678,40 @@ List(Dictionary(Int32, Utf8))
statement ok
drop table array_agg_distinct_dict_table;
+
+# array_agg(DISTINCT) per group, mixed with other aggregates and a FILTER
+statement ok
+create table array_agg_distinct_groups(g int, v varchar) as values
+ (1, 'b'), (1, 'a'), (1, 'b'), (1, null),
+ (2, 'a'), (2, 'a'),
+ (3, null),
+ (4, 'c');
+
+query II??
+select
+ g,
+ count(*),
+ array_sort(array_agg(DISTINCT v) FILTER (WHERE v <> 'c')),
+ array_sort(array_agg(DISTINCT v))
+from array_agg_distinct_groups
+group by g
+order by g;
+----
+1 4 [a, b] [NULL, a, b]
+2 2 [a] [a]
+3 1 NULL [NULL]
+4 1 NULL [c]
+
+query I?
+select g, array_agg(DISTINCT v ORDER BY v DESC) ignore nulls
+from array_agg_distinct_groups
+group by g
+order by g;
+----
+1 [b, a]
+2 [a]
+3 NULL
+4 [c]
+
+statement ok
+drop table array_agg_distinct_groups;