Skip to content

Apache DataFusion SQL Query Engine

perf(functions-aggregate): coalesce small ordered ARRAY_AGG batches (#25497)

Tiny_Murky431355fb0299
diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs
index ad6d502eb5..ebfe1f8cdd 100644
--- a/datafusion/functions-aggregate/benches/array_agg.rs
+++ b/datafusion/functions-aggregate/benches/array_agg.rs
@@ -19,21 +19,25 @@ use std::hint::black_box;
use std::sync::Arc;
use arrow::array::{
- Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder,
- StringArray,
+ Array, ArrayRef, ArrowPrimitiveType, AsArray, Int64Array, ListArray,
+ NullBufferBuilder, StringArray,
};
-use arrow::datatypes::{DataType, Field, Int64Type};
-use criterion::{Criterion, criterion_group, criterion_main};
-use datafusion_expr::Accumulator;
+use arrow::datatypes::{DataType, Field, FieldRef, Int64Type, Schema};
+use criterion::{BatchSize, Criterion, criterion_group, criterion_main};
+use datafusion_expr::function::AccumulatorArgs;
+use datafusion_expr::{Accumulator, AggregateUDFImpl};
use datafusion_functions_aggregate::array_agg::{
- ArrayAggAccumulator, DistinctArrayAggAccumulator,
+ ArrayAgg, ArrayAggAccumulator, DistinctArrayAggAccumulator,
};
+use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};
+use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use arrow::buffer::OffsetBuffer;
use arrow::util::bench_util::create_primitive_array;
use rand::Rng;
use rand::SeedableRng;
use rand::prelude::StdRng;
+use rand::seq::SliceRandom;
/// Returns fixed seedable RNG
pub fn seedable_rng() -> StdRng {
@@ -288,5 +292,229 @@ fn distinct_array_agg_benchmark(c: &mut Criterion) {
);
}
-criterion_group!(benches, array_agg_benchmark, distinct_array_agg_benchmark);
+/// Precomputes the schema, physical expressions, sort expression, and aggregate
+/// metadata so each benchmark iteration can focus on accumulator operations.
+struct OrderedArrayAggBenchFixture {
+ schema: Schema,
+ value_expr: Arc<dyn PhysicalExpr>,
+ value_field: FieldRef,
+ order_by: PhysicalSortExpr,
+ array_agg: Arc<dyn AggregateUDFImpl>,
+ return_field: FieldRef,
+}
+
+impl OrderedArrayAggBenchFixture {
+ fn new(value_type: &DataType, input_preordered: bool) -> Self {
+ let schema = Schema::new(vec![
+ Field::new("value", value_type.clone(), false),
+ Field::new("ordering", DataType::Int64, false),
+ ]);
+
+ let value_expr = col("value", &schema).unwrap();
+ let ordering_expr = col("ordering", &schema).unwrap();
+
+ let value_field = value_expr.return_field(&schema).unwrap();
+
+ let order_by = PhysicalSortExpr::new(
+ ordering_expr,
+ arrow::compute::SortOptions {
+ descending: false,
+ nulls_first: false,
+ },
+ );
+
+ let array_agg = Arc::new(ArrayAgg::default())
+ .with_beneficial_ordering(input_preordered)
+ .unwrap()
+ .unwrap();
+
+ let return_field = Field::new(
+ "array_agg",
+ DataType::List(Field::new_list_field(value_type.clone(), true).into()),
+ true,
+ )
+ .into();
+
+ Self {
+ schema,
+ value_expr,
+ value_field,
+ order_by,
+ array_agg,
+ return_field,
+ }
+ }
+ fn create_accumulator(&self) -> Box<dyn Accumulator> {
+ self.array_agg
+ .accumulator(AccumulatorArgs {
+ return_field: Arc::clone(&self.return_field),
+ schema: &self.schema,
+ expr_fields: std::slice::from_ref(&self.value_field),
+ ignore_nulls: false,
+ order_bys: std::slice::from_ref(&self.order_by),
+ is_reversed: false,
+ name: "array_agg(value ORDER BY ordering)",
+ is_distinct: false,
+ exprs: std::slice::from_ref(&self.value_expr),
+ })
+ .unwrap()
+ }
+}
+
+const ORDERED_ARRAY_AGG_ROWS: usize = 2048;
+
+fn create_ordered_array_agg_batches(
+ rows_per_batch: usize,
+ input_preordered: bool,
+) -> Vec<[ArrayRef; 2]> {
+ assert!(
+ rows_per_batch > 0,
+ "rows_per_batch must be greater than zero"
+ );
+
+ let upper_value: i64 = ORDERED_ARRAY_AGG_ROWS.try_into().unwrap();
+ let mut values = (0..upper_value).collect::<Vec<i64>>();
+
+ if !input_preordered {
+ let mut rng = StdRng::seed_from_u64(42);
+ values.shuffle(&mut rng);
+ }
+
+ values
+ .chunks(rows_per_batch)
+ .map(|batch_values| {
+ let values = Arc::new(Int64Array::from(batch_values.to_vec())) as ArrayRef;
+
+ [
+ Arc::clone(&values),
+ values, // Reuse the payload values as ordering keys.
+ ]
+ })
+ .collect()
+}
+
+fn ordered_array_agg_bench(
+ c: &mut Criterion,
+ name: &str,
+ value_type: &DataType,
+ batches: &[[ArrayRef; 2]],
+ input_preordered: bool,
+) {
+ c.bench_function(name, |b| {
+ let fixture = OrderedArrayAggBenchFixture::new(value_type, input_preordered);
+ b.iter(|| {
+ let mut accumulator = fixture.create_accumulator();
+
+ for batch in batches {
+ accumulator
+ .update_batch(batch)
+ .expect("update_batch should succeed");
+ }
+
+ let result = accumulator.evaluate().expect("evaluate should succeed");
+
+ black_box(result);
+ })
+ });
+}
+
+fn ordered_array_agg_benchmark(c: &mut Criterion) {
+ for rows_per_batch in [1, 8, 64, ORDERED_ARRAY_AGG_ROWS] {
+ let ordered_batches = create_ordered_array_agg_batches(rows_per_batch, true);
+ ordered_array_agg_bench(
+ c,
+ &format!(
+ "ordered_array_agg i64 ordered input, \
+ {rows_per_batch} rows per update_batch"
+ ),
+ &DataType::Int64,
+ &ordered_batches,
+ true,
+ );
+ let shuffled_batches = create_ordered_array_agg_batches(rows_per_batch, false);
+ ordered_array_agg_bench(
+ c,
+ &format!(
+ "ordered_array_agg i64 random input, \
+ {rows_per_batch} rows per update_batch"
+ ),
+ &DataType::Int64,
+ &shuffled_batches,
+ false,
+ );
+ }
+}
+
+/// Creates preordered batches with a 4 KiB Utf8 payload per row.
+fn create_wide_utf8_batches(rows_per_batch: usize) -> Vec<[ArrayRef; 2]> {
+ assert!(
+ rows_per_batch > 0,
+ "rows_per_batch must be greater than zero"
+ );
+
+ let values = vec!["x".repeat(4 * 1_024); ORDERED_ARRAY_AGG_ROWS];
+
+ values
+ .chunks(rows_per_batch)
+ .enumerate()
+ .map(|(idx, batch_values)| {
+ let payload =
+ Arc::new(StringArray::from_iter_values(batch_values.iter())) as ArrayRef;
+ let start = idx * rows_per_batch;
+ let ordering_values = Arc::new(Int64Array::from(
+ (start..start + batch_values.len())
+ .map(|row| i64::try_from(row).unwrap())
+ .collect::<Vec<_>>(),
+ )) as ArrayRef;
+
+ [payload, ordering_values]
+ })
+ .collect()
+}
+
+fn ordered_array_agg_update_only_bench(
+ c: &mut Criterion,
+ name: &str,
+ value_type: &DataType,
+ batches: &[[ArrayRef; 2]],
+ input_preordered: bool,
+) {
+ let fixture = OrderedArrayAggBenchFixture::new(value_type, input_preordered);
+
+ c.bench_function(name, |b| {
+ b.iter_batched(
+ || fixture.create_accumulator(),
+ |mut accumulator| {
+ for batch in batches {
+ accumulator
+ .update_batch(batch)
+ .expect("update_batch should succeed");
+ }
+
+ // Return the accumulator so Criterion drops it after timing the updates.
+ accumulator
+ },
+ BatchSize::PerIteration,
+ );
+ });
+}
+
+fn ordered_array_agg_wide_utf8_benchmark(c: &mut Criterion) {
+ let ordered_batches = create_wide_utf8_batches(1);
+ ordered_array_agg_update_only_bench(
+ c,
+ "ordered_array_agg utf8 4 KiB, 1 row per update_batch",
+ &DataType::Utf8,
+ &ordered_batches,
+ true,
+ );
+}
+
+criterion_group!(
+ benches,
+ array_agg_benchmark,
+ distinct_array_agg_benchmark,
+ ordered_array_agg_benchmark,
+ ordered_array_agg_wide_utf8_benchmark
+);
criterion_main!(benches);
diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs
index 546fad9738..abf1c8376f 100644
--- a/datafusion/functions-aggregate/src/array_agg.rs
+++ b/datafusion/functions-aggregate/src/array_agg.rs
@@ -1246,6 +1246,14 @@ struct OrderedArrayAggEntry {
row_idx: usize,
}
+/// Coalesce consecutive small payload batches up to this row count to reduce
+/// per-array memory overhead while limiting repeated concatenation costs.
+const ORDERED_ARRAY_AGG_COALESCE_ROWS: usize = 64;
+
+/// Skip coalescing once the tail batch is large enough that the fixed
+/// per-array overhead is negligible relative to the payload.
+const ORDERED_ARRAY_AGG_COALESCE_BYTES: usize = 4096;
+
#[derive(Debug)]
pub(crate) struct OrderSensitiveArrayAggAccumulator {
/// Arrow payload arrays. Entries refer to rows in these batches.
@@ -1515,27 +1523,98 @@ impl OrderSensitiveArrayAggAccumulator {
} else {
(values, None)
};
+
let ordering_values = filtered_ordering_values
.as_deref()
.unwrap_or(ordering_values);
- // Detach the stored payload from potentially oversized backing buffers.
- let values = make_array(copy_array_data(&values.to_data()));
- let values = compact_payload(values)?;
- let row_count = values.len();
// RowConverter validates the number, lengths, and types of ordering columns.
self.ordering_converter
.append(&mut self.ordering_rows, ordering_values)?;
+
+ let row_count = values.len();
+
if row_count == 0 {
return Ok(None);
}
- let start = self.entries.len();
- let batch_idx = self.batches.len();
- self.batches.push(values);
- self.entries.extend(
- (0..row_count).map(|row_idx| OrderedArrayAggEntry { batch_idx, row_idx }),
+ // Concat copies these payloads into fresh buffers when they fit in the tail.
+ let concat_copies = values.data_type().is_primitive()
+ || matches!(
+ values.data_type(),
+ DataType::Boolean
+ | DataType::Utf8
+ | DataType::LargeUtf8
+ | DataType::Binary
+ | DataType::LargeBinary
+ );
+
+ // View concat shares input buffers, but compacting the merged array
+ // detaches them.
+ // If the input fits in the tail, the earlier detach is unnecessary.
+ let compact_view_after_concat = matches!(
+ values.data_type(),
+ DataType::Utf8View | DataType::BinaryView
);
+
+ let fits_tail = |value: &ArrayRef| {
+ self.batches.last().is_some_and(|last| {
+ last.len() + row_count <= ORDERED_ARRAY_AGG_COALESCE_ROWS
+ && last.get_buffer_memory_size() + value.get_array_memory_size()
+ <= ORDERED_ARRAY_AGG_COALESCE_BYTES
+ })
+ };
+
+ // Skip the first detach only when merging will detach the input:
+ // concat copies the types above, and post-concat compaction copies views.
+ let skip_detach =
+ (concat_copies || compact_view_after_concat) && fits_tail(&values);
+
+ // Detach inputs stored separately or needing compaction before the cap check.
+ let values = if skip_detach {
+ values
+ } else {
+ compact_payload(make_array(copy_array_data(&values.to_data())))?
+ };
+
+ // Recheck the cap after compaction, which can shrink view and dictionary buffers.
+ let should_coalesce = fits_tail(&values);
+
+ let start = self.entries.len();
+ let (batch_idx, row_offset) = match self.batches.last() {
+ Some(last_batch) if should_coalesce => {
+ let merged =
+ arrow::compute::concat(&[last_batch.as_ref(), values.as_ref()])?;
+
+ // View concat shares input data buffers. Compact them so the
+ // tail does not retain one buffer per small input batch.
+ let merged = if compact_view_after_concat {
+ compact_payload(merged)?
+ } else {
+ merged
+ };
+
+ // Concatenation preserves the existing row offsets, while new rows
+ // start at the previous length of the tail batch.
+ let batch_idx = self.batches.len() - 1;
+ let row_offset = last_batch.len();
+
+ self.batches[batch_idx] = merged;
+ (batch_idx, row_offset)
+ }
+ _ => {
+ let batch_idx = self.batches.len();
+ self.batches.push(values);
+ (batch_idx, 0)
+ }
+ };
+
+ self.entries
+ .extend((0..row_count).map(|row_idx| OrderedArrayAggEntry {
+ batch_idx,
+ row_idx: row_offset + row_idx,
+ }));
+
self.sorted_entry_indices = None;
debug_assert_eq!(self.entries.len(), self.ordering_rows.num_rows());
@@ -2146,6 +2225,19 @@ mod tests {
Ok(())
}
+ #[test]
+ fn ordered_array_agg_detaches_unmerged_slices() -> Result<()> {
+ use arrow::array::{Int64Array, StringArray};
+
+ let integers = Int64Array::from_iter_values(0..10_000);
+ let strings = StringArray::from_iter_values(
+ (0..10_000).map(|i| format!("payload-value-{i:03}")),
+ );
+ assert_compact_ordered_payload(Arc::new(integers), 2_000)?;
+ assert_compact_ordered_payload(Arc::new(strings), 2_000)?;
+ Ok(())
+ }
+
#[test]
fn ordered_array_agg_compacts_views() -> Result<()> {
use arrow::array::{BinaryViewArray, StringViewArray};
@@ -2513,6 +2605,10 @@ mod tests {
final_acc.merge_batch(&state_a)?;
final_acc.merge_batch(&state_b)?;
+ // Small partial-state batches are coalesced into one payload batch.
+ assert_eq!(final_acc.batches.len(), 1);
+ assert_eq!(final_acc.batches[0].len(), 4);
+
let result = print_nulls(str_arr(final_acc.evaluate()?)?);
assert_eq!(result, vec!["a1", "a2", "b1", "b2"]);
Ok(())
@@ -2908,6 +3004,264 @@ mod tests {
Ok(())
}
+ #[test]
+ fn ordered_array_agg_coalesces_small_batches_up_to_threshold() -> Result<()> {
+ use arrow::array::Int64Array;
+
+ let mut acc = ordered_accumulator(
+ DataType::Int64,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ let threshold = ORDERED_ARRAY_AGG_COALESCE_ROWS;
+ let threshold_i64 = i64::try_from(threshold).unwrap();
+
+ for values in [
+ (0..(threshold_i64 - 1)).collect::<Vec<_>>(),
+ vec![threshold_i64 - 1],
+ vec![threshold_i64],
+ ((threshold_i64 + 1)..(threshold_i64 * 2)).collect::<Vec<i64>>(),
+ vec![threshold_i64 * 2],
+ ] {
+ let values = Arc::new(Int64Array::from(values)) as ArrayRef;
+
+ acc.update_batch(&[Arc::clone(&values), values])?;
+ }
+
+ assert_eq!(
+ acc.batches
+ .iter()
+ .map(|batch| batch.len())
+ .collect::<Vec<usize>>(),
+ vec![threshold, threshold, 1],
+ );
+
+ assert_eq!(acc.entries.len(), threshold * 2 + 1);
+ assert_eq!(acc.ordering_rows.num_rows(), threshold * 2 + 1);
+
+ assert_eq!(acc.entries[threshold - 1].batch_idx, 0);
+ assert_eq!(acc.entries[threshold - 1].row_idx, threshold - 1);
+
+ assert_eq!(acc.entries[threshold].batch_idx, 1);
+ assert_eq!(acc.entries[threshold].row_idx, 0);
+
+ assert_eq!(acc.entries[threshold * 2 - 1].batch_idx, 1);
+ assert_eq!(acc.entries[threshold * 2 - 1].row_idx, threshold - 1);
+
+ assert_eq!(acc.entries[threshold * 2].batch_idx, 2);
+ assert_eq!(acc.entries[threshold * 2].row_idx, 0);
+ Ok(())
+ }
+
+ #[test]
+ fn ordered_array_agg_sorts_across_coalesced_int64_batches() -> Result<()> {
+ use arrow::array::Int64Array;
+
+ let mut acc = ordered_accumulator(
+ DataType::Int64,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ for value in (0_i64..128).rev() {
+ let payload = Arc::new(Int64Array::from(vec![value])) as ArrayRef;
+ let ordering = Arc::new(Int64Array::from(vec![value + 1000])) as ArrayRef;
+ acc.update_batch(&[payload, ordering])?;
+ }
+
+ let ScalarValue::List(result) = acc.evaluate()? else {
+ return internal_err!("expect list");
+ };
+
+ assert_eq!(
+ result
+ .values()
+ .as_primitive::<arrow::datatypes::Int64Type>()
+ .values(),
+ &(0..128).collect::<Vec<i64>>()
+ );
+
+ Ok(())
+ }
+
+ #[test]
+ fn ordered_array_agg_does_not_coalesce_large_payloads() -> Result<()> {
+ use arrow::array::{Int64Array, StringArray};
+
+ let mut acc = ordered_accumulator(
+ DataType::Utf8,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ let value = "x".repeat(ORDERED_ARRAY_AGG_COALESCE_BYTES);
+
+ for i in 0_i64..2 {
+ let payload = Arc::new(StringArray::from(vec![value.as_str()])) as ArrayRef;
+ let ordering = Arc::new(Int64Array::from(vec![i])) as ArrayRef;
+ acc.update_batch(&[payload, ordering])?;
+ }
+
+ // Each payload exceeds the byte cap, so the tail batch must not be coalesced.
+ assert_eq!(acc.batches.len(), 2);
+ assert_eq!(acc.entries[1].batch_idx, 1);
+ assert_eq!(acc.entries[1].row_idx, 0);
+ Ok(())
+ }
+
+ #[test]
+ fn ordered_array_agg_sorts_across_coalesced_utf8_view_batches() -> Result<()> {
+ use arrow::array::{Int64Array, StringViewArray};
+
+ let mut acc = ordered_accumulator(
+ DataType::Utf8View,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ // Longer than 12 bytes so view payloads are stored in data buffers.
+ let value = |i: usize| format!("payload-value-{i:03}");
+ let total_payloads = 100;
+
+ for i in (0..total_payloads).rev() {
+ let payload =
+ Arc::new(StringViewArray::from_iter_values([value(i)])) as ArrayRef;
+ let ordering =
+ Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()])) as ArrayRef;
+ acc.update_batch(&[payload, ordering])?;
+ }
+
+ assert_eq!(
+ acc.batches
+ .iter()
+ .map(|batch| batch.len())
+ .collect::<Vec<_>>(),
+ vec![ORDERED_ARRAY_AGG_COALESCE_ROWS, 36],
+ );
+ assert_eq!(acc.batches[0].as_string_view().data_buffers().len(), 1);
+
+ let ScalarValue::List(result) = acc.evaluate()? else {
+ return internal_err!("expect list");
+ };
+
+ let values = result.values().as_string_view();
+
+ for i in 0..total_payloads {
+ assert_eq!(values.value(i), value(i));
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn ordered_array_agg_sorts_across_coalesced_binary_view_batches() -> Result<()> {
+ use arrow::array::{BinaryViewArray, Int64Array};
+
+ let mut acc = ordered_accumulator(
+ DataType::BinaryView,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ // Longer than 12 bytes so view payloads are stored in data buffers.
+ let value = |i: usize| format!("payload-value-{i:03}").into_bytes();
+ let total_payloads = 100;
+
+ for i in (0..total_payloads).rev() {
+ let payload =
+ Arc::new(BinaryViewArray::from_iter_values([value(i)])) as ArrayRef;
+ let ordering =
+ Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()])) as ArrayRef;
+ acc.update_batch(&[payload, ordering])?;
+ }
+
+ assert_eq!(
+ acc.batches
+ .iter()
+ .map(|batch| batch.len())
+ .collect::<Vec<_>>(),
+ vec![ORDERED_ARRAY_AGG_COALESCE_ROWS, 36],
+ );
+ assert_eq!(acc.batches[0].as_binary_view().data_buffers().len(), 1);
+
+ let ScalarValue::List(result) = acc.evaluate()? else {
+ return internal_err!("expect list");
+ };
+
+ let values = result.values().as_binary_view();
+ for i in 0..total_payloads {
+ assert_eq!(values.value(i), value(i));
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn ordered_array_agg_sorts_across_coalesced_dictionary_batches() -> Result<()> {
+ use arrow::array::{ArrayAccessor, DictionaryArray, Int64Array, StringArray};
+ use arrow::datatypes::Int8Type;
+
+ let dictionary_type =
+ DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
+ let mut acc = ordered_accumulator(
+ dictionary_type,
+ DataType::Int64,
+ SortOptions::new(false, false),
+ false,
+ false,
+ )?;
+
+ // Each input has its own dictionary, so coalescing must remap its keys.
+ let value = |i: usize| format!("payload-value-{i:03}");
+ let total_payloads = 100;
+
+ for i in (0..total_payloads).rev() {
+ let payload = Arc::new(
+ std::iter::once(value(i).as_str()).collect::<DictionaryArray<Int8Type>>(),
+ ) as ArrayRef;
+ let ordering =
+ Arc::new(Int64Array::from(vec![i64::try_from(i).unwrap()])) as ArrayRef;
+ acc.update_batch(&[payload, ordering])?;
+ }
+
+ assert_eq!(
+ acc.batches
+ .iter()
+ .map(|batch| batch.len())
+ .collect::<Vec<_>>(),
+ vec![ORDERED_ARRAY_AGG_COALESCE_ROWS, 36],
+ );
+
+ let ScalarValue::List(result) = acc.evaluate()? else {
+ return internal_err!("expect list");
+ };
+
+ let values = result
+ .values()
+ .as_any()
+ .downcast_ref::<DictionaryArray<Int8Type>>()
+ .expect("expected DictionaryArray<Int8Type>")
+ .downcast_dict::<StringArray>()
+ .expect("expected Utf8 dictionary values");
+
+ for i in 0..total_payloads {
+ assert_eq!(values.value(i), value(i));
+ }
+
+ Ok(())
+ }
+
struct ArrayAggAccumulatorBuilder {
return_field: FieldRef,
distinct: bool,