Skip to content

Apache DataFusion SQL Query Engine

chore: Add end-to-end benchmark for array_agg, code cleanup (#20496)

Neil Conwayd303f5817f69
diff --git a/datafusion/core/benches/aggregate_query_sql.rs b/datafusion/core/benches/aggregate_query_sql.rs
index f785c94580..b47512e5e9 100644
--- a/datafusion/core/benches/aggregate_query_sql.rs
+++ b/datafusion/core/benches/aggregate_query_sql.rs
@@ -251,6 +251,39 @@ fn criterion_benchmark(c: &mut Criterion) {
)
})
});
+
+ c.bench_function("array_agg_query_group_by_few_groups", |b| {
+ b.iter(|| {
+ query(
+ ctx.clone(),
+ &rt,
+ "SELECT u64_narrow, array_agg(f64) \
+ FROM t GROUP BY u64_narrow",
+ )
+ })
+ });
+
+ c.bench_function("array_agg_query_group_by_mid_groups", |b| {
+ b.iter(|| {
+ query(
+ ctx.clone(),
+ &rt,
+ "SELECT u64_mid, array_agg(f64) \
+ FROM t GROUP BY u64_mid",
+ )
+ })
+ });
+
+ c.bench_function("array_agg_query_group_by_many_groups", |b| {
+ b.iter(|| {
+ query(
+ ctx.clone(),
+ &rt,
+ "SELECT u64_wide, array_agg(f64) \
+ FROM t GROUP BY u64_wide",
+ )
+ })
+ });
}
criterion_group!(benches, criterion_benchmark);
diff --git a/datafusion/core/benches/data_utils/mod.rs b/datafusion/core/benches/data_utils/mod.rs
index accd51ae58..a30ada4205 100644
--- a/datafusion/core/benches/data_utils/mod.rs
+++ b/datafusion/core/benches/data_utils/mod.rs
@@ -45,7 +45,7 @@ pub fn create_table_provider(
) -> Result<Arc<MemTable>> {
let schema = Arc::new(create_schema());
let partitions =
- create_record_batches(schema.clone(), array_len, partitions_len, batch_size);
+ create_record_batches(&schema, array_len, partitions_len, batch_size);
// declare a table in memory. In spark API, this corresponds to createDataFrame(...).
MemTable::try_new(schema, partitions).map(Arc::new)
}
@@ -56,21 +56,19 @@ pub fn create_schema() -> Schema {
Field::new("utf8", DataType::Utf8, false),
Field::new("f32", DataType::Float32, false),
Field::new("f64", DataType::Float64, true),
- // This field will contain integers randomly selected from a large
- // range of values, i.e. [0, u64::MAX], such that there are none (or
- // very few) repeated values.
- Field::new("u64_wide", DataType::UInt64, true),
- // This field will contain integers randomly selected from a narrow
- // range of values such that there are a few distinct values, but they
- // are repeated often.
+ // Integers randomly selected from a wide range of values, i.e. [0,
+ // u64::MAX], such that there are ~no repeated values.
+ Field::new("u64_wide", DataType::UInt64, false),
+ // Integers randomly selected from a mid-range of values [0, 1000),
+ // providing ~1000 distinct groups.
+ Field::new("u64_mid", DataType::UInt64, false),
+ // Integers randomly selected from a narrow range of values such that
+ // there are a few distinct values, but they are repeated often.
Field::new("u64_narrow", DataType::UInt64, false),
])
}
-fn create_data(size: usize, null_density: f64) -> Vec<Option<f64>> {
- // use random numbers to avoid spurious compiler optimizations wrt to branching
- let mut rng = StdRng::seed_from_u64(42);
-
+fn create_data(rng: &mut StdRng, size: usize, null_density: f64) -> Vec<Option<f64>> {
(0..size)
.map(|_| {
if rng.random::<f64>() > null_density {
@@ -82,56 +80,43 @@ fn create_data(size: usize, null_density: f64) -> Vec<Option<f64>> {
.collect()
}
-fn create_integer_data(
- rng: &mut StdRng,
- size: usize,
- value_density: f64,
-) -> Vec<Option<u64>> {
- (0..size)
- .map(|_| {
- if rng.random::<f64>() > value_density {
- None
- } else {
- Some(rng.random::<u64>())
- }
- })
- .collect()
-}
-
fn create_record_batch(
schema: SchemaRef,
rng: &mut StdRng,
batch_size: usize,
- i: usize,
+ batch_index: usize,
) -> RecordBatch {
- // the 4 here is the number of different keys.
- // a higher number increase sparseness
- let vs = [0, 1, 2, 3];
- let keys: Vec<String> = (0..batch_size)
- .map(
- // use random numbers to avoid spurious compiler optimizations wrt to branching
- |_| format!("hi{:?}", vs.choose(rng)),
- )
- .collect();
- let keys: Vec<&str> = keys.iter().map(|e| &**e).collect();
+ // Randomly choose from 4 distinct key values; a higher number increases sparseness.
+ let key_suffixes = [0, 1, 2, 3];
+ let keys = StringArray::from_iter_values(
+ (0..batch_size).map(|_| format!("hi{}", key_suffixes.choose(rng).unwrap())),
+ );
- let values = create_data(batch_size, 0.5);
+ let values = create_data(rng, batch_size, 0.5);
// Integer values between [0, u64::MAX].
- let integer_values_wide = create_integer_data(rng, batch_size, 9.0);
+ let integer_values_wide = (0..batch_size)
+ .map(|_| rng.random::<u64>())
+ .collect::<Vec<_>>();
- // Integer values between [0, 9].
+ // Integer values between [0, 1000).
+ let integer_values_mid = (0..batch_size)
+ .map(|_| rng.random_range(0..1000))
+ .collect::<Vec<_>>();
+
+ // Integer values between [0, 10).
let integer_values_narrow = (0..batch_size)
- .map(|_| rng.random_range(0_u64..10))
+ .map(|_| rng.random_range(0..10))
.collect::<Vec<_>>();
RecordBatch::try_new(
schema,
vec![
- Arc::new(StringArray::from(keys)),
- Arc::new(Float32Array::from(vec![i as f32; batch_size])),
+ Arc::new(keys),
+ Arc::new(Float32Array::from(vec![batch_index as f32; batch_size])),
Arc::new(Float64Array::from(values)),
Arc::new(UInt64Array::from(integer_values_wide)),
+ Arc::new(UInt64Array::from(integer_values_mid)),
Arc::new(UInt64Array::from(integer_values_narrow)),
],
)
@@ -140,21 +125,29 @@ fn create_record_batch(
/// Create record batches of `partitions_len` partitions and `batch_size` for each batch,
/// with a total number of `array_len` records
-#[expect(clippy::needless_pass_by_value)]
pub fn create_record_batches(
- schema: SchemaRef,
+ schema: &SchemaRef,
array_len: usize,
partitions_len: usize,
batch_size: usize,
) -> Vec<Vec<RecordBatch>> {
let mut rng = StdRng::seed_from_u64(42);
- (0..partitions_len)
- .map(|_| {
- (0..array_len / batch_size / partitions_len)
- .map(|i| create_record_batch(schema.clone(), &mut rng, batch_size, i))
- .collect::<Vec<_>>()
- })
- .collect::<Vec<_>>()
+ let mut partitions = Vec::with_capacity(partitions_len);
+ let batches_per_partition = array_len / batch_size / partitions_len;
+
+ for _ in 0..partitions_len {
+ let mut batches = Vec::with_capacity(batches_per_partition);
+ for batch_index in 0..batches_per_partition {
+ batches.push(create_record_batch(
+ schema.clone(),
+ &mut rng,
+ batch_size,
+ batch_index,
+ ));
+ }
+ partitions.push(batches);
+ }
+ partitions
}
/// An enum that wraps either a regular StringBuilder or a GenericByteViewBuilder