Skip to content

Apache DataFusion SQL Query Engine

chore: use bench array helpers from Arrow bench_util (#21544)

theirix43d32a8c14b1
diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs
index 793c2aac96..b0d8148c3e 100644
--- a/datafusion/functions-aggregate/benches/array_agg.rs
+++ b/datafusion/functions-aggregate/benches/array_agg.rs
@@ -20,7 +20,6 @@ use std::sync::Arc;
use arrow::array::{
Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder,
- PrimitiveArray,
};
use arrow::datatypes::{Field, Int64Type};
use criterion::{Criterion, criterion_group, criterion_main};
@@ -28,6 +27,7 @@ use datafusion_expr::Accumulator;
use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator;
use arrow::buffer::OffsetBuffer;
+use arrow::util::bench_util::create_primitive_array;
use rand::Rng;
use rand::SeedableRng;
use rand::distr::{Distribution, StandardUniform};
@@ -54,24 +54,6 @@ fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) {
});
}
-pub fn create_primitive_array<T>(size: usize, null_density: f32) -> PrimitiveArray<T>
-where
- T: ArrowPrimitiveType,
- StandardUniform: Distribution<T::Native>,
-{
- let mut rng = seedable_rng();
-
- (0..size)
- .map(|_| {
- if rng.random::<f32>() < null_density {
- None
- } else {
- Some(rng.random())
- }
- })
- .collect()
-}
-
/// Create List array with the given item data type, null density, null locations and zero length lists density
/// Creates a random (but fixed-seeded) array of a given size and null density
pub fn create_list_array<T>(
diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml
index 13b0029615..d2204a7be7 100644
--- a/datafusion/functions-nested/Cargo.toml
+++ b/datafusion/functions-nested/Cargo.toml
@@ -64,6 +64,7 @@ log = { workspace = true }
memchr = { workspace = true }
[dev-dependencies]
+arrow = { workspace = true, features = ["test_utils"] }
criterion = { workspace = true, features = ["async_tokio"] }
rand = { workspace = true }
diff --git a/datafusion/functions-nested/benches/array_min_max.rs b/datafusion/functions-nested/benches/array_min_max.rs
index 391a84d148..45838da79f 100644
--- a/datafusion/functions-nested/benches/array_min_max.rs
+++ b/datafusion/functions-nested/benches/array_min_max.rs
@@ -17,73 +17,19 @@
use std::sync::Arc;
-use arrow::array::{ArrayRef, Int64Array, ListArray};
-use arrow::buffer::{NullBuffer, OffsetBuffer};
-use arrow::datatypes::{DataType, Field};
+use arrow::array::{Array, ArrayRef};
+use arrow::datatypes::{DataType, Field, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
use datafusion_functions_nested::min_max::ArrayMax;
-use rand::rngs::StdRng;
-use rand::{Rng, SeedableRng};
const NUM_ROWS: usize = 8192;
const SEED: u64 = 42;
const LIST_NULL_DENSITY: f64 = 0.1;
const ELEMENT_NULL_DENSITY: f64 = 0.1;
-fn create_int64_list_array(
- num_rows: usize,
- list_size: usize,
- element_null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let total_values = num_rows * list_size;
-
- if element_null_density > 0.0 {
- let values: Vec<Option<i64>> = (0..total_values)
- .map(|_| {
- if rng.random::<f64>() < element_null_density {
- None
- } else {
- Some(rng.random::<i64>() % 10_000)
- }
- })
- .collect();
- let values_array = Arc::new(Int64Array::from(values));
-
- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();
- let nulls: Vec<bool> = (0..num_rows)
- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)
- .collect();
-
- Arc::new(ListArray::new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- values_array,
- Some(NullBuffer::from(nulls)),
- ))
- } else {
- // No element nulls — values array has no null buffer
- let values: Vec<i64> = (0..total_values)
- .map(|_| rng.random::<i64>() % 10_000)
- .collect();
- let values_array = Arc::new(Int64Array::from(values));
-
- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();
- let nulls: Vec<bool> = (0..num_rows)
- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)
- .collect();
-
- Arc::new(ListArray::new(
- Arc::new(Field::new("item", DataType::Int64, false)),
- OffsetBuffer::new(offsets.into()),
- values_array,
- Some(NullBuffer::from(nulls)),
- ))
- }
-}
-
fn criterion_benchmark(c: &mut Criterion) {
let udf = ArrayMax::new();
let config_options = Arc::new(ConfigOptions::default());
@@ -91,7 +37,14 @@ fn criterion_benchmark(c: &mut Criterion) {
for list_size in [10, 100, 1000] {
for (label, null_density) in [("nulls", ELEMENT_NULL_DENSITY), ("no_nulls", 0.0)]
{
- let list_array = create_int64_list_array(NUM_ROWS, list_size, null_density);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ LIST_NULL_DENSITY as f32,
+ null_density as f32,
+ list_size,
+ SEED,
+ ));
let args = vec![ColumnarValue::Array(Arc::clone(&list_array))];
let arg_fields =
vec![Field::new("arg_0", list_array.data_type().clone(), true).into()];
diff --git a/datafusion/functions-nested/benches/array_remove.rs b/datafusion/functions-nested/benches/array_remove.rs
index 6fd153548a..3384bd0e46 100644
--- a/datafusion/functions-nested/benches/array_remove.rs
+++ b/datafusion/functions-nested/benches/array_remove.rs
@@ -16,11 +16,12 @@
// under the License.
use arrow::array::{
- Array, ArrayRef, BinaryArray, BooleanArray, Decimal128Array, FixedSizeBinaryArray,
- Float64Array, Int64Array, ListArray, StringArray,
+ Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeBinaryArray, ListArray,
+ StringArray,
};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Decimal128Type, Field, Float64Type, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{
criterion_group, criterion_main, {BenchmarkId, Criterion},
};
@@ -55,7 +56,15 @@ fn bench_array_remove_int64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_int64");
for &array_size in ARRAY_SIZES {
- let list_array = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
+
let element_to_remove = ScalarValue::Int64(Some(1));
let args = create_args(list_array.clone(), element_to_remove.clone());
@@ -96,7 +105,14 @@ fn bench_array_remove_f64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_f64");
for &array_size in ARRAY_SIZES {
- let list_array = create_f64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
let element_to_remove = ScalarValue::Float64(Some(1.0));
let args = create_args(list_array.clone(), element_to_remove.clone());
@@ -260,8 +276,17 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_remove_decimal64");
for &array_size in ARRAY_SIZES {
- let list_array = create_decimal64_list_array(NUM_ROWS, array_size, NULL_DENSITY);
- let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 10, 2);
+ let list_array: ArrayRef = Arc::new(create_primitive_list_array_with_seed::<
+ i32,
+ Decimal128Type,
+ >(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ));
+ let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 38, 10);
let args = create_args(list_array.clone(), element_to_remove.clone());
group.bench_with_input(
@@ -276,7 +301,7 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {
arg_fields: vec![
Field::new("arr", list_array.data_type().clone(), false)
.into(),
- Field::new("el", DataType::Decimal128(10, 2), false)
+ Field::new("el", DataType::Decimal128(38, 10), false)
.into(),
],
number_rows: NUM_ROWS,
@@ -348,66 +373,6 @@ fn create_args(list_array: ArrayRef, element: ScalarValue) -> Vec<ColumnarValue>
]
}
-fn create_int64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size as i64))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
-fn create_f64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size as i64) as f64)
- }
- })
- .collect::<Float64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Float64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
fn create_string_list_array(
num_rows: usize,
array_size: usize,
@@ -500,38 +465,6 @@ fn create_boolean_list_array(
)
}
-fn create_decimal64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..array_size) as i128 * 100)
- }
- })
- .collect::<Decimal128Array>()
- .with_precision_and_scale(10, 2)
- .unwrap();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Decimal128(10, 2), true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
fn create_fixed_size_binary_list_array(
num_rows: usize,
array_size: usize,
diff --git a/datafusion/functions-nested/benches/array_repeat.rs b/datafusion/functions-nested/benches/array_repeat.rs
index 0ce8db00ce..42372322e2 100644
--- a/datafusion/functions-nested/benches/array_repeat.rs
+++ b/datafusion/functions-nested/benches/array_repeat.rs
@@ -15,9 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int64Array, ListArray};
+use arrow::array::{ArrayRef, ListArray};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, Int64Type};
+use arrow::util::bench_util::{
+ create_boolean_array, create_f64_array, create_primitive_array,
+ create_primitive_list_array_with_seed, create_string_array_with_max_len,
+};
use criterion::{
criterion_group, criterion_main, {BenchmarkId, Criterion},
};
@@ -32,7 +36,8 @@ use std::hint::black_box;
use std::sync::Arc;
const NUM_ROWS: &[usize] = &[100, 1000, 10000];
-const REPEAT_COUNTS: &[u64] = &[5, 50];
+// Must be of type i64 because ArrayRepeat's second argument is Int64
+const REPEAT_COUNTS: &[i64] = &[5, 50];
const SEED: u64 = 42;
const NULL_DENSITY: f64 = 0.1;
@@ -52,7 +57,10 @@ fn bench_array_repeat_int64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_int64");
for &num_rows in NUM_ROWS {
- let element_array = create_int64_array(num_rows, NULL_DENSITY);
+ let element_array: ArrayRef = Arc::new(create_primitive_array::<Int64Type>(
+ num_rows,
+ NULL_DENSITY as f32,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -71,7 +79,7 @@ fn bench_array_repeat_int64(c: &mut Criterion) {
args: args.clone(),
arg_fields: vec![
Field::new("element", DataType::Int64, false).into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -100,7 +108,11 @@ fn bench_array_repeat_string(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_string");
for &num_rows in NUM_ROWS {
- let element_array = create_string_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_string_array_with_max_len::<i64>(
+ num_rows,
+ NULL_DENSITY as f32,
+ 100,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -119,7 +131,7 @@ fn bench_array_repeat_string(c: &mut Criterion) {
args: args.clone(),
arg_fields: vec![
Field::new("element", DataType::Utf8, false).into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -148,7 +160,14 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_nested_int64");
for &num_rows in NUM_ROWS {
- let list_array = create_int64_list_array(num_rows, 5, NULL_DENSITY);
+ let list_array: ArrayRef =
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ num_rows,
+ NULL_DENSITY as f32,
+ NULL_DENSITY as f32,
+ 5,
+ SEED,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -172,7 +191,7 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {
false,
)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -201,7 +220,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_float64");
for &num_rows in NUM_ROWS {
- let element_array = create_float64_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_f64_array(num_rows, NULL_DENSITY as f32));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -221,7 +240,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {
arg_fields: vec![
Field::new("element", DataType::Float64, false)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -250,7 +269,11 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {
let mut group = c.benchmark_group("array_repeat_boolean");
for &num_rows in NUM_ROWS {
- let element_array = create_boolean_array(num_rows, NULL_DENSITY);
+ let element_array = Arc::new(create_boolean_array(
+ num_rows,
+ NULL_DENSITY as f32,
+ f32::MAX,
+ ));
for &repeat_count in REPEAT_COUNTS {
let args = vec![
@@ -270,7 +293,7 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {
arg_fields: vec![
Field::new("element", DataType::Boolean, false)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -323,7 +346,7 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {
false,
)
.into(),
- Field::new("count", DataType::UInt64, false).into(),
+ Field::new("count", DataType::Int64, false).into(),
],
number_rows: num_rows,
return_field: Field::new(
@@ -348,98 +371,6 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {
group.finish();
}
-fn create_int64_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
-
- Arc::new(values)
-}
-
-fn create_string_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- use arrow::array::StringArray;
-
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(format!("value_{}", rng.random_range(0..100)))
- }
- })
- .collect::<StringArray>();
-
- Arc::new(values)
-}
-
-fn create_int64_list_array(
- num_rows: usize,
- array_size: usize,
- null_density: f64,
-) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows * array_size)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=num_rows)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
-}
-
-fn create_float64_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random_range(0.0..1000.0))
- }
- })
- .collect::<Float64Array>();
-
- Arc::new(values)
-}
-
-fn create_boolean_array(num_rows: usize, null_density: f64) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..num_rows)
- .map(|_| {
- if rng.random::<f64>() < null_density {
- None
- } else {
- Some(rng.random())
- }
- })
- .collect::<BooleanArray>();
-
- Arc::new(values)
-}
-
fn create_string_list_array(
num_rows: usize,
array_size: usize,
diff --git a/datafusion/functions-nested/benches/array_to_string.rs b/datafusion/functions-nested/benches/array_to_string.rs
index 286ed4eeb0..4b63d70548 100644
--- a/datafusion/functions-nested/benches/array_to_string.rs
+++ b/datafusion/functions-nested/benches/array_to_string.rs
@@ -15,9 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{ArrayRef, Float64Array, Int64Array, ListArray, StringArray};
+use arrow::array::{Array, ArrayRef, ListArray, StringArray};
use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, Field};
+use arrow::datatypes::{DataType, Field, Float64Type, Int64Type};
+use arrow::util::bench_util::create_primitive_list_array_with_seed;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -89,29 +90,13 @@ fn bench_array_to_string(
}
fn create_int64_list_array(array_size: usize) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..NUM_ROWS * array_size)
- .map(|_| {
- if rng.random::<f64>() < NULL_DENSITY {
- None
- } else {
- Some(rng.random_range(0..1000))
- }
- })
- .collect::<Int64Array>();
- let offsets = (0..=NUM_ROWS)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Int64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ))
}
fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {
@@ -133,29 +118,13 @@ fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {
}
fn create_float64_list_array(array_size: usize) -> ArrayRef {
- let mut rng = StdRng::seed_from_u64(SEED);
- let values = (0..NUM_ROWS * array_size)
- .map(|_| {
- if rng.random::<f64>() < NULL_DENSITY {
- None
- } else {
- Some(rng.random_range(-1000.0..1000.0))
- }
- })
- .collect::<Float64Array>();
- let offsets = (0..=NUM_ROWS)
- .map(|i| (i * array_size) as i32)
- .collect::<Vec<i32>>();
-
- Arc::new(
- ListArray::try_new(
- Arc::new(Field::new("item", DataType::Float64, true)),
- OffsetBuffer::new(offsets.into()),
- Arc::new(values),
- None,
- )
- .unwrap(),
- )
+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(
+ NUM_ROWS,
+ 0.0,
+ NULL_DENSITY as f32,
+ array_size,
+ SEED,
+ ))
}
fn create_string_list_array(array_size: usize) -> ArrayRef {