Apache DataFusion SQL Query Engine
chore: use bench array helpers from Arrow bench_util (#21544)
theirix43d32a8c14b1
diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rsindex 793c2aac96..b0d8148c3e 100644--- a/datafusion/functions-aggregate/benches/array_agg.rs+++ b/datafusion/functions-aggregate/benches/array_agg.rs@@ -20,7 +20,6 @@ use std::sync::Arc;use arrow::array::{Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder,- PrimitiveArray,};use arrow::datatypes::{Field, Int64Type};use criterion::{Criterion, criterion_group, criterion_main};@@ -28,6 +27,7 @@ use datafusion_expr::Accumulator;use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator;use arrow::buffer::OffsetBuffer;+use arrow::util::bench_util::create_primitive_array;use rand::Rng;use rand::SeedableRng;use rand::distr::{Distribution, StandardUniform};@@ -54,24 +54,6 @@ fn merge_batch_bench(c: &mut Criterion, name: &str, values: ArrayRef) {});}-pub fn create_primitive_array<T>(size: usize, null_density: f32) -> PrimitiveArray<T>-where- T: ArrowPrimitiveType,- StandardUniform: Distribution<T::Native>,-{- let mut rng = seedable_rng();-- (0..size)- .map(|_| {- if rng.random::<f32>() < null_density {- None- } else {- Some(rng.random())- }- })- .collect()-}-/// Create List array with the given item data type, null density, null locations and zero length lists density/// Creates a random (but fixed-seeded) array of a given size and null densitypub fn create_list_array<T>(diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.tomlindex 13b0029615..d2204a7be7 100644--- a/datafusion/functions-nested/Cargo.toml+++ b/datafusion/functions-nested/Cargo.toml@@ -64,6 +64,7 @@ log = { workspace = true }memchr = { workspace = true }[dev-dependencies]+arrow = { workspace = true, features = ["test_utils"] }criterion = { workspace = true, features = ["async_tokio"] }rand = { workspace = true }diff --git a/datafusion/functions-nested/benches/array_min_max.rs b/datafusion/functions-nested/benches/array_min_max.rsindex 391a84d148..45838da79f 100644--- a/datafusion/functions-nested/benches/array_min_max.rs+++ b/datafusion/functions-nested/benches/array_min_max.rs@@ -17,73 +17,19 @@use std::sync::Arc;-use arrow::array::{ArrayRef, Int64Array, ListArray};-use arrow::buffer::{NullBuffer, OffsetBuffer};-use arrow::datatypes::{DataType, Field};+use arrow::array::{Array, ArrayRef};+use arrow::datatypes::{DataType, Field, Int64Type};+use arrow::util::bench_util::create_primitive_list_array_with_seed;use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_common::config::ConfigOptions;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};use datafusion_functions_nested::min_max::ArrayMax;-use rand::rngs::StdRng;-use rand::{Rng, SeedableRng};const NUM_ROWS: usize = 8192;const SEED: u64 = 42;const LIST_NULL_DENSITY: f64 = 0.1;const ELEMENT_NULL_DENSITY: f64 = 0.1;-fn create_int64_list_array(- num_rows: usize,- list_size: usize,- element_null_density: f64,-) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let total_values = num_rows * list_size;-- if element_null_density > 0.0 {- let values: Vec<Option<i64>> = (0..total_values)- .map(|_| {- if rng.random::<f64>() < element_null_density {- None- } else {- Some(rng.random::<i64>() % 10_000)- }- })- .collect();- let values_array = Arc::new(Int64Array::from(values));-- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();- let nulls: Vec<bool> = (0..num_rows)- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)- .collect();-- Arc::new(ListArray::new(- Arc::new(Field::new("item", DataType::Int64, true)),- OffsetBuffer::new(offsets.into()),- values_array,- Some(NullBuffer::from(nulls)),- ))- } else {- // No element nulls — values array has no null buffer- let values: Vec<i64> = (0..total_values)- .map(|_| rng.random::<i64>() % 10_000)- .collect();- let values_array = Arc::new(Int64Array::from(values));-- let offsets: Vec<i32> = (0..=num_rows).map(|i| (i * list_size) as i32).collect();- let nulls: Vec<bool> = (0..num_rows)- .map(|_| rng.random::<f64>() >= LIST_NULL_DENSITY)- .collect();-- Arc::new(ListArray::new(- Arc::new(Field::new("item", DataType::Int64, false)),- OffsetBuffer::new(offsets.into()),- values_array,- Some(NullBuffer::from(nulls)),- ))- }-}-fn criterion_benchmark(c: &mut Criterion) {let udf = ArrayMax::new();let config_options = Arc::new(ConfigOptions::default());@@ -91,7 +37,14 @@ fn criterion_benchmark(c: &mut Criterion) {for list_size in [10, 100, 1000] {for (label, null_density) in [("nulls", ELEMENT_NULL_DENSITY), ("no_nulls", 0.0)]{- let list_array = create_int64_list_array(NUM_ROWS, list_size, null_density);+ let list_array: ArrayRef =+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(+ NUM_ROWS,+ LIST_NULL_DENSITY as f32,+ null_density as f32,+ list_size,+ SEED,+ ));let args = vec![ColumnarValue::Array(Arc::clone(&list_array))];let arg_fields =vec![Field::new("arg_0", list_array.data_type().clone(), true).into()];diff --git a/datafusion/functions-nested/benches/array_remove.rs b/datafusion/functions-nested/benches/array_remove.rsindex 6fd153548a..3384bd0e46 100644--- a/datafusion/functions-nested/benches/array_remove.rs+++ b/datafusion/functions-nested/benches/array_remove.rs@@ -16,11 +16,12 @@// under the License.use arrow::array::{- Array, ArrayRef, BinaryArray, BooleanArray, Decimal128Array, FixedSizeBinaryArray,- Float64Array, Int64Array, ListArray, StringArray,+ Array, ArrayRef, BinaryArray, BooleanArray, FixedSizeBinaryArray, ListArray,+ StringArray,};use arrow::buffer::OffsetBuffer;-use arrow::datatypes::{DataType, Field};+use arrow::datatypes::{DataType, Decimal128Type, Field, Float64Type, Int64Type};+use arrow::util::bench_util::create_primitive_list_array_with_seed;use criterion::{criterion_group, criterion_main, {BenchmarkId, Criterion},};@@ -55,7 +56,15 @@ fn bench_array_remove_int64(c: &mut Criterion) {let mut group = c.benchmark_group("array_remove_int64");for &array_size in ARRAY_SIZES {- let list_array = create_int64_list_array(NUM_ROWS, array_size, NULL_DENSITY);+ let list_array: ArrayRef =+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(+ NUM_ROWS,+ 0.0,+ NULL_DENSITY as f32,+ array_size,+ SEED,+ ));+let element_to_remove = ScalarValue::Int64(Some(1));let args = create_args(list_array.clone(), element_to_remove.clone());@@ -96,7 +105,14 @@ fn bench_array_remove_f64(c: &mut Criterion) {let mut group = c.benchmark_group("array_remove_f64");for &array_size in ARRAY_SIZES {- let list_array = create_f64_list_array(NUM_ROWS, array_size, NULL_DENSITY);+ let list_array: ArrayRef =+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(+ NUM_ROWS,+ 0.0,+ NULL_DENSITY as f32,+ array_size,+ SEED,+ ));let element_to_remove = ScalarValue::Float64(Some(1.0));let args = create_args(list_array.clone(), element_to_remove.clone());@@ -260,8 +276,17 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {let mut group = c.benchmark_group("array_remove_decimal64");for &array_size in ARRAY_SIZES {- let list_array = create_decimal64_list_array(NUM_ROWS, array_size, NULL_DENSITY);- let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 10, 2);+ let list_array: ArrayRef = Arc::new(create_primitive_list_array_with_seed::<+ i32,+ Decimal128Type,+ >(+ NUM_ROWS,+ 0.0,+ NULL_DENSITY as f32,+ array_size,+ SEED,+ ));+ let element_to_remove = ScalarValue::Decimal128(Some(100_i128), 38, 10);let args = create_args(list_array.clone(), element_to_remove.clone());group.bench_with_input(@@ -276,7 +301,7 @@ fn bench_array_remove_decimal64(c: &mut Criterion) {arg_fields: vec![Field::new("arr", list_array.data_type().clone(), false).into(),- Field::new("el", DataType::Decimal128(10, 2), false)+ Field::new("el", DataType::Decimal128(38, 10), false).into(),],number_rows: NUM_ROWS,@@ -348,66 +373,6 @@ fn create_args(list_array: ArrayRef, element: ScalarValue) -> Vec<ColumnarValue>]}-fn create_int64_list_array(- num_rows: usize,- array_size: usize,- null_density: f64,-) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows * array_size)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0..array_size as i64))- }- })- .collect::<Int64Array>();- let offsets = (0..=num_rows)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Int64, true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )-}--fn create_f64_list_array(- num_rows: usize,- array_size: usize,- null_density: f64,-) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows * array_size)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0..array_size as i64) as f64)- }- })- .collect::<Float64Array>();- let offsets = (0..=num_rows)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Float64, true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )-}-fn create_string_list_array(num_rows: usize,array_size: usize,@@ -500,38 +465,6 @@ fn create_boolean_list_array()}-fn create_decimal64_list_array(- num_rows: usize,- array_size: usize,- null_density: f64,-) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows * array_size)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0..array_size) as i128 * 100)- }- })- .collect::<Decimal128Array>()- .with_precision_and_scale(10, 2)- .unwrap();- let offsets = (0..=num_rows)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Decimal128(10, 2), true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )-}-fn create_fixed_size_binary_list_array(num_rows: usize,array_size: usize,diff --git a/datafusion/functions-nested/benches/array_repeat.rs b/datafusion/functions-nested/benches/array_repeat.rsindex 0ce8db00ce..42372322e2 100644--- a/datafusion/functions-nested/benches/array_repeat.rs+++ b/datafusion/functions-nested/benches/array_repeat.rs@@ -15,9 +15,13 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{ArrayRef, BooleanArray, Float64Array, Int64Array, ListArray};+use arrow::array::{ArrayRef, ListArray};use arrow::buffer::OffsetBuffer;-use arrow::datatypes::{DataType, Field};+use arrow::datatypes::{DataType, Field, Int64Type};+use arrow::util::bench_util::{+ create_boolean_array, create_f64_array, create_primitive_array,+ create_primitive_list_array_with_seed, create_string_array_with_max_len,+};use criterion::{criterion_group, criterion_main, {BenchmarkId, Criterion},};@@ -32,7 +36,8 @@ use std::hint::black_box;use std::sync::Arc;const NUM_ROWS: &[usize] = &[100, 1000, 10000];-const REPEAT_COUNTS: &[u64] = &[5, 50];+// Must be of type i64 because ArrayRepeat's second argument is Int64+const REPEAT_COUNTS: &[i64] = &[5, 50];const SEED: u64 = 42;const NULL_DENSITY: f64 = 0.1;@@ -52,7 +57,10 @@ fn bench_array_repeat_int64(c: &mut Criterion) {let mut group = c.benchmark_group("array_repeat_int64");for &num_rows in NUM_ROWS {- let element_array = create_int64_array(num_rows, NULL_DENSITY);+ let element_array: ArrayRef = Arc::new(create_primitive_array::<Int64Type>(+ num_rows,+ NULL_DENSITY as f32,+ ));for &repeat_count in REPEAT_COUNTS {let args = vec![@@ -71,7 +79,7 @@ fn bench_array_repeat_int64(c: &mut Criterion) {args: args.clone(),arg_fields: vec![Field::new("element", DataType::Int64, false).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -100,7 +108,11 @@ fn bench_array_repeat_string(c: &mut Criterion) {let mut group = c.benchmark_group("array_repeat_string");for &num_rows in NUM_ROWS {- let element_array = create_string_array(num_rows, NULL_DENSITY);+ let element_array = Arc::new(create_string_array_with_max_len::<i64>(+ num_rows,+ NULL_DENSITY as f32,+ 100,+ ));for &repeat_count in REPEAT_COUNTS {let args = vec![@@ -119,7 +131,7 @@ fn bench_array_repeat_string(c: &mut Criterion) {args: args.clone(),arg_fields: vec![Field::new("element", DataType::Utf8, false).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -148,7 +160,14 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {let mut group = c.benchmark_group("array_repeat_nested_int64");for &num_rows in NUM_ROWS {- let list_array = create_int64_list_array(num_rows, 5, NULL_DENSITY);+ let list_array: ArrayRef =+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(+ num_rows,+ NULL_DENSITY as f32,+ NULL_DENSITY as f32,+ 5,+ SEED,+ ));for &repeat_count in REPEAT_COUNTS {let args = vec![@@ -172,7 +191,7 @@ fn bench_array_repeat_nested_int64_list(c: &mut Criterion) {false,).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -201,7 +220,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {let mut group = c.benchmark_group("array_repeat_float64");for &num_rows in NUM_ROWS {- let element_array = create_float64_array(num_rows, NULL_DENSITY);+ let element_array = Arc::new(create_f64_array(num_rows, NULL_DENSITY as f32));for &repeat_count in REPEAT_COUNTS {let args = vec![@@ -221,7 +240,7 @@ fn bench_array_repeat_float64(c: &mut Criterion) {arg_fields: vec![Field::new("element", DataType::Float64, false).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -250,7 +269,11 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {let mut group = c.benchmark_group("array_repeat_boolean");for &num_rows in NUM_ROWS {- let element_array = create_boolean_array(num_rows, NULL_DENSITY);+ let element_array = Arc::new(create_boolean_array(+ num_rows,+ NULL_DENSITY as f32,+ f32::MAX,+ ));for &repeat_count in REPEAT_COUNTS {let args = vec![@@ -270,7 +293,7 @@ fn bench_array_repeat_boolean(c: &mut Criterion) {arg_fields: vec![Field::new("element", DataType::Boolean, false).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -323,7 +346,7 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {false,).into(),- Field::new("count", DataType::UInt64, false).into(),+ Field::new("count", DataType::Int64, false).into(),],number_rows: num_rows,return_field: Field::new(@@ -348,98 +371,6 @@ fn bench_array_repeat_nested_string_list(c: &mut Criterion) {group.finish();}-fn create_int64_array(num_rows: usize, null_density: f64) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0..1000))- }- })- .collect::<Int64Array>();-- Arc::new(values)-}--fn create_string_array(num_rows: usize, null_density: f64) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- use arrow::array::StringArray;-- let values = (0..num_rows)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(format!("value_{}", rng.random_range(0..100)))- }- })- .collect::<StringArray>();-- Arc::new(values)-}--fn create_int64_list_array(- num_rows: usize,- array_size: usize,- null_density: f64,-) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows * array_size)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0..1000))- }- })- .collect::<Int64Array>();- let offsets = (0..=num_rows)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Int64, true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )-}--fn create_float64_array(num_rows: usize, null_density: f64) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random_range(0.0..1000.0))- }- })- .collect::<Float64Array>();-- Arc::new(values)-}--fn create_boolean_array(num_rows: usize, null_density: f64) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..num_rows)- .map(|_| {- if rng.random::<f64>() < null_density {- None- } else {- Some(rng.random())- }- })- .collect::<BooleanArray>();-- Arc::new(values)-}-fn create_string_list_array(num_rows: usize,array_size: usize,diff --git a/datafusion/functions-nested/benches/array_to_string.rs b/datafusion/functions-nested/benches/array_to_string.rsindex 286ed4eeb0..4b63d70548 100644--- a/datafusion/functions-nested/benches/array_to_string.rs+++ b/datafusion/functions-nested/benches/array_to_string.rs@@ -15,9 +15,10 @@// specific language governing permissions and limitations// under the License.-use arrow::array::{ArrayRef, Float64Array, Int64Array, ListArray, StringArray};+use arrow::array::{Array, ArrayRef, ListArray, StringArray};use arrow::buffer::OffsetBuffer;-use arrow::datatypes::{DataType, Field};+use arrow::datatypes::{DataType, Field, Float64Type, Int64Type};+use arrow::util::bench_util::create_primitive_list_array_with_seed;use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;@@ -89,29 +90,13 @@ fn bench_array_to_string(}fn create_int64_list_array(array_size: usize) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..NUM_ROWS * array_size)- .map(|_| {- if rng.random::<f64>() < NULL_DENSITY {- None- } else {- Some(rng.random_range(0..1000))- }- })- .collect::<Int64Array>();- let offsets = (0..=NUM_ROWS)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Int64, true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )+ Arc::new(create_primitive_list_array_with_seed::<i32, Int64Type>(+ NUM_ROWS,+ 0.0,+ NULL_DENSITY as f32,+ array_size,+ SEED,+ ))}fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {@@ -133,29 +118,13 @@ fn create_nested_int64_list_array(array_size: usize) -> ArrayRef {}fn create_float64_list_array(array_size: usize) -> ArrayRef {- let mut rng = StdRng::seed_from_u64(SEED);- let values = (0..NUM_ROWS * array_size)- .map(|_| {- if rng.random::<f64>() < NULL_DENSITY {- None- } else {- Some(rng.random_range(-1000.0..1000.0))- }- })- .collect::<Float64Array>();- let offsets = (0..=NUM_ROWS)- .map(|i| (i * array_size) as i32)- .collect::<Vec<i32>>();-- Arc::new(- ListArray::try_new(- Arc::new(Field::new("item", DataType::Float64, true)),- OffsetBuffer::new(offsets.into()),- Arc::new(values),- None,- )- .unwrap(),- )+ Arc::new(create_primitive_list_array_with_seed::<i32, Float64Type>(+ NUM_ROWS,+ 0.0,+ NULL_DENSITY as f32,+ array_size,+ SEED,+ ))}fn create_string_list_array(array_size: usize) -> ArrayRef {