Skip to content

Apache DataFusion SQL Query Engine

bench: use seedable rng for reproducibility (#23653)

theirix2eaba476e767
diff --git a/benchmarks/src/cancellation.rs b/benchmarks/src/cancellation.rs
index d3da1b0e83..1048fa0989 100644
--- a/benchmarks/src/cancellation.rs
+++ b/benchmarks/src/cancellation.rs
@@ -39,9 +39,8 @@ use futures::TryStreamExt;
use object_store::ObjectStore;
use parquet::arrow::AsyncArrowWriter;
use parquet::arrow::async_writer::ParquetObjectWriter;
-use rand::Rng;
use rand::distr::Alphanumeric;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
use tokio::runtime::Runtime;
use tokio_util::sync::CancellationToken;
@@ -215,7 +214,8 @@ async fn find_or_generate_files(
if files_on_disk.is_empty() {
println!("No data files found, generating (this will take a bit)");
- generate_data(data_dir.as_ref(), num_files, num_rows_per_file).await?;
+ let mut rng = StdRng::seed_from_u64(0);
+ generate_data(&mut rng, data_dir.as_ref(), num_files, num_rows_per_file).await?;
println!("Done generating files");
let files_on_disk = find_files_on_disk(data_dir)?;
@@ -269,6 +269,7 @@ async fn load_data(
}
async fn generate_data(
+ rng: &mut StdRng,
data_dir: impl AsRef<Path>,
num_files: usize,
num_rows_per_file: usize,
@@ -295,7 +296,7 @@ async fn generate_data(
for file_num in 1..=num_files {
println!("Generating file {file_num} of {num_files}");
let data = columns.iter().map(|(column_name, column_type)| {
- let column = random_data(column_type, num_rows_per_file);
+ let column = random_data(rng, column_type, num_rows_per_file);
(column_name, column)
});
let to_write = RecordBatch::try_from_iter(data).unwrap();
@@ -311,13 +312,12 @@ async fn generate_data(
Ok(())
}
-fn random_data(column_type: &DataType, rows: usize) -> Arc<dyn Array> {
- let mut rng = rand::rng();
- let values = (0..rows).map(|_| random_value(&mut rng, column_type));
+fn random_data(rng: &mut StdRng, column_type: &DataType, rows: usize) -> Arc<dyn Array> {
+ let values = (0..rows).map(|_| random_value(rng, column_type));
ScalarValue::iter_to_array(values).unwrap()
}
-fn random_value(rng: &mut ThreadRng, column_type: &DataType) -> ScalarValue {
+fn random_value(rng: &mut StdRng, column_type: &DataType) -> ScalarValue {
match column_type {
DataType::Float64 => ScalarValue::Float64(Some(rng.random())),
DataType::Boolean => ScalarValue::Boolean(Some(rng.random())),
diff --git a/datafusion/core/benches/map_query_sql.rs b/datafusion/core/benches/map_query_sql.rs
index 67904197bc..6e7d584c6f 100644
--- a/datafusion/core/benches/map_query_sql.rs
+++ b/datafusion/core/benches/map_query_sql.rs
@@ -22,8 +22,7 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
use criterion::{Criterion, criterion_group, criterion_main};
use parking_lot::Mutex;
-use rand::Rng;
-use rand::prelude::ThreadRng;
+use rand::prelude::*;
use tokio::runtime::Runtime;
use datafusion::prelude::SessionContext;
@@ -33,7 +32,7 @@ use datafusion_functions_nested::map::map;
mod data_utils;
-fn build_keys(rng: &mut ThreadRng) -> Vec<String> {
+fn build_keys(rng: &mut StdRng) -> Vec<String> {
let mut keys = HashSet::with_capacity(1000);
while keys.len() < 1000 {
let key = rng.random_range(0..9999).to_string();
@@ -42,7 +41,7 @@ fn build_keys(rng: &mut ThreadRng) -> Vec<String> {
keys.into_iter().collect()
}
-fn build_values(rng: &mut ThreadRng) -> Vec<i32> {
+fn build_values(rng: &mut StdRng) -> Vec<i32> {
let mut values = vec![];
for _ in 0..1000 {
values.push(rng.random_range(0..9999));
@@ -67,7 +66,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let rt = Runtime::new().unwrap();
let df = rt.block_on(ctx.lock().table("t")).unwrap();
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let keys = build_keys(&mut rng);
let values = build_values(&mut rng);
let mut key_buffer = Vec::new();
diff --git a/datafusion/core/benches/parquet_query_sql.rs b/datafusion/core/benches/parquet_query_sql.rs
index f099137973..2e7794bfd1 100644
--- a/datafusion/core/benches/parquet_query_sql.rs
+++ b/datafusion/core/benches/parquet_query_sql.rs
@@ -32,7 +32,6 @@ use parquet::file::properties::{WriterProperties, WriterVersion};
use rand::distr::Alphanumeric;
use rand::distr::uniform::SampleUniform;
use rand::prelude::*;
-use rand::rng;
use std::fs::File;
use std::io::Read;
use std::ops::Range;
@@ -69,36 +68,36 @@ fn schema() -> SchemaRef {
]))
}
-fn generate_batch() -> RecordBatch {
+fn generate_batch(rng: &mut StdRng) -> RecordBatch {
let schema = schema();
let len = WRITE_RECORD_BATCH_SIZE;
RecordBatch::try_new(
schema,
vec![
- generate_string_dictionary("prefix", 10, len, 1.0),
- generate_string_dictionary("prefix", 10, len, 0.5),
- generate_string_dictionary("prefix", 100, len, 1.0),
- generate_string_dictionary("prefix", 100, len, 0.5),
- generate_string_dictionary("prefix", 1000, len, 1.0),
- generate_string_dictionary("prefix", 1000, len, 0.5),
- generate_strings(0..100, len, 1.0),
- generate_strings(0..100, len, 0.5),
- generate_primitive::<Int64Type>(len, 1.0, -2000..2000),
- generate_primitive::<Int64Type>(len, 0.5, -2000..2000),
- generate_primitive::<Float64Type>(len, 1.0, -1000.0..1000.0),
- generate_primitive::<Float64Type>(len, 0.5, -1000.0..1000.0),
+ generate_string_dictionary(rng, "prefix", 10, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 10, len, 0.5),
+ generate_string_dictionary(rng, "prefix", 100, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 100, len, 0.5),
+ generate_string_dictionary(rng, "prefix", 1000, len, 1.0),
+ generate_string_dictionary(rng, "prefix", 1000, len, 0.5),
+ generate_strings(rng, 0..100, len, 1.0),
+ generate_strings(rng, 0..100, len, 0.5),
+ generate_primitive::<Int64Type>(rng, len, 1.0, -2000..2000),
+ generate_primitive::<Int64Type>(rng, len, 0.5, -2000..2000),
+ generate_primitive::<Float64Type>(rng, len, 1.0, -1000.0..1000.0),
+ generate_primitive::<Float64Type>(rng, len, 0.5, -1000.0..1000.0),
],
)
.unwrap()
}
fn generate_string_dictionary(
+ rng: &mut StdRng,
prefix: &str,
cardinality: usize,
len: usize,
valid_percent: f64,
) -> ArrayRef {
- let mut rng = rng();
let strings: Vec<_> = (0..cardinality).map(|x| format!("{prefix}#{x}")).collect();
Arc::new(DictionaryArray::<Int32Type>::from_iter((0..len).map(
@@ -110,11 +109,11 @@ fn generate_string_dictionary(
}
fn generate_strings(
+ rng: &mut StdRng,
string_length_range: Range<usize>,
len: usize,
valid_percent: f64,
) -> ArrayRef {
- let mut rng = rng();
Arc::new(StringArray::from_iter((0..len).map(|_| {
rng.random_bool(valid_percent).then(|| {
let string_len = rng.random_range(string_length_range.clone());
@@ -126,6 +125,7 @@ fn generate_strings(
}
fn generate_primitive<T>(
+ rng: &mut StdRng,
len: usize,
valid_percent: f64,
range: Range<T::Native>,
@@ -134,7 +134,6 @@ where
T: ArrowPrimitiveType,
T::Native: SampleUniform,
{
- let mut rng = rng();
Arc::new(PrimitiveArray::<T>::from_iter((0..len).map(|_| {
rng.random_bool(valid_percent)
.then(|| rng.random_range(range.clone()))
@@ -160,8 +159,9 @@ fn generate_file() -> NamedTempFile {
let mut writer =
ArrowWriter::try_new(&mut named_file, schema, Some(properties)).unwrap();
+ let mut rng = StdRng::seed_from_u64(0);
for _ in 0..NUM_BATCHES {
- let batch = generate_batch();
+ let batch = generate_batch(&mut rng);
writer.write(&batch).unwrap();
}
diff --git a/datafusion/core/benches/parquet_struct_query.rs b/datafusion/core/benches/parquet_struct_query.rs
index e7e91f0dd0..b7132973c1 100644
--- a/datafusion/core/benches/parquet_struct_query.rs
+++ b/datafusion/core/benches/parquet_struct_query.rs
@@ -27,7 +27,6 @@ use parquet::arrow::ArrowWriter;
use parquet::file::properties::{WriterProperties, WriterVersion};
use rand::distr::Alphanumeric;
use rand::prelude::*;
-use rand::rng;
use std::hint::black_box;
use std::ops::Range;
use std::path::Path;
@@ -59,8 +58,7 @@ fn schema() -> SchemaRef {
]))
}
-fn generate_strings(len: usize) -> ArrayRef {
- let mut rng = rng();
+fn generate_strings(rng: &mut StdRng, len: usize) -> ArrayRef {
Arc::new(StringArray::from_iter((0..len).map(|_| {
let string_len = rng.random_range(STRING_LENGTH_RANGE.clone());
Some(
@@ -71,7 +69,7 @@ fn generate_strings(len: usize) -> ArrayRef {
})))
}
-fn generate_batch(batch_id: usize) -> RecordBatch {
+fn generate_batch(rng: &mut StdRng, batch_id: usize) -> RecordBatch {
let schema = schema();
let len = WRITE_RECORD_BATCH_SIZE;
@@ -84,7 +82,7 @@ fn generate_batch(batch_id: usize) -> RecordBatch {
let struct_id_array = Arc::new(Int32Array::from(id_values));
// Generate random strings for struct value field
- let value_array = generate_strings(len);
+ let value_array = generate_strings(rng, len);
// Construct StructArray
let struct_array = StructArray::from(vec![
@@ -120,8 +118,9 @@ fn generate_file() -> NamedTempFile {
let mut writer =
ArrowWriter::try_new(&mut named_file, schema, Some(properties)).unwrap();
+ let mut rng = StdRng::seed_from_u64(0);
for batch_id in 0..NUM_BATCHES {
- let batch = generate_batch(batch_id);
+ let batch = generate_batch(&mut rng, batch_id);
writer.write(&batch).unwrap();
}
diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs
index 67e7f314d2..9cc4289ca1 100644
--- a/datafusion/functions-nested/benches/map.rs
+++ b/datafusion/functions-nested/benches/map.rs
@@ -28,8 +28,7 @@ use datafusion_expr::planner::ExprPlanner;
use datafusion_expr::{ColumnarValue, Expr, ScalarFunctionArgs};
use datafusion_functions_nested::map::map_udf;
use datafusion_functions_nested::planner::NestedFunctionPlanner;
-use rand::Rng;
-use rand::prelude::ThreadRng;
+use rand::prelude::*;
use std::collections::HashSet;
use std::hash::Hash;
use std::hint::black_box;
@@ -38,10 +37,7 @@ use std::sync::Arc;
const MAP_ROWS: usize = 1000;
const MAP_KEYS_PER_ROW: usize = 1000;
-fn gen_unique_values<T>(
- rng: &mut ThreadRng,
- mut make_value: impl FnMut(i32) -> T,
-) -> Vec<T>
+fn gen_unique_values<T>(rng: &mut StdRng, mut make_value: impl FnMut(i32) -> T) -> Vec<T>
where
T: Eq + Hash,
{
@@ -64,15 +60,15 @@ fn gen_repeat_values<T: Clone>(values: &[T], repeats: usize) -> Vec<T> {
repeated
}
-fn gen_utf8_values(rng: &mut ThreadRng) -> Vec<String> {
+fn gen_utf8_values(rng: &mut StdRng) -> Vec<String> {
gen_unique_values(rng, |value| value.to_string())
}
-fn gen_binary_values(rng: &mut ThreadRng) -> Vec<Vec<u8>> {
+fn gen_binary_values(rng: &mut StdRng) -> Vec<Vec<u8>> {
gen_unique_values(rng, |value| value.to_le_bytes().to_vec())
}
-fn gen_primitive_values(rng: &mut ThreadRng) -> Vec<i32> {
+fn gen_primitive_values(rng: &mut StdRng) -> Vec<i32> {
gen_unique_values(rng, |value| value)
}
@@ -122,7 +118,7 @@ fn bench_map_case(c: &mut Criterion, name: &str, keys: ArrayRef, values: ArrayRe
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("make_map_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let keys = gen_utf8_values(&mut rng);
let values = gen_primitive_values(&mut rng);
let mut buffer = Vec::new();
@@ -143,7 +139,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
});
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = Arc::new(Int32Array::from(gen_repeat_values(
&gen_primitive_values(&mut rng),
MAP_ROWS,
diff --git a/datafusion/functions/benches/concat.rs b/datafusion/functions/benches/concat.rs
index 0fb910800e..6736625be0 100644
--- a/datafusion/functions/benches/concat.rs
+++ b/datafusion/functions/benches/concat.rs
@@ -23,8 +23,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string::concat;
-use rand::Rng;
use rand::distr::Alphanumeric;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
@@ -48,17 +48,20 @@ fn create_array_args_view(size: usize) -> Vec<ColumnarValue> {
]
}
-fn generate_random_string(str_len: usize) -> String {
- rand::rng()
- .sample_iter(&Alphanumeric)
+fn generate_random_string(rng: &mut StdRng, str_len: usize) -> String {
+ rng.sample_iter(&Alphanumeric)
.take(str_len)
.map(char::from)
.collect()
}
-fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
+fn create_scalar_args(
+ rng: &mut StdRng,
+ count: usize,
+ str_len: usize,
+) -> Vec<ColumnarValue> {
std::iter::repeat_with(|| {
- let s = generate_random_string(str_len);
+ let s = generate_random_string(rng, str_len);
ColumnarValue::Scalar(ScalarValue::Utf8(Some(s)))
})
.take(count)
@@ -67,6 +70,7 @@ fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
fn criterion_benchmark(c: &mut Criterion) {
// Benchmark for array concat
+ let mut rng = StdRng::seed_from_u64(0);
for size in [1024, 4096, 8192] {
let args = create_array_args(size, 32);
let arg_fields = args
@@ -138,7 +142,7 @@ fn criterion_benchmark(c: &mut Criterion) {
}
// Benchmark for scalar concat
- let scalar_args = create_scalar_args(10, 100);
+ let scalar_args = create_scalar_args(&mut rng, 10, 100);
let scalar_arg_fields = scalar_args
.iter()
.enumerate()
diff --git a/datafusion/functions/benches/concat_ws.rs b/datafusion/functions/benches/concat_ws.rs
index 97d6d96411..d437f38773 100644
--- a/datafusion/functions/benches/concat_ws.rs
+++ b/datafusion/functions/benches/concat_ws.rs
@@ -23,8 +23,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::string::concat_ws;
-use rand::Rng;
use rand::distr::Alphanumeric;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
@@ -38,9 +38,8 @@ fn create_array_args(size: usize, str_len: usize) -> Vec<ColumnarValue> {
]
}
-fn generate_random_string(str_len: usize) -> String {
- rand::rng()
- .sample_iter(&Alphanumeric)
+fn generate_random_string(rng: &mut StdRng, str_len: usize) -> String {
+ rng.sample_iter(&Alphanumeric)
.take(str_len)
.map(char::from)
.collect()
@@ -53,8 +52,9 @@ fn create_scalar_args(count: usize, str_len: usize) -> Vec<ColumnarValue> {
",".to_string(),
))));
+ let mut rng = StdRng::seed_from_u64(0);
for _ in 0..count {
- let s = generate_random_string(str_len);
+ let s = generate_random_string(&mut rng, str_len);
args.push(ColumnarValue::Scalar(ScalarValue::Utf8(Some(s))));
}
args
diff --git a/datafusion/functions/benches/date_bin.rs b/datafusion/functions/benches/date_bin.rs
index 28dee96987..bae1438fa4 100644
--- a/datafusion/functions/benches/date_bin.rs
+++ b/datafusion/functions/benches/date_bin.rs
@@ -25,10 +25,9 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::date_bin;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn timestamps(rng: &mut ThreadRng) -> TimestampSecondArray {
+fn timestamps(rng: &mut StdRng) -> TimestampSecondArray {
let mut seconds = vec![];
for _ in 0..1000 {
seconds.push(rng.random_range(0..1_000_000));
@@ -39,7 +38,7 @@ fn timestamps(rng: &mut ThreadRng) -> TimestampSecondArray {
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("date_bin_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let timestamps_array = Arc::new(timestamps(&mut rng)) as ArrayRef;
let batch_len = timestamps_array.len();
let interval = ColumnarValue::Scalar(ScalarValue::new_interval_dt(0, 1_000_000));
diff --git a/datafusion/functions/benches/gcd.rs b/datafusion/functions/benches/gcd.rs
index 3c72a46e66..ca49415b0f 100644
--- a/datafusion/functions/benches/gcd.rs
+++ b/datafusion/functions/benches/gcd.rs
@@ -25,12 +25,12 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::math::gcd;
-use rand::Rng;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
fn generate_i64_array(n_rows: usize) -> ArrayRef {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = (0..n_rows)
.map(|_| rng.random_range(0..1000))
.collect::<Vec<_>>();
diff --git a/datafusion/functions/benches/lcm.rs b/datafusion/functions/benches/lcm.rs
index 247c0ec749..5a4e5d2bce 100644
--- a/datafusion/functions/benches/lcm.rs
+++ b/datafusion/functions/benches/lcm.rs
@@ -24,12 +24,12 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::math::lcm;
-use rand::Rng;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
fn generate_i64_array(n_rows: usize) -> ArrayRef {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let values = (0..n_rows)
.map(|_| rng.random_range(0..1000))
.collect::<Vec<_>>();
diff --git a/datafusion/functions/benches/make_date.rs b/datafusion/functions/benches/make_date.rs
index 1c7b61ec60..2e82a871eb 100644
--- a/datafusion/functions/benches/make_date.rs
+++ b/datafusion/functions/benches/make_date.rs
@@ -25,10 +25,9 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::make_date;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn years(rng: &mut ThreadRng) -> Int32Array {
+fn years(rng: &mut StdRng) -> Int32Array {
let mut years = vec![];
for _ in 0..8192 {
years.push(rng.random_range(1900..2050));
@@ -37,7 +36,7 @@ fn years(rng: &mut ThreadRng) -> Int32Array {
Int32Array::from(years)
}
-fn months(rng: &mut ThreadRng) -> Int32Array {
+fn months(rng: &mut StdRng) -> Int32Array {
let mut months = vec![];
for _ in 0..8192 {
months.push(rng.random_range(1..13));
@@ -46,7 +45,7 @@ fn months(rng: &mut ThreadRng) -> Int32Array {
Int32Array::from(months)
}
-fn days(rng: &mut ThreadRng) -> Int32Array {
+fn days(rng: &mut StdRng) -> Int32Array {
let mut days = vec![];
for _ in 0..8192 {
days.push(rng.random_range(1..29));
@@ -56,7 +55,7 @@ fn days(rng: &mut ThreadRng) -> Int32Array {
}
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("make_date_col_col_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let years_array = Arc::new(years(&mut rng)) as ArrayRef;
let batch_len = years_array.len();
let years = ColumnarValue::Array(years_array);
@@ -86,7 +85,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("make_date_scalar_col_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let months_arr = Arc::new(months(&mut rng)) as ArrayRef;
let batch_len = months_arr.len();
@@ -116,7 +115,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("make_date_scalar_scalar_col_8192", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let year = ColumnarValue::Scalar(ScalarValue::Int32(Some(2025)));
let month = ColumnarValue::Scalar(ScalarValue::Int32(Some(11)));
let day_arr = Arc::new(days(&mut rng));
diff --git a/datafusion/functions/benches/pad.rs b/datafusion/functions/benches/pad.rs
index c71d5a7161..78ebf12236 100644
--- a/datafusion/functions/benches/pad.rs
+++ b/datafusion/functions/benches/pad.rs
@@ -28,8 +28,8 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::unicode;
-use rand::Rng;
-use rand::distr::{Distribution, Uniform};
+use rand::distr::Uniform;
+use rand::prelude::*;
use std::hint::black_box;
use std::sync::Arc;
use std::time::Duration;
@@ -51,7 +51,7 @@ fn create_unicode_string_array<O: OffsetSizeTrait>(
size: usize,
null_density: f32,
) -> arrow::array::GenericStringArray<O> {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let mut builder = GenericStringBuilder::<O>::new();
for i in 0..size {
if rng.random::<f32>() < null_density {
@@ -67,7 +67,7 @@ fn create_unicode_string_view_array(
size: usize,
null_density: f32,
) -> arrow::array::StringViewArray {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let mut builder = StringViewBuilder::with_capacity(size);
for i in 0..size {
if rng.random::<f32>() < null_density {
@@ -104,7 +104,7 @@ where
dist: Uniform::new_inclusive::<i64, i64>(0, len as i64),
};
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
(0..size)
.map(|_| {
if rng.random::<f32>() < null_density {
diff --git a/datafusion/functions/benches/regx.rs b/datafusion/functions/benches/regx.rs
index a46b548236..dd263e41f6 100644
--- a/datafusion/functions/benches/regx.rs
+++ b/datafusion/functions/benches/regx.rs
@@ -32,11 +32,9 @@ use datafusion_functions::regex::regexpinstr::regexp_instr_func;
use datafusion_functions::regex::regexplike::{RegexpLikeFunc, regexp_like};
use datafusion_functions::regex::regexpmatch::regexp_match;
use datafusion_functions::regex::regexpreplace::regexp_replace;
-use rand::Rng;
use rand::distr::Alphanumeric;
-use rand::prelude::IndexedRandom;
-use rand::rngs::ThreadRng;
-fn data(rng: &mut ThreadRng) -> StringArray {
+use rand::prelude::*;
+fn data(rng: &mut StdRng) -> StringArray {
let mut data: Vec<String> = vec![];
for _ in 0..1000 {
data.push(
@@ -50,7 +48,7 @@ fn data(rng: &mut ThreadRng) -> StringArray {
StringArray::from(data)
}
-fn regex(rng: &mut ThreadRng) -> StringArray {
+fn regex(rng: &mut StdRng) -> StringArray {
let samples = [
".*([A-Z]{1}).*".to_string(),
"^(A).*".to_string(),
@@ -66,7 +64,7 @@ fn regex(rng: &mut ThreadRng) -> StringArray {
StringArray::from(data)
}
-fn start(rng: &mut ThreadRng) -> Int64Array {
+fn start(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -75,7 +73,7 @@ fn start(rng: &mut ThreadRng) -> Int64Array {
Int64Array::from(data)
}
-fn n(rng: &mut ThreadRng) -> Int64Array {
+fn n(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -84,7 +82,7 @@ fn n(rng: &mut ThreadRng) -> Int64Array {
Int64Array::from(data)
}
-fn flags(rng: &mut ThreadRng) -> StringArray {
+fn flags(rng: &mut StdRng) -> StringArray {
let samples = [Some("i".to_string()), Some("im".to_string()), None];
let mut sb = StringBuilder::new();
for _ in 0..1000 {
@@ -99,7 +97,7 @@ fn flags(rng: &mut ThreadRng) -> StringArray {
sb.finish()
}
-fn subexp(rng: &mut ThreadRng) -> Int64Array {
+fn subexp(rng: &mut StdRng) -> Int64Array {
let mut data: Vec<i64> = vec![];
for _ in 0..1000 {
data.push(rng.random_range(1..5));
@@ -112,7 +110,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let regexp_like_func = RegexpLikeFunc::new();
let config_options = Arc::new(ConfigOptions::default());
c.bench_function("regexp_count_1000 string", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -132,7 +130,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_count_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(&regex(&mut rng), &DataType::Utf8View).unwrap();
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -152,7 +150,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_instr_1000 string", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -176,7 +174,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_instr_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(&regex(&mut rng), &DataType::Utf8View).unwrap();
let start = Arc::new(start(&mut rng)) as ArrayRef;
@@ -198,7 +196,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_like_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -212,7 +210,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_like_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(&regex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
@@ -252,7 +250,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_match_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -270,7 +268,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_match_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(&regex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
@@ -288,7 +286,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_replace_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = Arc::new(data(&mut rng)) as ArrayRef;
let regex = Arc::new(regex(&mut rng)) as ArrayRef;
let flags = Arc::new(flags(&mut rng)) as ArrayRef;
@@ -310,7 +308,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("regexp_replace_1000 utf8view", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap();
let regex = cast(&regex(&mut rng), &DataType::Utf8View).unwrap();
let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap();
diff --git a/datafusion/functions/benches/to_char.rs b/datafusion/functions/benches/to_char.rs
index 350a55a371..8a9497bb33 100644
--- a/datafusion/functions/benches/to_char.rs
+++ b/datafusion/functions/benches/to_char.rs
@@ -27,12 +27,10 @@ use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_char;
-use rand::Rng;
-use rand::prelude::IndexedRandom;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
fn pick_date_in_range(
- rng: &mut ThreadRng,
+ rng: &mut StdRng,
start_date: NaiveDate,
end_date: NaiveDate,
) -> NaiveDate {
@@ -41,7 +39,7 @@ fn pick_date_in_range(
start_date + TimeDelta::try_days(random_days).unwrap()
}
-fn generate_date32_array(rng: &mut ThreadRng) -> Date32Array {
+fn generate_date32_array(rng: &mut StdRng) -> Date32Array {
let mut data: Vec<i32> = vec![];
let unix_days_from_ce = NaiveDate::from_ymd_opt(1970, 1, 1)
.unwrap()
@@ -62,7 +60,7 @@ fn generate_date32_array(rng: &mut ThreadRng) -> Date32Array {
Date32Array::from(data)
}
-fn generate_date64_array(rng: &mut ThreadRng) -> Date64Array {
+fn generate_date64_array(rng: &mut StdRng) -> Date64Array {
let start_date = "1970-01-01"
.parse::<NaiveDate>()
.expect("Date should parse");
@@ -96,21 +94,21 @@ const DATETIME_PATTERNS: [&str; 8] = [
"%c",
];
-fn pick_date_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_pattern(rng: &mut StdRng) -> String {
(*DATE_PATTERNS
.choose(rng)
.expect("Empty list of date patterns"))
.to_string()
}
-fn pick_date_time_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_time_pattern(rng: &mut StdRng) -> String {
(*DATETIME_PATTERNS
.choose(rng)
.expect("Empty list of date time patterns"))
.to_string()
}
-fn pick_date_and_date_time_mixed_pattern(rng: &mut ThreadRng) -> String {
+fn pick_date_and_date_time_mixed_pattern(rng: &mut StdRng) -> String {
match rng.random_bool(0.5) {
true => pick_date_pattern(rng),
false => pick_date_time_pattern(rng),
@@ -118,8 +116,8 @@ fn pick_date_and_date_time_mixed_pattern(rng: &mut ThreadRng) -> String {
}
fn generate_pattern_array(
- rng: &mut ThreadRng,
- pick_fn: impl Fn(&mut ThreadRng) -> String,
+ rng: &mut StdRng,
+ pick_fn: impl Fn(&mut StdRng) -> String,
) -> StringArray {
let mut data = Vec::with_capacity(1000);
@@ -130,15 +128,15 @@ fn generate_pattern_array(
StringArray::from(data)
}
-fn generate_date_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_date_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_pattern)
}
-fn generate_datetime_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_datetime_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_time_pattern)
}
-fn generate_mixed_pattern_array(rng: &mut ThreadRng) -> StringArray {
+fn generate_mixed_pattern_array(rng: &mut StdRng) -> StringArray {
generate_pattern_array(rng, pick_date_and_date_time_mixed_pattern)
}
@@ -146,7 +144,7 @@ fn criterion_benchmark(c: &mut Criterion) {
let config_options = Arc::new(ConfigOptions::default());
c.bench_function("to_char_array_date_only_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -173,7 +171,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_array_datetime_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -200,7 +198,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_array_mixed_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -227,7 +225,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_scalar_date_only_pattern_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -253,7 +251,7 @@ fn criterion_benchmark(c: &mut Criterion) {
});
c.bench_function("to_char_scalar_datetime_pattern_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date64_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -285,7 +283,7 @@ fn criterion_benchmark(c: &mut Criterion) {
// Covers full fallback (every row triggers the cast)
c.bench_function("to_char_array_date32_datetime_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
@@ -313,7 +311,7 @@ fn criterion_benchmark(c: &mut Criterion) {
// Covers partial fallback (roughly half the rows trigger it)
c.bench_function("to_char_array_date32_mixed_patterns_1000", |b| {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
let data_arr = generate_date32_array(&mut rng);
let batch_len = data_arr.len();
let data = ColumnarValue::Array(Arc::new(data_arr) as ArrayRef);
diff --git a/datafusion/functions/benches/to_local_time.rs b/datafusion/functions/benches/to_local_time.rs
index 42d1e27198..04440bf0ac 100644
--- a/datafusion/functions/benches/to_local_time.rs
+++ b/datafusion/functions/benches/to_local_time.rs
@@ -24,17 +24,16 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_local_time;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn timestamps(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+fn timestamps(rng: &mut StdRng) -> TimestampNanosecondArray {
let nanos: Vec<i64> = (0..100_000)
.map(|_| rng.random_range(0..1_000_000_000_000_000_000i64))
.collect();
TimestampNanosecondArray::from(nanos).with_timezone("America/New_York")
}
-fn timestamps_with_nulls(rng: &mut ThreadRng) -> TimestampNanosecondArray {
+fn timestamps_with_nulls(rng: &mut StdRng) -> TimestampNanosecondArray {
let values: Vec<Option<i64>> = (0..100_000)
.map(|_| {
if rng.random_range(0..10u32) == 0 {
@@ -73,7 +72,7 @@ fn bench_to_local_time(c: &mut Criterion, name: &str, array: ArrayRef) {
}
fn criterion_benchmark(c: &mut Criterion) {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
bench_to_local_time(
c,
"to_local_time_no_nulls_100k",
diff --git a/datafusion/functions/benches/to_time.rs b/datafusion/functions/benches/to_time.rs
index 6b3aa19241..f4499e2a7d 100644
--- a/datafusion/functions/benches/to_time.rs
+++ b/datafusion/functions/benches/to_time.rs
@@ -24,10 +24,9 @@ use criterion::{Criterion, criterion_group, criterion_main};
use datafusion_common::config::ConfigOptions;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::datetime::to_time;
-use rand::Rng;
-use rand::rngs::ThreadRng;
+use rand::prelude::*;
-fn random_time_string(rng: &mut ThreadRng) -> String {
+fn random_time_string(rng: &mut StdRng) -> String {
format!(
"{:02}:{:02}:{:02}.{:06}",
rng.random_range(0..24u32),
@@ -37,12 +36,12 @@ fn random_time_string(rng: &mut ThreadRng) -> String {
)
}
-fn time_strings(rng: &mut ThreadRng) -> StringArray {
+fn time_strings(rng: &mut StdRng) -> StringArray {
let strings: Vec<String> = (0..100_000).map(|_| random_time_string(rng)).collect();
StringArray::from(strings)
}
-fn time_strings_with_nulls(rng: &mut ThreadRng) -> StringArray {
+fn time_strings_with_nulls(rng: &mut StdRng) -> StringArray {
let values: Vec<Option<String>> = (0..100_000)
.map(|_| {
if rng.random_range(0..10u32) == 0 {
@@ -81,7 +80,7 @@ fn bench_to_time(c: &mut Criterion, name: &str, array: ArrayRef) {
}
fn criterion_benchmark(c: &mut Criterion) {
- let mut rng = rand::rng();
+ let mut rng = StdRng::seed_from_u64(0);
bench_to_time(c, "to_time_no_nulls_100k", Arc::new(time_strings(&mut rng)));
bench_to_time(
c,