Skip to content

Apache DataFusion SQL Query Engine

bench(sort): fix sort axis benchmark run on single partition (#23614)

Raz Luvatona19a1781d0d6
diff --git a/datafusion/core/benches/sort.rs b/datafusion/core/benches/sort.rs
index 4c4cb2ea1e..ac4be5b8b2 100644
--- a/datafusion/core/benches/sort.rs
+++ b/datafusion/core/benches/sort.rs
@@ -66,8 +66,6 @@
//! ~10% duplicates rows)
//! ```
-use std::sync::Arc;
-
use arrow::array::{ArrayRef, StringViewArray, StringViewBuilder};
use arrow::{
array::{Array, DictionaryArray, Float64Array, Int64Array, StringArray},
@@ -87,10 +85,14 @@ use datafusion::{
use datafusion_datasource::memory::MemorySourceConfig;
use datafusion_physical_expr::{PhysicalSortExpr, expressions::col};
use datafusion_physical_expr_common::sort_expr::LexOrdering;
+use std::sync::Arc;
+use std::time::Duration;
/// Benchmarks for SortPreservingMerge stream
use criterion::{Criterion, criterion_group, criterion_main};
+use datafusion_execution::config::SessionConfig;
use futures::StreamExt;
+use itertools::Itertools;
use rand::rngs::StdRng;
use rand::seq::SliceRandom;
use rand::{Rng, SeedableRng};
@@ -224,25 +226,25 @@ fn criterion_benchmark(c: &mut Criterion) {
for (name, f) in &cases {
c.bench_function(&format!("merge sorted {name} {size_label}"), |b| {
let data = f(true);
- let case = BenchCase::merge_sorted(&data);
+ let case = BenchCase::merge_sorted(BATCH_SIZE, &data);
b.iter(move || case.run())
});
c.bench_function(&format!("sort merge {name} {size_label}"), |b| {
let data = f(false);
- let case = BenchCase::sort_merge(&data);
+ let case = BenchCase::sort_merge(BATCH_SIZE, &data);
b.iter(move || case.run())
});
c.bench_function(&format!("sort {name} {size_label}"), |b| {
let data = f(false);
- let case = BenchCase::sort(&data);
+ let case = BenchCase::sort(BATCH_SIZE, &data);
b.iter(move || case.run())
});
c.bench_function(&format!("sort partitioned {name} {size_label}"), |b| {
let data = f(false);
- let case = BenchCase::sort_partitioned(&data);
+ let case = BenchCase::sort_partitioned(BATCH_SIZE, &data);
b.iter(move || case.run())
});
}
@@ -261,9 +263,11 @@ struct BenchCase {
impl BenchCase {
/// Prepare to run a benchmark that merges the specified
/// pre-sorted partitions (streams) together using all keys
- fn merge_sorted(partitions: &[Vec<RecordBatch>]) -> Self {
+ fn merge_sorted(batch_size: usize, partitions: &[Vec<RecordBatch>]) -> Self {
let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap();
- let session_ctx = SessionContext::new();
+ let session_ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_batch_size(batch_size),
+ );
let task_ctx = session_ctx.task_ctx();
let schema = partitions[0][0].schema();
@@ -280,9 +284,11 @@ impl BenchCase {
}
/// Test SortExec in "partitioned" mode followed by a SortPreservingMerge
- fn sort_merge(partitions: &[Vec<RecordBatch>]) -> Self {
+ fn sort_merge(batch_size: usize, partitions: &[Vec<RecordBatch>]) -> Self {
let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap();
- let session_ctx = SessionContext::new();
+ let session_ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_batch_size(batch_size),
+ );
let task_ctx = session_ctx.task_ctx();
let schema = partitions[0][0].schema();
@@ -301,9 +307,11 @@ impl BenchCase {
/// Test SortExec in "partitioned" mode which sorts the input streams
/// individually into some number of output streams
- fn sort(partitions: &[Vec<RecordBatch>]) -> Self {
+ fn sort(batch_size: usize, partitions: &[Vec<RecordBatch>]) -> Self {
let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap();
- let session_ctx = SessionContext::new();
+ let session_ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_batch_size(batch_size),
+ );
let task_ctx = session_ctx.task_ctx();
let schema = partitions[0][0].schema();
@@ -322,9 +330,11 @@ impl BenchCase {
/// Test SortExec in "partitioned" mode which sorts the input streams
/// individually into some number of output streams
- fn sort_partitioned(partitions: &[Vec<RecordBatch>]) -> Self {
+ fn sort_partitioned(batch_size: usize, partitions: &[Vec<RecordBatch>]) -> Self {
let runtime = tokio::runtime::Builder::new_multi_thread().build().unwrap();
- let session_ctx = SessionContext::new();
+ let session_ctx = SessionContext::new_with_config(
+ SessionConfig::new().with_batch_size(batch_size),
+ );
let task_ctx = session_ctx.task_ctx();
let schema = partitions[0][0].schema();
@@ -813,9 +823,46 @@ where
.collect()
}
-type AxisGenerator = Box<dyn Fn(DataProfile, Cardinality, usize) -> PartitionedBatches>;
+fn create_single_partition<T, F>(
+ input: Vec<T>,
+ f: F,
+ batch_size: usize,
+) -> Vec<RecordBatch>
+where
+ F: Fn(Vec<T>) -> RecordBatch,
+{
+ input
+ .into_iter()
+ .chunks(batch_size)
+ .into_iter()
+ .map(|x| f(x.collect_vec()))
+ .collect()
+}
+
+/// Read a duration (seconds, may be fractional) from `var`. panics if set to a value that isn't a number.
+fn env_duration(var: &str) -> Option<Duration> {
+ let s = std::env::var(var).ok()?;
+
+ let secs = s
+ .parse::<f64>()
+ .unwrap_or_else(|e| panic!("invalid {var}={s:?}: {e}"));
+
+ Some(Duration::from_secs_f64(secs))
+}
+
+/// Read a `usize` from `var`. panics if set to a value that isn't an integer.
+fn env_usize(var: &str) -> Option<usize> {
+ let s = std::env::var(var).ok()?;
+
+ Some(
+ s.parse::<usize>()
+ .unwrap_or_else(|e| panic!("invalid {var}={s:?}: {e}")),
+ )
+}
+
+type AxisGenerator = Box<dyn Fn(DataProfile, Cardinality, usize) -> Vec<RecordBatch>>;
-/// Benchmarks `SortExec` (at the 1M input size) across the following axes:
+/// Benchmarks `SortExec` (at the 1M input size) on single partition across the following axes:
/// 1. Sort columns
/// - single column with a specialized impl (primitive or byte(view))
/// - multiple columns, which will use fallback impl
@@ -827,21 +874,43 @@ fn sort_axis_benchmark(c: &mut Criterion) {
let input_size = 1_000_000u64;
let size_label = "1M";
+ const AXIS_BATCH_SIZE: usize = 8192;
+
let cases: Vec<(&str, AxisGenerator)> = vec![
(
"i64",
- Box::new(move |p, card, extra| i64_axis(p, card, extra, input_size)),
+ Box::new(move |p, card, extra| {
+ i64_axis(p, card, extra, input_size, AXIS_BATCH_SIZE)
+ }),
),
(
"utf8 view",
- Box::new(move |p, card, extra| utf8_view_axis(p, card, extra, input_size)),
+ Box::new(move |p, card, extra| {
+ utf8_view_axis(p, card, extra, input_size, AXIS_BATCH_SIZE)
+ }),
),
(
"mixed tuple",
- Box::new(move |p, card, extra| mixed_tuple_axis(p, card, extra, input_size)),
+ Box::new(move |p, card, extra| {
+ mixed_tuple_axis(p, card, extra, input_size, AXIS_BATCH_SIZE)
+ }),
),
];
+ let mut group = c.benchmark_group("sort_axis");
+
+ if let Some(sample_size) = env_usize("SORT_AXIS_SAMPLE_SIZE") {
+ group.sample_size(sample_size);
+ }
+
+ if let Some(warm_up_time) = env_duration("SORT_AXIS_WARMUP_SECS") {
+ group.warm_up_time(warm_up_time);
+ }
+
+ if let Some(measurement_time) = env_duration("SORT_AXIS_MEASUREMENT_SECS") {
+ group.measurement_time(measurement_time);
+ }
+
for (name, f) in &cases {
for card in [Cardinality::Low, Cardinality::High] {
for &extra in EXTRA_COLUMN_COUNTS {
@@ -850,13 +919,13 @@ fn sort_axis_benchmark(c: &mut Criterion) {
DataProfile::Unsorted,
DataProfile::NearlySorted,
] {
- c.bench_function(
- &format!(
+ group.bench_function(
+ format!(
"sort {name} {size_label} {card:?} cardinality {profile:?} +{extra}cols",
),
|b| {
let data = f(profile, card, extra);
- let case = BenchCase::sort(&data);
+ let case = BenchCase::sort_partitioned(AXIS_BATCH_SIZE, &[data]);
b.iter(move || case.run())
},
);
@@ -864,6 +933,8 @@ fn sort_axis_benchmark(c: &mut Criterion) {
}
}
}
+
+ group.finish();
}
/// Single-column i64 batches
@@ -872,9 +943,10 @@ fn i64_axis(
card: Cardinality,
extra: usize,
input_size: u64,
-) -> PartitionedBatches {
+ batch_size: usize,
+) -> Vec<RecordBatch> {
let values = profile.apply(DataGenerator::new(input_size).i64_values_by(card));
- let batches = split_tuples(values, build_i64_batch);
+ let batches = create_single_partition(values, build_i64_batch, batch_size);
with_extra_columns(batches, extra)
}
@@ -884,9 +956,14 @@ fn utf8_view_axis(
card: Cardinality,
extra: usize,
input_size: u64,
-) -> PartitionedBatches {
+ batch_size: usize,
+) -> Vec<RecordBatch> {
let values = profile.apply(DataGenerator::new(input_size).utf8_values_by(card));
- let batches = split_tuples(values, |v| build_utf8_view_batch("utf_view", v));
+ let batches = create_single_partition(
+ values,
+ |v| build_utf8_view_batch("utf_view", v),
+ batch_size,
+ );
with_extra_columns(batches, extra)
}
@@ -896,7 +973,8 @@ fn mixed_tuple_axis(
card: Cardinality,
extra: usize,
input_size: u64,
-) -> PartitionedBatches {
+ batch_size: usize,
+) -> Vec<RecordBatch> {
let mut data_gen = DataGenerator::new(input_size);
let tuples: Vec<MixedTuple> = data_gen
.i64_values_by(card)
@@ -905,12 +983,16 @@ fn mixed_tuple_axis(
.zip(data_gen.utf8_values_by(card))
.zip(data_gen.i64_values_by(card))
.collect();
- let batches = split_tuples(profile.apply(tuples), build_mixed_tuple_batch);
+ let batches = create_single_partition(
+ profile.apply(tuples),
+ build_mixed_tuple_batch,
+ batch_size,
+ );
with_extra_columns(batches, extra)
}
/// Append `n` extra non-sort-key payload columns to every batch, split across i64, string, string view and dictionary
-fn with_extra_columns(batches: PartitionedBatches, n: usize) -> PartitionedBatches {
+fn with_extra_columns(batches: Vec<RecordBatch>, n: usize) -> Vec<RecordBatch> {
if n == 0 {
return batches;
}
@@ -960,31 +1042,25 @@ fn with_extra_columns(batches: PartitionedBatches, n: usize) -> PartitionedBatch
batches
.into_iter()
- .map(|stream| {
- stream
- .into_iter()
- .map(|batch| {
- let num_rows = batch.num_rows();
- let mut fields =
- batch.schema().fields().iter().cloned().collect::<Vec<_>>();
- let mut columns = batch.columns().to_vec();
- generator.input_size = num_rows as u64;
-
- for (col_index, gen_index) in generator_index.iter().enumerate() {
- let gen_fn = &generators[*gen_index];
-
- let array = gen_fn(&mut generator);
- fields.push(Arc::new(Field::new(
- format!("{EXTRA_COLUMN_NAME_PREFIX}{col_index}"),
- array.data_type().clone(),
- array.logical_null_count() > 0,
- )));
- columns.push(array);
- }
-
- RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
- })
- .collect()
+ .map(|batch| {
+ let num_rows = batch.num_rows();
+ let mut fields = batch.schema().fields().iter().cloned().collect::<Vec<_>>();
+ let mut columns = batch.columns().to_vec();
+ generator.input_size = num_rows as u64;
+
+ for (col_index, gen_index) in generator_index.iter().enumerate() {
+ let gen_fn = &generators[*gen_index];
+
+ let array = gen_fn(&mut generator);
+ fields.push(Arc::new(Field::new(
+ format!("{EXTRA_COLUMN_NAME_PREFIX}{col_index}"),
+ array.data_type().clone(),
+ array.logical_null_count() > 0,
+ )));
+ columns.push(array);
+ }
+
+ RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
})
.collect()
}