Skip to content

Apache DataFusion SQL Query Engine

bench: bound sort-axis smoke inputs (#26837)

Bruno Volpato209917419fc0
diff --git a/datafusion/core/benches/sort.rs b/datafusion/core/benches/sort.rs
index b7c1fa41af..9f43fc2d17 100644
--- a/datafusion/core/benches/sort.rs
+++ b/datafusion/core/benches/sort.rs
@@ -849,7 +849,7 @@ fn env_usize(var: &str) -> Option<usize> {
type AxisGenerator = Box<dyn Fn(DataProfile, Cardinality, usize) -> Vec<RecordBatch>>;
-/// Benchmarks `SortExec` (at the 1M input size) on single partition across the following axes:
+/// Benchmarks `SortExec` on a single partition (default 1M rows) across the following axes.
/// 1. Sort columns
/// - single column with a specialized impl (primitive or byte(view))
/// - multiple columns, which will use fallback impl
@@ -858,8 +858,14 @@ type AxisGenerator = Box<dyn Fn(DataProfile, Cardinality, usize) -> Vec<RecordBa
/// 3. Value cardinality - whether the sort-key values overlap or not
/// 4. Input ordering - already sorted / unsorted / nearly sorted
fn sort_axis_benchmark(c: &mut Criterion) {
- let input_size = 1_000_000u64;
- let size_label = "1M";
+ let input_size = env_usize("SORT_AXIS_ROWS").unwrap_or(1_000_000) as u64;
+ let size_label = if input_size.is_multiple_of(1_000_000) {
+ format!("{}M", input_size / 1_000_000)
+ } else if input_size.is_multiple_of(1_000) {
+ format!("{}K", input_size / 1_000)
+ } else {
+ input_size.to_string()
+ };
const AXIS_BATCH_SIZE: usize = 8192;
diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rs
index c6b605d36e..a23dd563af 100644
--- a/xtask/src/ci_steps.rs
+++ b/xtask/src/ci_steps.rs
@@ -329,6 +329,8 @@ impl StepContext {
The SQL benchmark harness needs separately generated datasets and\n\
is covered by the benchmark result verification job.",
);
+ // Keep the i64 key payload above 1 MiB in the smoke run.
+ cmd_builder.env("SORT_AXIS_ROWS", "200000");
cmd_builder.env(
"CLICKBENCH_DATA_PATH",
self.root
@@ -833,6 +835,7 @@ cargo test \
cd /workspace && \
CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \
+SORT_AXIS_ROWS=200000 \
cargo test \
--profile ci-bench \
--no-fail-fast \