Apache DataFusion SQL Query Engine
test: run microbenchmarks in extended CI (#25836)
aetos457ccaa56be2
diff --git a/.asf.yaml b/.asf.yamlindex 4515eb3feb..435e65ccc6 100644--- a/.asf.yaml+++ b/.asf.yaml@@ -69,6 +69,7 @@ github:- "cargo check datafusion-spark features"- "cargo test (amd64)"- "cargo test 'extended_tests' (amd64)"+ - "microbenchmark tests (amd64)"- "cargo test hash collisions (amd64)"- "cargo test datafusion-cli (amd64)"- "cargo examples (amd64)"diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.ymlindex 119a6da234..46195097ac 100644--- a/.github/workflows/extended.yml+++ b/.github/workflows/extended.yml@@ -90,6 +90,28 @@ jobs:- name: Cleanuprun: cargo clean+ # Execute Criterion benchmarks once to catch panics without measuring performance.+ linux-test-microbenchmarks:+ name: microbenchmark tests (amd64)+ # Long-running job: run in the merge queue, on release pushes, or manually.+ if: github.event_name != 'pull_request'+ runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}+ container:+ image: amd64/rust+ steps:+ - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group+ submodules: true+ fetch-depth: 1+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-builder+ with:+ rust-version: stable+ - name: Test microbenchmarks+ run: cargo xtask ci step test microbenchmarks+# Check answers are correct when hash values collidehash-collisions:name: cargo test hash collisions (amd64)diff --git a/Cargo.toml b/Cargo.tomlindex e9ec33e5d0..763206a0fd 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -409,3 +409,8 @@ debug = falsedebug-assertions = falsestrip = "debuginfo"incremental = false++# Execute benchmark tests with debug checks, without release-mode build costs.+[profile.ci-bench]+inherits = "ci"+opt-level = 1diff --git a/datafusion/core/benches/sql_planner.rs b/datafusion/core/benches/sql_planner.rsindex 5e5a63c0e9..12d50840ac 100644--- a/datafusion/core/benches/sql_planner.rs+++ b/datafusion/core/benches/sql_planner.rs@@ -203,17 +203,10 @@ fn register_defs(ctx: SessionContext, defs: Vec<TableDef>) -> SessionContext {ctx}-fn register_clickbench_hits_table(rt: &Runtime) -> SessionContext {+fn register_clickbench_hits_table(rt: &Runtime, path: &str) -> SessionContext {let ctx = SessionContext::new();// use an external table for clickbench benchmarks- let path =- if PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists() {- format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")- } else {- format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")- };-let sql =format!("CREATE EXTERNAL TABLE hits_raw STORED AS PARQUET LOCATION '{path}'");@@ -355,15 +348,23 @@ fn union_orderby_query(n: usize) -> String {}fn criterion_benchmark(c: &mut Criterion) {+ // CI uses the small checked-in fixture; performance runs use the full dataset.+ let clickbench_data_path =+ std::env::var("CLICKBENCH_DATA_PATH").unwrap_or_else(|_| {+ let path = format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}");+ if PathBuf::from(&path).exists() {+ path+ } else {+ format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")+ }+ });// verify that we can load the clickbench data prior to running the benchmark- if !PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists()- && !PathBuf::from(format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")).exists()- {- panic!(- "benchmarks/data/hits_partitioned/ could not be loaded. Please run \- 'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark"- )- }+ assert!(+ PathBuf::from(&clickbench_data_path).exists(),+ "{clickbench_data_path} could not be loaded. Please run \+ 'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark \+ or set CLICKBENCH_DATA_PATH to an existing dataset"+ );let ctx = create_context();let rt = Runtime::new().unwrap();@@ -693,7 +694,7 @@ fn criterion_benchmark(c: &mut Criterion) {))).collect::<Vec<_>>();- let clickbench_ctx = register_clickbench_hits_table(&rt);+ let clickbench_ctx = register_clickbench_hits_table(&rt, &clickbench_data_path);// for (i, sql) in clickbench_queries.iter().enumerate() {// c.bench_function(&format!("logical_plan_clickbench_q{}", i + 1), |b| {diff --git a/datafusion/functions/benches/datetime_expressions/to_timestamp.rs b/datafusion/functions/benches/datetime_expressions/to_timestamp.rsindex aaf6610e1c..bcfdb26843 100644--- a/datafusion/functions/benches/datetime_expressions/to_timestamp.rs+++ b/datafusion/functions/benches/datetime_expressions/to_timestamp.rs@@ -21,7 +21,7 @@ use std::sync::Arc;use arrow::array::builder::StringBuilder;use arrow::array::{Array, ArrayRef, StringArray};use arrow::compute::cast;-use arrow::datatypes::{DataType, Field, TimeUnit};+use arrow::datatypes::{DataType, Field};use criterion::{Criterion, criterion_group};use datafusion_common::config::ConfigOptions;use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};@@ -108,8 +108,6 @@ fn data_with_formats() -> (StringArray, StringArray, StringArray, StringArray) {)}fn criterion_benchmark(c: &mut Criterion) {- let return_field =- Field::new("f", DataType::Timestamp(TimeUnit::Nanosecond, None), true).into();let arg_field = Field::new("a", DataType::Utf8, false).into();let arg_fields = vec![arg_field];let mut options = ConfigOptions::default();@@ -117,6 +115,12 @@ fn criterion_benchmark(c: &mut Criterion) {let config_options = Arc::new(options);let to_timestamp_udf = to_timestamp(config_options.as_ref());+ let return_field = Field::new(+ "f",+ to_timestamp_udf.return_type(&[DataType::Utf8]).unwrap(),+ true,+ )+ .into();c.bench_function("to_timestamp_no_formats_utf8", |b| {let to_timestamp_udf = Arc::clone(&to_timestamp_udf);diff --git a/datafusion/functions/benches/string_expressions/repeat.rs b/datafusion/functions/benches/string_expressions/repeat.rsindex faa4953838..a374be1f89 100644--- a/datafusion/functions/benches/string_expressions/repeat.rs+++ b/datafusion/functions/benches/string_expressions/repeat.rs@@ -60,7 +60,6 @@ fn create_args<O: OffsetSizeTrait>(fn invoke_repeat_with_args(args: Vec<ColumnarValue>,- repeat_times: i64,) -> Result<ColumnarValue, DataFusionError> {let arg_fields = args.iter()@@ -69,11 +68,22 @@ fn invoke_repeat_with_args(.collect::<Vec<_>>();let config_options = Arc::new(ConfigOptions::default());+ let return_type = string::repeat().return_type(+ &args+ .iter()+ .map(ColumnarValue::data_type)+ .collect::<Vec<_>>(),+ )?;+ let number_rows = match &args[0] {+ ColumnarValue::Array(array) => array.len(),+ ColumnarValue::Scalar(_) => 1,+ };+string::repeat().invoke_with_args(ScalarFunctionArgs {args,arg_fields,- number_rows: repeat_times as usize,- return_field: Field::new("f", DataType::Utf8, true).into(),+ number_rows,+ return_field: Field::new("f", return_type, true).into(),config_options: Arc::clone(&config_options),})}@@ -111,7 +121,7 @@ fn criterion_benchmark(c: &mut Criterion) {Field::new("b", DataType::Int64, false).into(),],number_rows: 1,- return_field: Field::new("f", DataType::Utf8, true).into(),+ return_field: Field::new("f", DataType::Utf8View, true).into(),config_options: Arc::clone(&config_options),};b.iter(|| black_box(repeat_fn.invoke_with_args(args.clone()).unwrap()))@@ -131,7 +141,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -142,7 +152,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -153,7 +163,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -173,7 +183,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -184,7 +194,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -195,7 +205,7 @@ fn criterion_benchmark(c: &mut Criterion) {|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(invoke_repeat_with_args(args_cloned).unwrap())})},);@@ -210,12 +220,25 @@ fn criterion_benchmark(c: &mut Criterion) {group.measurement_time(Duration::from_secs(10));let args = create_args::<i32>(size, 2, repeat_times, false);+ // Verify once outside the timed loop that the failure is the expected+ // overflow error, and not an unrelated error that also happens to fail.+ let err = invoke_repeat_with_args(args.clone())+ .expect_err("repeat should reject an overflowing output size")+ .to_string();+ assert!(+ err.contains("string size overflow on repeat"),+ "unexpected error: {err}"+ );group.bench_function(format!("repeat_string overflow [size={size}, repeat_times={repeat_times}]"),|b| {b.iter(|| {let args_cloned = args.clone();- black_box(invoke_repeat_with_args(args_cloned, repeat_times))+ black_box(+ invoke_repeat_with_args(args_cloned).expect_err(+ "repeat should reject an overflowing output size",+ ),+ )})},);diff --git a/datafusion/spark/benches/slice.rs b/datafusion/spark/benches/slice.rsindex da392dc042..dd2ab787e8 100644--- a/datafusion/spark/benches/slice.rs+++ b/datafusion/spark/benches/slice.rs@@ -23,7 +23,8 @@ use arrow::datatypes::{DataType, Field, Int64Type};use criterion::{Criterion, criterion_group, criterion_main};use datafusion_common::ScalarValue;use datafusion_common::config::ConfigOptions;-use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};+use datafusion_expr::type_coercion::functions::fields_with_udf;+use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};use datafusion_spark::function::array::slice;use rand::rngs::StdRng;use rand::{Rng, SeedableRng};@@ -115,6 +116,19 @@ fn array_slice_benchmark(<Arc<Field>>::from(Field::new(format!("arg_{idx}"), arg.data_type(), true))}).collect::<Vec<_>>();+ // Apply planner coercion (including ListView to List) before timing the UDF.+ let arg_fields = fields_with_udf(&arg_fields, array_slice.as_ref()).unwrap();+ let args = args+ .iter()+ .zip(&arg_fields)+ .map(|(arg, field)| arg.cast_to(field.data_type(), None).unwrap())+ .collect::<Vec<_>>();+ let return_field = array_slice+ .return_field_from_args(ReturnFieldArgs {+ arg_fields: &arg_fields,+ scalar_arguments: &vec![None; arg_fields.len()],+ })+ .unwrap();c.bench_function(name, |b| {b.iter(|| {black_box(@@ -123,8 +137,7 @@ fn array_slice_benchmark(args: args.clone(),arg_fields: arg_fields.clone(),number_rows: size,- return_field: Field::new_list_field(args[0].data_type(), true)- .into(),+ return_field: Arc::clone(&return_field),config_options: Arc::new(ConfigOptions::default()),}).unwrap(),@@ -161,7 +174,10 @@ fn criterion_benchmark(c: &mut Criterion) {let scalar_to = ColumnarValue::Scalar(ScalarValue::from(child_array_size as i64 / 2));for input in [list_array, list_view_array] {- let input_type = input.data_type().to_string();+ let input_type = match input.data_type() {+ DataType::ListView(_) => "ListView -> List (coercion excluded)".to_string(),+ _ => input.data_type().to_string(),+ };array_slice_benchmark(&format!("slice: input {input_type}, array args, no stride"),diff --git a/xtask/src/ci_steps.rs b/xtask/src/ci_steps.rsindex bbb769ecee..76ca9100e4 100644--- a/xtask/src/ci_steps.rs+++ b/xtask/src/ci_steps.rs@@ -55,7 +55,7 @@ static CI_STEPS: &[StepInfo] = &[},StepInfo {command: "test",- help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",+ help_usage: "test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",help_examples: &["test cli", "test workspace"],help_description: "Run a CI test suite",error_message: "Cargo test step failed",@@ -290,6 +290,38 @@ impl StepContext {"integration-tests",]);}+ "microbenchmarks" => {+ // The SQL benchmark harness needs separately generated datasets+ // and is covered by the benchmark result verification job.+ // Select only bench targets (not lib/bin unit tests), and tell+ // Criterion to execute each benchmark once instead of timing it.+ // Keep debug assertions and overflow checks enabled without the+ // cost of unoptimized execution on the larger benchmark datasets.+ command.env(+ "CLICKBENCH_DATA_PATH",+ self.root+ .join("datafusion/core/tests/data/clickbench_hits_10.parquet"),+ );+ command.args([+ "test",+ "--profile",+ "ci-bench",+ "--no-fail-fast",+ "--workspace",+ "--exclude",+ "datafusion-examples",+ "--exclude",+ "datafusion-benchmarks",+ "--exclude",+ "datafusion-cli",+ "--bench",+ "*",+ "--features",+ "datafusion-physical-plan/test_utils",+ "--",+ "--test",+ ]);+ }"benchmark-plan" => {command.args([@@ -616,7 +648,7 @@ fn required_env(name: &str) -> Result<String> {mod tests {use super::*;- const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";+ const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";fn args(args: &[&str]) -> Vec<String> {args.iter().map(|arg| (*arg).to_string()).collect()@@ -796,6 +828,25 @@ cargo test \--lib \--tests \--features integration-tests+")+ },+ },+ Cmd {+ command: &["test", "microbenchmarks"],+ expected: |actual| {+ insta::assert_snapshot!(actual, @r"+cd /workspace && \+CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \+cargo test \+--profile ci-bench \+--no-fail-fast \+--workspace \+--exclude datafusion-examples \+--exclude datafusion-benchmarks \+--exclude datafusion-cli \+--bench '*' \+--features datafusion-physical-plan/test_utils \+-- --test")},},