Apache DataFusion SQL Query Engine
feat: benchmark_runner, improve `--list`, optional `DATA_DIR` (#23354)
Bruce Ritchie1c6350a28222
diff --git a/benchmarks/src/bin/benchmark_runner.rs b/benchmarks/src/bin/benchmark_runner.rsindex 5a46e9d8a0..71b3b1b9e0 100644--- a/benchmarks/src/bin/benchmark_runner.rs+++ b/benchmarks/src/bin/benchmark_runner.rs@@ -17,7 +17,24 @@//! DataFusion SQL benchmark runner.-use datafusion_benchmarks::sql_benchmark_runner;+use clap::{ArgAction, ArgMatches, CommandFactory, FromArgMatches, Parser};+use criterion::Criterion;+use datafusion::error::Result;+use datafusion::prelude::SessionContext;+use datafusion_benchmarks::sql_benchmark::SqlBenchmark;+use datafusion_benchmarks::sql_benchmark_runner::{+ BenchmarkFilter, SqlRunConfig, default_sql_benchmark_directory, ensure_selection,+ filter_benchmarks, finish_benchmark, format_benchmark_list,+ load_benchmark_definitions, make_ctx, prepare_benchmark,+ run_criterion_benchmarks_impl, sort_benchmarks,+};+use datafusion_benchmarks::util::{BenchmarkRun, CommonOpt, print_memory_stats};+use datafusion_common::instant::Instant;+use datafusion_common::{DataFusionError, exec_datafusion_err};+use datafusion_common_runtime::SpawnedTask;+use std::collections::BTreeMap;+use std::io::IsTerminal;+use std::path::Path;#[cfg(feature = "snmalloc")]#[global_allocator]@@ -32,8 +49,932 @@ static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;#[tokio::main]async fn main() {env_logger::init();- if let Err(error) = sql_benchmark_runner::run_cli().await {+ if let Err(error) = run_cli().await {eprintln!("Error: {error}");std::process::exit(1);}}++#[derive(Debug)]+enum CliAction {+ List,+ Simple(SqlRunConfig),+ Criterion {+ config: SqlRunConfig,+ save_baseline: Option<String>,+ },+}++#[derive(Debug, Parser)]+#[command(+ name = "benchmark_runner",+ about = "Run DataFusion SQL benchmarks",+ styles = criterion_like_styles(),+)]+struct Cli {+ #[arg(value_name = "BENCHMARK", help = "SQL benchmark group to run")]+ benchmark: Option<String>,++ #[arg(short = 'q', long = "query", env = "BENCH_QUERY")]+ query: Option<String>,++ #[arg(long = "subgroup", env = "BENCH_SUBGROUP")]+ subgroup: Option<String>,++ #[command(flatten)]+ common: CommonOpt,++ #[arg(+ long = "criterion",+ action = ArgAction::SetTrue,+ help = "Run benchmarks with Criterion"+ )]+ criterion: bool,++ #[arg(+ long = "list",+ action = ArgAction::SetTrue,+ help = "List available SQL benchmark groups"+ )]+ list: bool,++ #[arg(+ short = 'o',+ long = "output",+ help = "Write simple runner results as JSON to this path"+ )]+ output: Option<std::path::PathBuf>,++ #[arg(+ long = "save-baseline",+ value_name = "BASELINE",+ help = "Save Criterion measurements to the named baseline"+ )]+ save_baseline: Option<String>,+}++/// Parses CLI arguments, runs the selected action, and prints any output.+async fn run_cli() -> Result<()> {+ let matches = Cli::command().get_matches();+ let action = cli_action_from_matches(&matches)?;+ let output = run_cli_action(action, &default_sql_benchmark_directory()).await?;++ if !output.is_empty() {+ println!("{output}");+ }++ Ok(())+}++/// Converts parsed arguments into an executable action and validates mode options.+fn cli_action_from_matches(matches: &ArgMatches) -> Result<CliAction> {+ let cli = Cli::from_arg_matches(matches)+ .map_err(|e| DataFusionError::External(Box::new(e)))?;++ if cli.list || cli.benchmark.is_none() {+ return Ok(CliAction::List);+ }++ if cli.criterion && cli.output.is_some() {+ return Err(exec_datafusion_err!(+ "--output cannot be used with --criterion"+ ));+ }+ if !cli.criterion && cli.save_baseline.is_some() {+ return Err(exec_datafusion_err!(+ "--save-baseline cannot be used without --criterion"+ ));+ }++ // we need to know if iterations was set on the command line, not the default value+ let iterations_from_cli = matches.value_source("iterations")+ == Some(clap::parser::ValueSource::CommandLine);++ if cli.criterion && iterations_from_cli {+ return Err(exec_datafusion_err!(+ "--iterations cannot be used with --criterion"+ ));+ }+ if !cli.criterion && cli.common.iterations == 0 {+ return Err(exec_datafusion_err!("iterations must be greater than zero"));+ }++ let config = SqlRunConfig {+ common: cli.common,+ filter: BenchmarkFilter {+ name: cli.benchmark,+ subgroup: cli.subgroup,+ query: cli.query,+ },+ persist_results: false,+ validate_results: false,+ output: cli.output,+ };++ if cli.criterion {+ Ok(CliAction::Criterion {+ config,+ save_baseline: cli.save_baseline,+ })+ } else {+ Ok(CliAction::Simple(config))+ }+}++/// Executes a parsed CLI action and returns any text that should be printed.+async fn run_cli_action(action: CliAction, benchmark_dir: &Path) -> Result<String> {+ match action {+ CliAction::List => {+ let ctx = SessionContext::new();+ let benchmarks =+ load_benchmarks(&BenchmarkFilter::default(), &ctx, benchmark_dir).await?;++ Ok(format_benchmark_list(&benchmarks))+ }+ CliAction::Simple(config) => {+ run_simple_benchmarks(benchmark_dir, config).await?;+ Ok(String::new())+ }+ CliAction::Criterion {+ config,+ save_baseline,+ } => {+ if config.output.is_some() {+ return Err(exec_datafusion_err!(+ "--output cannot be used with --criterion"+ ));+ }+ let benchmark_dir = benchmark_dir.to_path_buf();++ SpawnedTask::spawn_blocking(move || {+ run_criterion_benchmarks(+ &benchmark_dir,+ &config,+ save_baseline.as_deref(),+ )+ })+ .await+ .map_err(|e| DataFusionError::External(Box::new(e)))??;++ Ok(String::new())+ }+ }+}++/// Loads benchmark definitions, applies CLI-style filters, and sorts each group.+pub async fn load_benchmarks(+ filter: &BenchmarkFilter,+ ctx: &SessionContext,+ benchmark_dir: &Path,+) -> Result<BTreeMap<String, Vec<SqlBenchmark>>> {+ let benches = load_benchmark_definitions(filter, ctx, benchmark_dir).await?;+ let mut benches = filter_benchmarks(filter, benches);++ sort_benchmarks(&mut benches);++ Ok(benches)+}++/// Builds the default Criterion runner and optionally records a named baseline.+fn run_criterion_benchmarks(+ benchmark_dir: &Path,+ config: &SqlRunConfig,+ save_baseline: Option<&str>,+) -> Result<()> {+ let mut criterion = Criterion::default()+ .sample_size(10)+ .with_output_color(std::io::stdout().is_terminal());++ if let Some(save_baseline) = save_baseline {+ criterion = criterion.save_baseline(save_baseline.to_string());+ }++ run_criterion_benchmarks_impl(benchmark_dir, config, &mut criterion)?;+ criterion.final_summary();++ Ok(())+}++/// Runs selected benchmarks with fixed iteration counts and optional JSON output.+pub async fn run_simple_benchmarks(+ benchmark_dir: &Path,+ config: SqlRunConfig,+) -> Result<()> {+ if config.common.iterations == 0 {+ return Err(exec_datafusion_err!("iterations must be greater than zero"));+ }++ let listing_ctx = make_ctx(&config.common)?;+ let all_benchmarks =+ load_benchmark_definitions(&config.filter, &listing_ctx, benchmark_dir).await?;+ let selected = filter_benchmarks(&config.filter, all_benchmarks.clone());+ let mut run = BenchmarkRun::new();++ ensure_selection(&config.filter, &all_benchmarks, &selected)?;++ for (_group, benchmarks) in selected {+ for mut benchmark in benchmarks {+ let ctx = make_ctx(&config.common)?;+ let result =+ run_simple_benchmark(&ctx, &mut benchmark, &config, &mut run).await;+ let cleanup_result = benchmark.cleanup(&ctx).await;++ finish_benchmark(result, cleanup_result)?;+ }+ }++ run.maybe_write_json(config.output.as_ref())?;++ Ok(())+}++/// Runs one benchmark case, recording each timed iteration.+async fn run_simple_benchmark(+ ctx: &SessionContext,+ benchmark: &mut SqlBenchmark,+ config: &SqlRunConfig,+ run: &mut BenchmarkRun,+) -> Result<()> {+ prepare_benchmark(ctx, benchmark, config).await?;++ let case_name = benchmark_case_name(benchmark);++ run.start_new_case(&case_name);++ for iteration in 0..config.common.iterations {+ let start = Instant::now();+ let row_count = benchmark.run(ctx, false).await?;+ let elapsed = start.elapsed();+ let ms = elapsed.as_secs_f64() * 1000.0;++ println!("{case_name} iteration {iteration}: {ms:.1} ms, {row_count} rows");++ run.write_iter(elapsed, row_count);+ }++ print_memory_stats();++ Ok(())+}++fn benchmark_case_name(benchmark: &SqlBenchmark) -> String {+ let mut name = format!("{}/{}", benchmark.group(), benchmark.name());++ if !benchmark.subgroup().is_empty() {+ name.push('/');+ name.push_str(benchmark.subgroup());+ }++ name+}++fn criterion_like_styles() -> clap::builder::Styles {+ use clap::builder::styling::AnsiColor;++ clap::builder::Styles::styled()+ .header(AnsiColor::Green.on_default().bold())+ .usage(AnsiColor::Green.on_default().bold())+ .literal(AnsiColor::Cyan.on_default().bold())+ .placeholder(AnsiColor::Cyan.on_default())+}++#[cfg(test)]+mod tests {+ use super::*;+ use datafusion_benchmarks::sql_benchmark_runner::unknown_benchmark_error;+ use std::fs;+ use std::path::{Path, PathBuf};++ fn write_benchmark(root: &Path, relative_path: &str, contents: &str) -> PathBuf {+ let path = root.join(relative_path);++ fs::create_dir_all(path.parent().unwrap()).unwrap();+ fs::write(&path, contents).unwrap();++ path+ }++ fn common(iterations: usize) -> CommonOpt {+ CommonOpt {+ iterations,+ partitions: None,+ batch_size: None,+ mem_pool_type: "fair".to_string(),+ memory_limit: None,+ sort_spill_reservation_bytes: None,+ debug: false,+ simulate_latency: false,+ }+ }++ fn parse_cli_from<I, T>(args: I) -> Result<CliAction>+ where+ I: IntoIterator<Item = T>,+ T: Into<std::ffi::OsString> + Clone,+ {+ let matches = Cli::command()+ .try_get_matches_from(args)+ .map_err(|e| DataFusionError::External(Box::new(e)))?;++ cli_action_from_matches(&matches)+ }++ async fn run_cli_with_dir<I, T>(args: I, benchmark_dir: &Path) -> Result<String>+ where+ I: IntoIterator<Item = T>,+ T: Into<std::ffi::OsString> + Clone,+ {+ run_cli_action(parse_cli_from(args)?, benchmark_dir).await+ }++ #[test]+ fn cli_lists_when_benchmark_is_omitted() {+ let action = parse_cli_from(["benchmark_runner"]).unwrap();++ assert!(matches!(action, CliAction::List));+ }++ #[test]+ fn cli_lists_with_explicit_list_flag() {+ let action = parse_cli_from(["benchmark_runner", "--list"]).unwrap();++ assert!(matches!(action, CliAction::List));+ }++ #[test]+ fn cli_defaults_to_basic_runner() {+ let action =+ parse_cli_from(["benchmark_runner", "tpch", "--query", "1"]).unwrap();+ let CliAction::Simple(config) = action else {+ panic!("expected basic runner");+ };++ assert_eq!(config.filter.name.as_deref(), Some("tpch"));+ assert_eq!(config.filter.query.as_deref(), Some("1"));+ }++ #[test]+ fn cli_reads_query_from_env() {+ let previous = std::env::var_os("BENCH_QUERY");+ // SAFETY: This test restores BENCH_QUERY before returning and does not+ // spawn threads while the environment variable is overridden.+ unsafe {+ std::env::set_var("BENCH_QUERY", "8");+ }++ let action = parse_cli_from(["benchmark_runner", "tpch"]);++ unsafe {+ match previous {+ Some(value) => std::env::set_var("BENCH_QUERY", value),+ None => std::env::remove_var("BENCH_QUERY"),+ }+ }++ let action = action.unwrap();+ let CliAction::Simple(config) = action else {+ panic!("expected basic runner");+ };++ assert_eq!(config.filter.name.as_deref(), Some("tpch"));+ assert_eq!(config.filter.query.as_deref(), Some("8"));+ }++ #[test]+ fn cli_accepts_criterion_runner() {+ let action = parse_cli_from([+ "benchmark_runner",+ "tpch",+ "--criterion",+ "--save-baseline",+ "main",+ ])+ .unwrap();++ let CliAction::Criterion {+ config,+ save_baseline,+ } = action+ else {+ panic!("expected criterion runner");+ };++ assert_eq!(config.filter.name.as_deref(), Some("tpch"));+ assert_eq!(save_baseline.as_deref(), Some("main"));+ }++ #[test]+ fn cli_rejects_output_with_criterion() {+ let err = parse_cli_from([+ "benchmark_runner",+ "tpch",+ "--criterion",+ "--output",+ "results.json",+ ])+ .unwrap_err();++ assert!(err.to_string().contains("--output"));+ assert!(err.to_string().contains("--criterion"));+ }++ #[test]+ fn cli_rejects_save_baseline_without_criterion() {+ let err = parse_cli_from(["benchmark_runner", "tpch", "--save-baseline", "main"])+ .unwrap_err();++ assert!(err.to_string().contains("--save-baseline"));+ assert!(err.to_string().contains("--criterion"));+ }++ #[test]+ fn cli_rejects_iterations_with_criterion() {+ let err = parse_cli_from([+ "benchmark_runner",+ "tpch",+ "--criterion",+ "--iterations",+ "3",+ ])+ .unwrap_err();++ assert!(err.to_string().contains("--iterations"));+ assert!(err.to_string().contains("--criterion"));+ }++ #[test]+ fn cli_rejects_zero_basic_iterations() {+ let err = parse_cli_from(["benchmark_runner", "tpch", "--iterations", "0"])+ .unwrap_err();++ assert!(err.to_string().contains("iterations"));+ }++ #[tokio::test]+ async fn run_cli_lists_when_no_benchmark_is_supplied() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );++ let output = run_cli_with_dir(["benchmark_runner"], temp.path())+ .await+ .unwrap();++ assert!(output.contains("SQL benchmarks:"));+ assert!(output.contains("alpha"));+ }++ #[tokio::test]+ async fn run_cli_lists_with_explicit_list_flag() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );++ let output = run_cli_with_dir(["benchmark_runner", "--list"], temp.path())+ .await+ .unwrap();++ assert!(output.contains("SQL benchmarks:"));+ assert!(output.contains("alpha"));+ }++ #[tokio::test]+ async fn run_cli_reports_unknown_benchmark_with_list() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );++ let err = run_cli_with_dir(["benchmark_runner", "missing"], temp.path())+ .await+ .unwrap_err();+ let message = err.to_string();++ assert!(message.contains("unknown benchmark 'missing'"), "{message}");+ assert!(message.contains("alpha"), "{message}");+ }++ #[test]+ fn criterion_runner_saves_named_baseline() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );++ let output = tempfile::tempdir().unwrap();+ let mut criterion = Criterion::default()+ .sample_size(10)+ .warm_up_time(std::time::Duration::from_millis(1))+ .measurement_time(std::time::Duration::from_millis(10))+ .without_plots()+ .output_directory(output.path())+ .save_baseline("acceptance".to_string());+ let config = SqlRunConfig {+ common: common(3),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ },+ persist_results: false,+ validate_results: false,+ output: None,+ };++ run_criterion_benchmarks_impl(temp.path(), &config, &mut criterion).unwrap();+ criterion.final_summary();++ assert!(+ output+ .path()+ .join("alpha")+ .join("Q01")+ .join("acceptance")+ .join("estimates.json")+ .exists()+ );+ }++ #[tokio::test]+ async fn simple_runner_reports_unknown_query_for_known_benchmark() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );++ let config = SqlRunConfig {+ common: common(1),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("9".to_string()),+ },+ persist_results: false,+ validate_results: false,+ output: None,+ };+ let err = run_simple_benchmarks(temp.path(), config)+ .await+ .unwrap_err();+ let message = err.to_string();++ assert!(+ message.contains("no SQL benchmark query matched benchmark 'alpha'"),+ "{message}"+ );+ assert!(message.contains("query '9'"), "{message}");+ assert!(message.contains("normalized: 'Q09'"), "{message}");+ assert!(message.contains("Available alpha queries:"), "{message}");+ assert!(message.contains("Q01"), "{message}");+ assert!(!message.contains("unknown benchmark 'alpha'"), "{message}");+ }++ #[tokio::test]+ async fn simple_runner_reports_unknown_subgroup_for_known_benchmark() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\nsubgroup wide\n\nrun\nSELECT 1\n",+ );++ let config = SqlRunConfig {+ common: common(1),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: Some("narrow".to_string()),+ query: None,+ },+ persist_results: false,+ validate_results: false,+ output: None,+ };+ let err = run_simple_benchmarks(temp.path(), config)+ .await+ .unwrap_err();+ let message = err.to_string();++ assert!(+ message.contains(+ "no SQL benchmark subgroup matched benchmark 'alpha' with subgroup 'narrow'"+ ),+ "{message}"+ );+ assert!(message.contains("Available alpha subgroups:"), "{message}");+ assert!(message.contains("wide"), "{message}");+ assert!(!message.contains("unknown benchmark 'alpha'"), "{message}");+ }++ #[tokio::test]+ async fn basic_runner_executes_iterations_and_writes_json() {+ let temp = tempfile::tempdir().unwrap();+ let output = temp.path().join("results.json");++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT * FROM (VALUES (1), (2)) AS t(v)\n",+ );++ let config = SqlRunConfig {+ common: common(2),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ },+ persist_results: false,+ validate_results: false,+ output: Some(output.clone()),+ };++ run_simple_benchmarks(temp.path(), config).await.unwrap();++ let json = fs::read_to_string(output).unwrap();++ assert!(json.contains("\"query\": \"alpha/Q01\""));+ assert!(json.contains("\"row_count\": 2"));+ assert_eq!(json.matches("\"row_count\": 2").count(), 2);+ }++ #[tokio::test]+ async fn basic_runner_reports_run_and_cleanup_failures() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT * FROM missing_run_table\n\ncleanup\nSELECT * FROM missing_cleanup_table\n",+ );++ let config = SqlRunConfig {+ common: common(1),+ filter: BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ },+ persist_results: false,+ validate_results: false,+ output: None,+ };+ let err = run_simple_benchmarks(temp.path(), config)+ .await+ .unwrap_err();+ let message = err.to_string();++ assert!(message.contains("missing_run_table"), "{message}");+ assert!(message.contains("cleanup also failed"), "{message}");+ assert!(message.contains("missing_cleanup_table"), "{message}");+ }++ #[tokio::test]+ async fn discovery_lists_groups_from_directories() {+ let temp = tempfile::tempdir().unwrap();+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );+ write_benchmark(+ temp.path(),+ "beta/benchmarks/q02.benchmark",+ "name Q02\n\nrun\nSELECT 2\n",+ );+ let ctx = SessionContext::new();+ let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())+ .await+ .unwrap();++ assert_eq!(benches["alpha"].len(), 1);+ assert_eq!(benches["beta"].len(), 1);+ }++ #[tokio::test]+ async fn discovery_filters_benchmark_subgroup_and_query() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\nsubgroup wide\n\nrun\nSELECT 1\n",+ );+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q02.benchmark",+ "name Q02\nsubgroup narrow\n\nrun\nSELECT 2\n",+ );++ let ctx = SessionContext::new();+ let filter = BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: Some("wide".to_string()),+ query: Some("1".to_string()),+ };+ let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();++ assert_eq!(benches.len(), 1);+ assert_eq!(benches["alpha"].len(), 1);+ assert_eq!(benches["alpha"][0].name(), "Q01");+ }++ #[tokio::test]+ async fn cli_subgroup_filter_is_used_for_benchmark_replacements() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "wide_schema/benchmarks/q01.benchmark",+ "name Q01\nsubgroup ${BENCH_SUBGROUP:-wide}\n\nrun\nSELECT '${BENCH_SUBGROUP:-wide}'\n",+ );++ let ctx = SessionContext::new();+ let filter = BenchmarkFilter {+ name: Some("wide_schema".to_string()),+ subgroup: Some("narrow".to_string()),+ query: None,+ };+ let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();++ assert_eq!(benches["wide_schema"].len(), 1);+ assert_eq!(benches["wide_schema"][0].subgroup(), "narrow");+ }++ #[tokio::test]+ async fn benchmark_replacements_default_data_dir_to_benchmarks_data() {+ let temp = tempfile::tempdir().unwrap();+ let previous = std::env::var_os("DATA_DIR");++ unsafe {+ std::env::remove_var("DATA_DIR");+ }++ write_benchmark(+ temp.path(),+ "clickbench/benchmarks/q01.benchmark",+ "name Q01\nsubgroup ${DATA_DIR:-data}\n\nrun\nSELECT 1\n",+ );++ let ctx = SessionContext::new();+ let benches =+ load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path()).await;++ unsafe {+ match previous {+ Some(value) => std::env::set_var("DATA_DIR", value),+ None => std::env::remove_var("DATA_DIR"),+ }+ }++ let benches = benches.unwrap();++ let expected = PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .join("data")+ .to_string_lossy()+ .into_owned();++ assert_eq!(benches["clickbench"][0].subgroup(), expected);+ }++ #[tokio::test]+ async fn query_filter_matches_starts_with_when_exact_match_is_absent() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01a.benchmark",+ "name Q01a\n\nrun\nSELECT 1\n",+ );++ let ctx = SessionContext::new();+ let filter = BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ };+ let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();++ assert_eq!(benches["alpha"].len(), 1);+ assert_eq!(benches["alpha"][0].name(), "Q01a");+ }++ #[tokio::test]+ async fn query_filter_matches_token_start_when_exact_match_is_absent() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "predicate_eval/benchmarks/costsel/q01.benchmark",+ "name costsel_q01_regexp_selective_last\n\nrun\nSELECT 1\n",+ );++ let ctx = SessionContext::new();+ let filter = BenchmarkFilter {+ name: Some("predicate_eval".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ };+ let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();++ assert_eq!(benches["predicate_eval"].len(), 1);+ assert_eq!(+ benches["predicate_eval"][0].name(),+ "costsel_q01_regexp_selective_last"+ );+ }++ #[tokio::test]+ async fn query_filter_prefers_starts_with_match_over_token_match() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/token.benchmark",+ "name costsel_q01_regexp_selective_last\n\nrun\nSELECT 1\n",+ );+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01a.benchmark",+ "name Q01a\n\nrun\nSELECT 2\n",+ );++ let ctx = SessionContext::new();+ let filter = BenchmarkFilter {+ name: Some("alpha".to_string()),+ subgroup: None,+ query: Some("1".to_string()),+ };+ let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();++ assert_eq!(benches["alpha"].len(), 1);+ assert_eq!(benches["alpha"][0].name(), "Q01a");+ }++ #[tokio::test]+ async fn list_output_is_sorted_and_includes_counts() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "beta/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );+ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q02.benchmark",+ "name Q02\n\nrun\nSELECT 2\n",+ );++ let ctx = SessionContext::new();+ let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())+ .await+ .unwrap();+ let output = format_benchmark_list(&benches);++ assert!(output.starts_with("SQL benchmarks:\n alpha"));+ assert!(output.contains("alpha 2 queries"));+ assert!(output.contains("beta 1 query"));+ }++ #[tokio::test]+ async fn unknown_benchmark_error_includes_available_benchmarks() {+ let temp = tempfile::tempdir().unwrap();++ write_benchmark(+ temp.path(),+ "alpha/benchmarks/q01.benchmark",+ "name Q01\n\nrun\nSELECT 1\n",+ );+ let ctx = SessionContext::new();+ let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())+ .await+ .unwrap();+ let message = unknown_benchmark_error("missing", &benches).to_string();++ assert!(message.contains("unknown benchmark 'missing'"), "{message}");+ assert!(message.contains("alpha"), "{message}");+ }+}diff --git a/benchmarks/src/sql_benchmark_runner.rs b/benchmarks/src/sql_benchmark_runner.rsindex edbf43d39b..f1cb3ad2f7 100644--- a/benchmarks/src/sql_benchmark_runner.rs+++ b/benchmarks/src/sql_benchmark_runner.rs@@ -19,17 +19,14 @@//! SQL benchmark harness.use crate::sql_benchmark::SqlBenchmark;-use crate::util::{BenchmarkRun, CommonOpt, print_memory_stats};-use clap::{ArgAction, ArgMatches, CommandFactory, FromArgMatches, Parser};+use crate::util::{CommonOpt, print_memory_stats};use criterion::{Criterion, SamplingMode};use datafusion::error::Result;use datafusion::prelude::SessionContext;-use datafusion_common::{DataFusionError, exec_datafusion_err, instant::Instant};-use datafusion_common_runtime::SpawnedTask;+use datafusion_common::{DataFusionError, exec_datafusion_err};use std::any::Any;use std::collections::{BTreeMap, HashMap};use std::fs;-use std::io::IsTerminal;use std::panic::{AssertUnwindSafe, catch_unwind};use std::path::{Path, PathBuf};use tokio::runtime::Runtime;@@ -50,70 +47,6 @@ pub struct SqlRunConfig {pub output: Option<PathBuf>,}-#[derive(Debug)]-pub enum CliAction {- List,- Simple(SqlRunConfig),- Criterion {- config: SqlRunConfig,- save_baseline: Option<String>,- },-}--#[derive(Debug, Parser)]-#[command(- name = "benchmark_runner",- about = "Run DataFusion SQL benchmarks",- styles = criterion_like_styles(),-)]-pub struct Cli {- #[arg(value_name = "BENCHMARK", help = "SQL benchmark group to run")]- pub benchmark: Option<String>,-- #[arg(short = 'q', long = "query", env = "BENCH_QUERY")]- pub query: Option<String>,-- #[arg(long = "subgroup", env = "BENCH_SUBGROUP")]- pub subgroup: Option<String>,-- #[command(flatten)]- pub common: CommonOpt,-- #[arg(- long = "criterion",- action = ArgAction::SetTrue,- help = "Run benchmarks with Criterion"- )]- pub criterion: bool,-- #[arg(- short = 'o',- long = "output",- help = "Write simple runner results as JSON to this path"- )]- pub output: Option<PathBuf>,-- #[arg(- long = "save-baseline",- value_name = "BASELINE",- help = "Save Criterion measurements to the named baseline"- )]- pub save_baseline: Option<String>,-}--/// Parses CLI arguments, runs the selected action, and prints any list output.-pub async fn run_cli() -> Result<()> {- let matches = Cli::command().get_matches();- let action = cli_action_from_matches(&matches)?;- let output = run_cli_action(action, &default_sql_benchmark_directory()).await?;-- if !output.is_empty() {- println!("{output}");- }-- Ok(())-}-/// Runs the selected SQL benchmarks through a caller-provided Criterion instance.pub fn run_criterion_benchmarks_impl(benchmark_dir: &Path,@@ -152,6 +85,51 @@ pub fn run_criterion_benchmarks_impl(Ok(())}+/// Runs one benchmark case inside Criterion and converts benchmark panics to errors.+fn run_criterion_benchmark(+ rt: &Runtime,+ ctx: &SessionContext,+ benchmark: &mut SqlBenchmark,+ config: &SqlRunConfig,+ group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>,+) -> Result<()> {+ rt.block_on(prepare_benchmark(ctx, benchmark, config))?;++ let name = criterion_function_name(benchmark);+ let result = catch_unwind(AssertUnwindSafe(|| {+ group.bench_function(name.clone(), |b| {+ b.iter(|| {+ let _ = rt.block_on(async {+ benchmark.run(ctx, false).await.unwrap_or_else(|err| {+ panic!("Failed to run benchmark {name}: {err:?}")+ })+ });+ });+ });+ }));++ match result {+ Ok(()) => {+ print_memory_stats();+ Ok(())+ }+ Err(payload) => Err(panic_payload_to_error(payload.as_ref())),+ }+}++/// Extracts a readable message from a panic payload.+fn panic_payload_to_error(payload: &(dyn Any + Send)) -> DataFusionError {+ let message = if let Some(message) = payload.downcast_ref::<String>() {+ message.as_str()+ } else if let Some(message) = payload.downcast_ref::<&str>() {+ message+ } else {+ "unknown panic"+ };++ exec_datafusion_err!("criterion benchmark failed: {message}")+}+pub fn default_sql_benchmark_directory() -> PathBuf {PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("sql_benchmarks")}@@ -163,7 +141,7 @@ fn make_tokio_runtime() -> Result<Runtime> {.map_err(|e| DataFusionError::External(Box::new(e)))}-fn make_ctx(common: &CommonOpt) -> Result<SessionContext> {+pub fn make_ctx(common: &CommonOpt) -> Result<SessionContext> {let config = common.config()?;let rt = common.build_runtime()?;@@ -180,22 +158,8 @@ fn discover_benchmark_paths(path: &Path) -> Result<Vec<PathBuf>> {Ok(paths)}-/// Loads benchmark definitions, applies CLI-style filters, and sorts each group.-async fn load_benchmarks(- filter: &BenchmarkFilter,- ctx: &SessionContext,- benchmark_dir: &Path,-) -> Result<BTreeMap<String, Vec<SqlBenchmark>>> {- let benches = load_benchmark_definitions(filter, ctx, benchmark_dir).await?;- let mut benches = filter_benchmarks(filter, benches);-- sort_benchmarks(&mut benches);-- Ok(benches)-}-/// Loads all benchmark definitions with replacements derived from the filter.-async fn load_benchmark_definitions(+pub async fn load_benchmark_definitions(filter: &BenchmarkFilter,ctx: &SessionContext,benchmark_dir: &Path,@@ -225,6 +189,14 @@ async fn load_benchmark_definitions(/// Builds template replacements from CLI values that also appear in benchmark files.fn benchmark_replacements(filter: &BenchmarkFilter) -> HashMap<String, String> {let mut replacements = HashMap::new();+ let data_dir = std::env::var("DATA_DIR").unwrap_or_else(|_| {+ PathBuf::from(env!("CARGO_MANIFEST_DIR"))+ .join("data")+ .to_string_lossy()+ .into_owned()+ });++ replacements.insert("data_dir".to_string(), data_dir);if let Some(subgroup) = &filter.subgroup {replacements.insert("bench_subgroup".to_string(), subgroup.to_string());@@ -233,14 +205,14 @@ fn benchmark_replacements(filter: &BenchmarkFilter) -> HashMap<String, String> {replacements}-fn sort_benchmarks(benchmarks: &mut BTreeMap<String, Vec<SqlBenchmark>>) {+pub fn sort_benchmarks(benchmarks: &mut BTreeMap<String, Vec<SqlBenchmark>>) {benchmarks.values_mut().for_each(|benchmarks| benchmarks.sort_by(|a, b| a.name().cmp(b.name())));}/// Applies benchmark, subgroup, and query filters to discovered benchmark groups.-fn filter_benchmarks(+pub fn filter_benchmarks(filter: &BenchmarkFilter,benchmarks: BTreeMap<String, Vec<SqlBenchmark>>,) -> BTreeMap<String, Vec<SqlBenchmark>> {@@ -344,7 +316,7 @@ fn normalize_query(query: &str) -> String {format!("Q{number:0>2}{suffix}")}-fn format_benchmark_list(benchmarks: &BTreeMap<String, Vec<SqlBenchmark>>) -> String {+pub fn format_benchmark_list(benchmarks: &BTreeMap<String, Vec<SqlBenchmark>>) -> String {let mut output = String::from("SQL benchmarks:\n");for (name, benchmarks) in benchmarks {@@ -359,121 +331,6 @@ fn format_benchmark_list(benchmarks: &BTreeMap<String, Vec<SqlBenchmark>>) -> Stoutput.trim_end().to_string()}-/// Runs selected benchmarks with fixed iteration counts and optional JSON output.-async fn run_simple_benchmarks(benchmark_dir: &Path, config: SqlRunConfig) -> Result<()> {- if config.common.iterations == 0 {- return Err(exec_datafusion_err!("iterations must be greater than zero"));- }-- let listing_ctx = make_ctx(&config.common)?;- let all_benchmarks =- load_benchmark_definitions(&config.filter, &listing_ctx, benchmark_dir).await?;- let selected = filter_benchmarks(&config.filter, all_benchmarks.clone());- let mut run = BenchmarkRun::new();-- ensure_selection(&config.filter, &all_benchmarks, &selected)?;-- for (_group, benchmarks) in selected {- for mut benchmark in benchmarks {- let ctx = make_ctx(&config.common)?;- let result =- run_simple_benchmark(&ctx, &mut benchmark, &config, &mut run).await;- let cleanup_result = benchmark.cleanup(&ctx).await;-- finish_benchmark(result, cleanup_result)?;- }- }-- run.maybe_write_json(config.output.as_ref())?;-- Ok(())-}--/// Builds the default Criterion runner and optionally records a named baseline.-fn run_criterion_benchmarks(- benchmark_dir: &Path,- config: &SqlRunConfig,- save_baseline: Option<&str>,-) -> Result<()> {- let mut criterion = Criterion::default()- .sample_size(10)- .with_output_color(std::io::stdout().is_terminal());-- if let Some(save_baseline) = save_baseline {- criterion = criterion.save_baseline(save_baseline.to_string());- }-- run_criterion_benchmarks_impl(benchmark_dir, config, &mut criterion)?;- criterion.final_summary();-- Ok(())-}--/// Converts parsed arguments into an executable action and validates mode options.-fn cli_action_from_matches(matches: &ArgMatches) -> Result<CliAction> {- let cli = Cli::from_arg_matches(matches)- .map_err(|e| DataFusionError::External(Box::new(e)))?;-- if cli.benchmark.is_none() {- return Ok(CliAction::List);- }-- if cli.criterion && cli.output.is_some() {- return Err(exec_datafusion_err!(- "--output cannot be used with --criterion"- ));- }- if !cli.criterion && cli.save_baseline.is_some() {- return Err(exec_datafusion_err!(- "--save-baseline cannot be used without --criterion"- ));- }-- // we need to know if iterations was set on the command line, not the default value- let iterations_from_cli = matches.value_source("iterations")- == Some(clap::parser::ValueSource::CommandLine);-- if cli.criterion && iterations_from_cli {- return Err(exec_datafusion_err!(- "--iterations cannot be used with --criterion"- ));- }- if !cli.criterion && cli.common.iterations == 0 {- return Err(exec_datafusion_err!("iterations must be greater than zero"));- }-- let config = SqlRunConfig {- common: cli.common,- filter: BenchmarkFilter {- name: cli.benchmark,- subgroup: cli.subgroup,- query: cli.query,- },- persist_results: false,- validate_results: false,- output: cli.output,- };-- if cli.criterion {- Ok(CliAction::Criterion {- config,- save_baseline: cli.save_baseline,- })- } else {- Ok(CliAction::Simple(config))- }-}--fn criterion_like_styles() -> clap::builder::Styles {- use clap::builder::styling::AnsiColor;-- clap::builder::Styles::styled()- .header(AnsiColor::Green.on_default().bold())- .usage(AnsiColor::Green.on_default().bold())- .literal(AnsiColor::Cyan.on_default().bold())- .placeholder(AnsiColor::Cyan.on_default())-}-/// Recursively collects `.benchmark` files below `path`.fn collect_benchmark_paths(path: &Path, paths: &mut Vec<PathBuf>) -> Result<()> {let mut entries = fs::read_dir(path)?@@ -494,7 +351,7 @@ fn collect_benchmark_paths(path: &Path, paths: &mut Vec<PathBuf>) -> Result<()>Ok(())}-fn unknown_benchmark_error(+pub fn unknown_benchmark_error(requested: &str,benchmarks: &BTreeMap<String, Vec<SqlBenchmark>>,) -> DataFusionError {@@ -603,37 +460,8 @@ fn format_query_list(output.trim_end().to_string()}-/// Runs one benchmark case, recording each timed iteration.-async fn run_simple_benchmark(- ctx: &SessionContext,- benchmark: &mut SqlBenchmark,- config: &SqlRunConfig,- run: &mut BenchmarkRun,-) -> Result<()> {- prepare_benchmark(ctx, benchmark, config).await?;-- let case_name = benchmark_case_name(benchmark);-- run.start_new_case(&case_name);-- for iteration in 0..config.common.iterations {- let start = Instant::now();- let row_count = benchmark.run(ctx, false).await?;- let elapsed = start.elapsed();- let ms = elapsed.as_secs_f64() * 1000.0;-- println!("{case_name} iteration {iteration}: {ms:.1} ms, {row_count} rows");-- run.write_iter(elapsed, row_count);- }-- print_memory_stats();-- Ok(())-}-/// Initializes a benchmark and performs any configured assertion or validation step.-async fn prepare_benchmark(+pub async fn prepare_benchmark(ctx: &SessionContext,benchmark: &mut SqlBenchmark,config: &SqlRunConfig,@@ -652,7 +480,7 @@ async fn prepare_benchmark(}/// Ensures filtering selected at least one benchmark and emits targeted errors.-fn ensure_selection(+pub fn ensure_selection(filter: &BenchmarkFilter,all_benchmarks: &BTreeMap<String, Vec<SqlBenchmark>>,selected: &BTreeMap<String, Vec<SqlBenchmark>>,@@ -694,19 +522,8 @@ fn ensure_selection(Ok(())}-fn benchmark_case_name(benchmark: &SqlBenchmark) -> String {- let mut name = format!("{}/{}", benchmark.group(), benchmark.name());-- if !benchmark.subgroup().is_empty() {- name.push('/');- name.push_str(benchmark.subgroup());- }-- name-}-/// Combines benchmark and cleanup results without hiding cleanup failures.-fn finish_benchmark(result: Result<()>, cleanup_result: Result<()>) -> Result<()> {+pub fn finish_benchmark(result: Result<()>, cleanup_result: Result<()>) -> Result<()> {match (result, cleanup_result) {(Ok(()), Ok(())) => Ok(()),(Ok(()), Err(cleanup_error)) => Err(cleanup_error),@@ -717,38 +534,6 @@ fn finish_benchmark(result: Result<()>, cleanup_result: Result<()>) -> Result<()}}-/// Runs one benchmark case inside Criterion and converts benchmark panics to errors.-fn run_criterion_benchmark(- rt: &Runtime,- ctx: &SessionContext,- benchmark: &mut SqlBenchmark,- config: &SqlRunConfig,- group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>,-) -> Result<()> {- rt.block_on(prepare_benchmark(ctx, benchmark, config))?;-- let name = criterion_function_name(benchmark);- let result = catch_unwind(AssertUnwindSafe(|| {- group.bench_function(name.clone(), |b| {- b.iter(|| {- let _ = rt.block_on(async {- benchmark.run(ctx, false).await.unwrap_or_else(|err| {- panic!("Failed to run benchmark {name}: {err:?}")- })- });- });- });- }));-- match result {- Ok(()) => {- print_memory_stats();- Ok(())- }- Err(payload) => Err(panic_payload_to_error(payload.as_ref())),- }-}-fn criterion_function_name(benchmark: &SqlBenchmark) -> String {let mut name = benchmark.name().to_string();@@ -760,63 +545,9 @@ fn criterion_function_name(benchmark: &SqlBenchmark) -> String {name}-/// Extracts a readable message from a panic payload.-fn panic_payload_to_error(payload: &(dyn Any + Send)) -> DataFusionError {- let message = if let Some(message) = payload.downcast_ref::<String>() {- message.as_str()- } else if let Some(message) = payload.downcast_ref::<&str>() {- message- } else {- "unknown panic"- };-- exec_datafusion_err!("criterion benchmark failed: {message}")-}--/// Executes a parsed CLI action and returns any text that should be printed.-async fn run_cli_action(action: CliAction, benchmark_dir: &Path) -> Result<String> {- match action {- CliAction::List => {- let ctx = SessionContext::new();- let benchmarks =- load_benchmarks(&BenchmarkFilter::default(), &ctx, benchmark_dir).await?;-- Ok(format_benchmark_list(&benchmarks))- }- CliAction::Simple(config) => {- run_simple_benchmarks(benchmark_dir, config).await?;- Ok(String::new())- }- CliAction::Criterion {- config,- save_baseline,- } => {- if config.output.is_some() {- return Err(exec_datafusion_err!(- "--output cannot be used with --criterion"- ));- }- let benchmark_dir = benchmark_dir.to_path_buf();-- SpawnedTask::spawn_blocking(move || {- run_criterion_benchmarks(- &benchmark_dir,- &config,- save_baseline.as_deref(),- )- })- .await- .map_err(|e| DataFusionError::External(Box::new(e)))??;-- Ok(String::new())- }- }-}-#[cfg(test)]mod tests {use super::*;- use criterion::Criterion;use datafusion::prelude::SessionContext;use std::path::{Path, PathBuf};@@ -829,301 +560,6 @@ mod tests {path}- fn common(iterations: usize) -> CommonOpt {- CommonOpt {- iterations,- partitions: None,- batch_size: None,- mem_pool_type: "fair".to_string(),- memory_limit: None,- sort_spill_reservation_bytes: None,- debug: false,- simulate_latency: false,- }- }-- fn parse_cli_from<I, T>(args: I) -> Result<CliAction>- where- I: IntoIterator<Item = T>,- T: Into<std::ffi::OsString> + Clone,- {- let matches = Cli::command()- .try_get_matches_from(args)- .map_err(|e| DataFusionError::External(Box::new(e)))?;-- cli_action_from_matches(&matches)- }-- async fn run_cli_with_dir<I, T>(args: I, benchmark_dir: &Path) -> Result<String>- where- I: IntoIterator<Item = T>,- T: Into<std::ffi::OsString> + Clone,- {- run_cli_action(parse_cli_from(args)?, benchmark_dir).await- }-- #[test]- fn cli_lists_when_benchmark_is_omitted() {- let action = parse_cli_from(["benchmark_runner"]).unwrap();-- assert!(matches!(action, CliAction::List));- }-- #[test]- fn cli_defaults_to_basic_runner() {- let action =- parse_cli_from(["benchmark_runner", "tpch", "--query", "1"]).unwrap();- let CliAction::Simple(config) = action else {- panic!("expected basic runner");- };-- assert_eq!(config.filter.name.as_deref(), Some("tpch"));- assert_eq!(config.filter.query.as_deref(), Some("1"));- }-- #[test]- fn cli_reads_query_from_env() {- let previous = std::env::var_os("BENCH_QUERY");- // SAFETY: This test restores BENCH_QUERY before returning and does not- // spawn threads while the environment variable is overridden.- unsafe {- std::env::set_var("BENCH_QUERY", "8");- }-- let action = parse_cli_from(["benchmark_runner", "tpch"]);-- unsafe {- match previous {- Some(value) => std::env::set_var("BENCH_QUERY", value),- None => std::env::remove_var("BENCH_QUERY"),- }- }-- let action = action.unwrap();- let CliAction::Simple(config) = action else {- panic!("expected basic runner");- };-- assert_eq!(config.filter.name.as_deref(), Some("tpch"));- assert_eq!(config.filter.query.as_deref(), Some("8"));- }-- #[test]- fn cli_accepts_criterion_runner() {- let action = parse_cli_from([- "benchmark_runner",- "tpch",- "--criterion",- "--save-baseline",- "main",- ])- .unwrap();-- let CliAction::Criterion {- config,- save_baseline,- } = action- else {- panic!("expected criterion runner");- };-- assert_eq!(config.filter.name.as_deref(), Some("tpch"));- assert_eq!(save_baseline.as_deref(), Some("main"));- }-- #[test]- fn cli_rejects_output_with_criterion() {- let err = parse_cli_from([- "benchmark_runner",- "tpch",- "--criterion",- "--output",- "results.json",- ])- .unwrap_err();-- assert!(err.to_string().contains("--output"));- assert!(err.to_string().contains("--criterion"));- }-- #[test]- fn cli_rejects_save_baseline_without_criterion() {- let err = parse_cli_from(["benchmark_runner", "tpch", "--save-baseline", "main"])- .unwrap_err();-- assert!(err.to_string().contains("--save-baseline"));- assert!(err.to_string().contains("--criterion"));- }-- #[test]- fn cli_rejects_iterations_with_criterion() {- let err = parse_cli_from([- "benchmark_runner",- "tpch",- "--criterion",- "--iterations",- "3",- ])- .unwrap_err();-- assert!(err.to_string().contains("--iterations"));- assert!(err.to_string().contains("--criterion"));- }-- #[test]- fn cli_rejects_zero_basic_iterations() {- let err = parse_cli_from(["benchmark_runner", "tpch", "--iterations", "0"])- .unwrap_err();-- assert!(err.to_string().contains("iterations"));- }-- #[tokio::test]- async fn discovery_lists_groups_from_directories() {- let temp = tempfile::tempdir().unwrap();- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );- write_benchmark(- temp.path(),- "beta/benchmarks/q02.benchmark",- "name Q02\n\nrun\nSELECT 2\n",- );- let ctx = SessionContext::new();- let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())- .await- .unwrap();-- assert_eq!(benches["alpha"].len(), 1);- assert_eq!(benches["beta"].len(), 1);- }-- #[tokio::test]- async fn discovery_filters_benchmark_subgroup_and_query() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\nsubgroup wide\n\nrun\nSELECT 1\n",- );- write_benchmark(- temp.path(),- "alpha/benchmarks/q02.benchmark",- "name Q02\nsubgroup narrow\n\nrun\nSELECT 2\n",- );-- let ctx = SessionContext::new();- let filter = BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: Some("wide".to_string()),- query: Some("1".to_string()),- };- let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();-- assert_eq!(benches.len(), 1);- assert_eq!(benches["alpha"].len(), 1);- assert_eq!(benches["alpha"][0].name(), "Q01");- }-- #[tokio::test]- async fn cli_subgroup_filter_is_used_for_benchmark_replacements() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "wide_schema/benchmarks/q01.benchmark",- "name Q01\nsubgroup ${BENCH_SUBGROUP:-wide}\n\nrun\nSELECT '${BENCH_SUBGROUP:-wide}'\n",- );-- let ctx = SessionContext::new();- let filter = BenchmarkFilter {- name: Some("wide_schema".to_string()),- subgroup: Some("narrow".to_string()),- query: None,- };- let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();-- assert_eq!(benches["wide_schema"].len(), 1);- assert_eq!(benches["wide_schema"][0].subgroup(), "narrow");- }-- #[tokio::test]- async fn query_filter_matches_starts_with_when_exact_match_is_absent() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01a.benchmark",- "name Q01a\n\nrun\nSELECT 1\n",- );-- let ctx = SessionContext::new();- let filter = BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: None,- query: Some("1".to_string()),- };- let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();-- assert_eq!(benches["alpha"].len(), 1);- assert_eq!(benches["alpha"][0].name(), "Q01a");- }-- #[tokio::test]- async fn query_filter_matches_token_start_when_exact_match_is_absent() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "predicate_eval/benchmarks/costsel/q01.benchmark",- "name costsel_q01_regexp_selective_last\n\nrun\nSELECT 1\n",- );-- let ctx = SessionContext::new();- let filter = BenchmarkFilter {- name: Some("predicate_eval".to_string()),- subgroup: None,- query: Some("1".to_string()),- };- let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();-- assert_eq!(benches["predicate_eval"].len(), 1);- assert_eq!(- benches["predicate_eval"][0].name(),- "costsel_q01_regexp_selective_last"- );- }-- #[tokio::test]- async fn query_filter_prefers_starts_with_match_over_token_match() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/token.benchmark",- "name costsel_q01_regexp_selective_last\n\nrun\nSELECT 1\n",- );- write_benchmark(- temp.path(),- "alpha/benchmarks/q01a.benchmark",- "name Q01a\n\nrun\nSELECT 2\n",- );-- let ctx = SessionContext::new();- let filter = BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: None,- query: Some("1".to_string()),- };- let benches = load_benchmarks(&filter, &ctx, temp.path()).await.unwrap();-- assert_eq!(benches["alpha"].len(), 1);- assert_eq!(benches["alpha"][0].name(), "Q01a");- }-#[test]fn normalizes_query_like_existing_sql_harness() {assert_eq!(normalize_query("1"), "Q01");@@ -1132,191 +568,6 @@ mod tests {assert_eq!(normalize_query("Q06a"), "Q06a");}- #[tokio::test]- async fn list_output_is_sorted_and_includes_counts() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "beta/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );- write_benchmark(- temp.path(),- "alpha/benchmarks/q02.benchmark",- "name Q02\n\nrun\nSELECT 2\n",- );-- let ctx = SessionContext::new();- let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())- .await- .unwrap();- let output = format_benchmark_list(&benches);-- assert!(output.starts_with("SQL benchmarks:\n alpha"));- assert!(output.contains("alpha 2 queries"));- assert!(output.contains("beta 1 query"));- }-- #[tokio::test]- async fn unknown_benchmark_error_includes_available_benchmarks() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );- let ctx = SessionContext::new();- let benches = load_benchmarks(&BenchmarkFilter::default(), &ctx, temp.path())- .await- .unwrap();- let message = unknown_benchmark_error("missing", &benches).to_string();-- assert!(message.contains("unknown benchmark 'missing'"), "{message}");- assert!(message.contains("alpha"), "{message}");- }-- #[tokio::test]- async fn run_cli_reports_unknown_query_for_known_benchmark() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );-- let err = run_cli_with_dir(- [- "benchmark_runner",- "alpha",- "--query",- "9",- "--iterations",- "1",- ],- temp.path(),- )- .await- .unwrap_err();- let message = err.to_string();-- assert!(- message.contains("no SQL benchmark query matched benchmark 'alpha'"),- "{message}"- );- assert!(message.contains("query '9'"), "{message}");- assert!(message.contains("normalized: 'Q09'"), "{message}");- assert!(message.contains("Available alpha queries:"), "{message}");- assert!(message.contains("Q01"), "{message}");- assert!(!message.contains("unknown benchmark 'alpha'"), "{message}");- }-- #[tokio::test]- async fn run_cli_reports_unknown_subgroup_for_known_benchmark() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\nsubgroup wide\n\nrun\nSELECT 1\n",- );-- let err = run_cli_with_dir(- [- "benchmark_runner",- "alpha",- "--subgroup",- "narrow",- "--iterations",- "1",- ],- temp.path(),- )- .await- .unwrap_err();- let message = err.to_string();-- assert!(- message.contains(- "no SQL benchmark subgroup matched benchmark 'alpha' with subgroup 'narrow'"- ),- "{message}"- );- assert!(message.contains("Available alpha subgroups:"), "{message}");- assert!(message.contains("wide"), "{message}");- assert!(!message.contains("unknown benchmark 'alpha'"), "{message}");- }-- #[tokio::test]- async fn basic_runner_executes_iterations_and_writes_json() {- let temp = tempfile::tempdir().unwrap();- let output = temp.path().join("results.json");-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT * FROM (VALUES (1), (2)) AS t(v)\n",- );-- let config = SqlRunConfig {- common: common(2),- filter: BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: None,- query: Some("1".to_string()),- },- persist_results: false,- validate_results: false,- output: Some(output.clone()),- };-- run_simple_benchmarks(temp.path(), config).await.unwrap();-- let json = fs::read_to_string(output).unwrap();-- assert!(json.contains("\"query\": \"alpha/Q01\""));- assert!(json.contains("\"row_count\": 2"));- assert_eq!(json.matches("\"row_count\": 2").count(), 2);- }-- #[tokio::test]- async fn basic_runner_reports_run_and_cleanup_failures() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT * FROM missing_run_table\n\ncleanup\nSELECT * FROM missing_cleanup_table\n",- );-- let config = SqlRunConfig {- common: common(1),- filter: BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: None,- query: Some("1".to_string()),- },- persist_results: false,- validate_results: false,- output: None,- };- let err = run_simple_benchmarks(temp.path(), config)- .await- .unwrap_err();- let message = err.to_string();-- assert!(message.contains("missing_run_table"), "{message}");- assert!(message.contains("cleanup also failed"), "{message}");- assert!(message.contains("missing_cleanup_table"), "{message}");- }-#[test]fn criterion_names_match_existing_sql_harness() {let temp = tempfile::tempdir().unwrap();@@ -1334,85 +585,4 @@ mod tests {assert_eq!(benchmark.group(), "tpch");assert_eq!(criterion_function_name(&benchmark), "Q01_sf1");}-- #[test]- fn criterion_runner_saves_named_baseline() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );-- let output = tempfile::tempdir().unwrap();- let mut criterion = Criterion::default()- .sample_size(10)- .warm_up_time(std::time::Duration::from_millis(1))- .measurement_time(std::time::Duration::from_millis(10))- .without_plots()- .output_directory(output.path())- .save_baseline("acceptance".to_string());- let config = SqlRunConfig {- common: common(3),- filter: BenchmarkFilter {- name: Some("alpha".to_string()),- subgroup: None,- query: Some("1".to_string()),- },- persist_results: false,- validate_results: false,- output: None,- };-- run_criterion_benchmarks_impl(temp.path(), &config, &mut criterion).unwrap();- criterion.final_summary();-- assert!(- output- .path()- .join("alpha")- .join("Q01")- .join("acceptance")- .join("estimates.json")- .exists()- );- }-- #[tokio::test]- async fn run_cli_lists_when_no_benchmark_is_supplied() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );-- let output = run_cli_with_dir(["benchmark_runner"], temp.path())- .await- .unwrap();-- assert!(output.contains("SQL benchmarks:"));- assert!(output.contains("alpha"));- }-- #[tokio::test]- async fn run_cli_reports_unknown_benchmark_with_list() {- let temp = tempfile::tempdir().unwrap();-- write_benchmark(- temp.path(),- "alpha/benchmarks/q01.benchmark",- "name Q01\n\nrun\nSELECT 1\n",- );-- let err = run_cli_with_dir(["benchmark_runner", "missing"], temp.path())- .await- .unwrap_err();- let message = err.to_string();-- assert!(message.contains("unknown benchmark 'missing'"), "{message}");- assert!(message.contains("alpha"), "{message}");- }}