Apache DataFusion SQL Query Engine
Add SQL based benchmarking harness, port tpch to use framework (#21707)
Bruce Ritchie2b95cde31b80
diff --git a/Cargo.lock b/Cargo.lockindex 9177551851..c18fd20128 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1740,6 +1740,7 @@ dependencies = ["async-trait","bytes","clap",+ "criterion","datafusion","datafusion-common","datafusion-proto",@@ -1755,6 +1756,7 @@ dependencies = ["serde","serde_json","snmalloc-rs",+ "tempfile","tokio","tokio-util",]@@ -4866,7 +4868,7 @@ dependencies = ["once_cell","socket2","tracing",- "windows-sys 0.60.2",+ "windows-sys 0.52.0",][[package]]diff --git a/benchmarks/.gitignore b/benchmarks/.gitignoreindex c35b1a7c19..e3de51f76e 100644--- a/benchmarks/.gitignore+++ b/benchmarks/.gitignore@@ -1,3 +1,5 @@data-results+data_csv+./results/venv+!sql_benchmarks/**/results/diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.tomlindex 107bb0b76e..1815f8bc42 100644--- a/benchmarks/Cargo.toml+++ b/benchmarks/Cargo.toml@@ -43,7 +43,8 @@ mimalloc_extended = ["libmimalloc-sys/extended"]arrow = { workspace = true }async-trait = "0.1"bytes = { workspace = true }-clap = { version = "4.5.60", features = ["derive"] }+clap = { version = "4.6.0", features = ["derive", "env"] }+criterion = { workspace = true, features = ["html_reports"] }datafusion = { workspace = true, default-features = true }datafusion-common = { workspace = true, default-features = true }env_logger = { workspace = true }@@ -63,3 +64,8 @@ tokio-util = { version = "0.7.17" }[dev-dependencies]datafusion-proto = { workspace = true }+tempfile = { workspace = true }++[[bench]]+harness = false+name = "sql"diff --git a/benchmarks/bench.sh b/benchmarks/bench.shindex 6c5e0e483e..7aa0418e1d 100755--- a/benchmarks/bench.sh+++ b/benchmarks/bench.sh@@ -41,6 +41,7 @@ BENCHMARK=allDATAFUSION_DIR=${DATAFUSION_DIR:-$SCRIPT_DIR/..}DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data}CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"}+SQL_CARGO_COMMAND=${SQL_CARGO_COMMAND:-"cargo bench --bench sql"}PREFER_HASH_JOIN=${PREFER_HASH_JOIN:-true}SIMULATE_LATENCY=${SIMULATE_LATENCY:-false}@@ -685,14 +686,16 @@ run_tpch() {echo "Internal error: Scale factor not specified"exit 1fi- TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"-- RESULTS_FILE="${RESULTS_DIR}/tpch_sf${SCALE_FACTOR}.json"- echo "RESULTS_FILE: ${RESULTS_FILE}"+ FORMAT=$2echo "Running tpch benchmark..."- FORMAT=$2- debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --scale-factor "${SCALE_FACTOR}" --prefer_hash_join "${PREFER_HASH_JOIN}" --format ${FORMAT} -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}+ debug_run env BENCH_NAME=tpch \+ BENCH_SIZE="${SCALE_FACTOR}" \+ PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \+ TPCH_FILE_TYPE="${FORMAT}" \+ SIMULATE_LATENCY="${SIMULATE_LATENCY}" \+ ${QUERY:+BENCH_QUERY="${QUERY}"} \+ bash -c "$SQL_CARGO_COMMAND"}# Runs the tpch in memory (needs tpch parquet data)@@ -702,13 +705,15 @@ run_tpch_mem() {echo "Internal error: Scale factor not specified"exit 1fi- TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}"-- RESULTS_FILE="${RESULTS_DIR}/tpch_mem_sf${SCALE_FACTOR}.json"- echo "RESULTS_FILE: ${RESULTS_FILE}"echo "Running tpch_mem benchmark..."- # -m means in memory- debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --scale-factor "${SCALE_FACTOR}" --prefer_hash_join "${PREFER_HASH_JOIN}" -m --format parquet -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}++ debug_run env BENCH_NAME=tpch \+ BENCH_SIZE="${SCALE_FACTOR}" \+ TPCH_FILE_TYPE="mem" \+ PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \+ SIMULATE_LATENCY="${SIMULATE_LATENCY}" \+ ${QUERY:+BENCH_QUERY="${QUERY}"} \+ bash -c "$SQL_CARGO_COMMAND"}# Runs the tpcds benchmarkdiff --git a/benchmarks/benches/sql.rs b/benchmarks/benches/sql.rsnew file mode 100644index 0000000000..eade3194d1--- /dev/null+++ b/benchmarks/benches/sql.rs@@ -0,0 +1,321 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Criterion benchmark harness for SQL benchmark files under `sql_benchmarks`.+//!+//! SQL benchmarks describe setup, queries, result validation, and cleanup in+//! `.benchmark` files. Run them with `benchmarks/bench.sh` or directly with+//! Cargo, for example: `BENCH_NAME=tpch cargo bench --bench sql`.++use clap::Parser;+use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};+use datafusion::error::Result;+use datafusion::prelude::SessionContext;+use datafusion_benchmarks::sql_benchmark::SqlBenchmark;+use datafusion_benchmarks::util::{CommonOpt, print_memory_stats};+use datafusion_common::instant::Instant;+use log::{debug, info};+use std::collections::BTreeMap;+use std::fs;+use std::sync::LazyLock;+use tokio::runtime::Runtime;++static SQL_BENCHMARK_DIRECTORY: LazyLock<String> = LazyLock::new(|| {+ format!(+ "{}{}{}",+ env!("CARGO_MANIFEST_DIR"),+ std::path::MAIN_SEPARATOR,+ "sql_benchmarks"+ )+});++#[cfg(all(feature = "snmalloc", feature = "mimalloc"))]+compile_error!(+ "feature \"snmalloc\" and feature \"mimalloc\" cannot be enabled at the same time"+);++#[cfg(feature = "snmalloc")]+#[global_allocator]+static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;++#[cfg(feature = "mimalloc")]+#[global_allocator]+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;++#[derive(Debug, Parser)]+#[command(ignore_errors = true)]+struct EnvParser {+ #[command(flatten)]+ options: CommonOpt,++ #[arg(+ env = "BENCH_PERSIST_RESULTS",+ long = "persist_results",+ default_value = "false",+ action = clap::ArgAction::SetTrue+ )]+ persist_results: bool,++ #[arg(+ env = "BENCH_VALIDATE",+ long = "validate_results",+ default_value = "false",+ action = clap::ArgAction::SetTrue+ )]+ validate: bool,++ #[arg(env = "BENCH_NAME")]+ name: Option<String>,++ #[arg(env = "BENCH_SUBGROUP")]+ subgroup: Option<String>,++ #[arg(env = "BENCH_QUERY")]+ query: Option<i32>,+}++pub fn sql(c: &mut Criterion) {+ env_logger::init();++ let start = Instant::now();+ let args = EnvParser::parse();+ let rt = make_tokio_runtime();++ println!("Loading benchmarks...");++ let benchmarks = rt.block_on(async {+ let ctx = make_ctx(&args).expect("SessionContext creation failed");++ load_benchmarks(&args, &ctx, &SQL_BENCHMARK_DIRECTORY)+ .await+ .unwrap_or_else(|err| panic!("failed load benchmarks: {err:?}"))+ });++ println!(+ "Loaded benchmarks in {} ms ...",+ start.elapsed().as_millis()+ );++ for (group, benchmarks) in benchmarks {+ let mut group = c.benchmark_group(group);+ group.sample_size(10);+ group.sampling_mode(SamplingMode::Flat);++ for mut benchmark in benchmarks {+ // create a context+ let ctx = make_ctx(&args).expect("SessionContext creation failed");++ // initialize the benchmark. This parses the benchmark file and does any pre-execution+ // work such as loading data into tables+ rt.block_on(async {+ benchmark+ .initialize(&ctx)+ .await+ .expect("initialization failed");++ // run assertions+ benchmark.assert(&ctx).await.expect("assertion failed");+ });++ let mut name = benchmark.name().to_string();+ if !benchmark.subgroup().is_empty() {+ name.push('_');+ name.push_str(benchmark.subgroup());+ }++ if args.persist_results {+ handle_persist(&rt, &ctx, &name, &mut benchmark);+ } else if args.validate {+ handle_verify(&rt, &ctx, &name, &mut benchmark);+ } else {+ info!("Running benchmark {name} ...");++ let name = name.clone();+ group.bench_function(name.clone(), |b| {+ b.iter(|| handle_run(&rt, &ctx, &args, &mut benchmark, &name))+ });++ print_memory_stats();++ info!("Benchmark {name} completed");+ }++ // run cleanup+ rt.block_on(async {+ benchmark.cleanup(&ctx).await.expect("Cleanup failed");+ });+ }++ group.finish();+ }+}++fn handle_run(+ rt: &Runtime,+ ctx: &SessionContext,+ args: &EnvParser,+ benchmark: &mut SqlBenchmark,+ name: &str,+) {+ rt.block_on(async {+ benchmark+ .run(ctx, args.validate)+ .await+ .unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"))+ });+}++fn handle_persist(+ rt: &Runtime,+ ctx: &SessionContext,+ name: &str,+ benchmark: &mut SqlBenchmark,+) {+ info!("Running benchmark {name} prior to persisting results ...");++ rt.block_on(async {+ info!("Persisting benchmark {name} ...");++ benchmark+ .persist(ctx)+ .await+ .expect("Failed to persist results");+ });++ info!("Persisted benchmark {name} successfully");+}++fn handle_verify(+ rt: &Runtime,+ ctx: &SessionContext,+ name: &str,+ benchmark: &mut SqlBenchmark,+) {+ info!("Verifying benchmark {name} results ...");++ rt.block_on(async {+ benchmark+ .run(ctx, true)+ .await+ .unwrap_or_else(|err| panic!("Failed to run benchmark {name}: {err:?}"));+ benchmark+ .verify(ctx)+ .await+ .unwrap_or_else(|err| panic!("Verification failed: {err:?}"));+ });++ info!("Verified benchmark {name} results successfully");+}++criterion_group!(benches, sql);+criterion_main!(benches);++fn make_tokio_runtime() -> Runtime {+ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()+ .unwrap()+}++fn make_ctx(args: &EnvParser) -> Result<SessionContext> {+ let config = args.options.config()?;+ let rt = args.options.build_runtime()?;++ Ok(SessionContext::new_with_config_rt(config, rt))+}++/// Recursively walks the directory tree starting at `path` and+/// calls the call back function for every file encountered.+pub fn list_files<F>(path: &str, callback: &mut F)+where+ F: FnMut(&str),+{+ let mut entries: Vec<fs::DirEntry> =+ fs::read_dir(path).unwrap().filter_map(Result::ok).collect();+ entries.sort_by_key(|entry| entry.path());++ for dir_entry in entries {+ let path = dir_entry.path();+ if path.is_dir() {+ // Recurse into the sub‑directory+ list_files(&path.to_string_lossy(), callback);+ } else {+ // For files, invoke the callback with the full path as a string+ let full_str = path.to_string_lossy();+ callback(&full_str);+ }+ }+}++/// Loads all benchmark files in the `sql_benchmarks` directory.+/// For each file ending with `.benchmark` it creates a new+/// `SqlBenchmark` instance.+async fn load_benchmarks(+ args: &EnvParser,+ ctx: &SessionContext,+ path: &str,+) -> Result<BTreeMap<String, Vec<SqlBenchmark>>> {+ let mut benches = BTreeMap::new();+ let mut paths = Vec::new();++ list_files(path, &mut |path: &str| {+ if path.ends_with(".benchmark") {+ paths.push(path.to_string());+ }+ });++ for path in paths {+ debug!("Loading benchmark from {path}");++ let benchmark = SqlBenchmark::new(ctx, &path, &*SQL_BENCHMARK_DIRECTORY).await?;+ let entries = benches+ .entry(benchmark.group().to_string())+ .or_insert(vec![]);++ entries.push(benchmark);+ }++ benches = filter_benchmarks(args, benches);+ benches.iter_mut().for_each(|(_, benchmarks)| {+ benchmarks.sort_by(|b1, b2| b1.name().cmp(b2.name()))+ });++ Ok(benches)+}++fn filter_benchmarks(+ args: &EnvParser,+ benchmarks: BTreeMap<String, Vec<SqlBenchmark>>,+) -> BTreeMap<String, Vec<SqlBenchmark>> {+ match &args.name {+ Some(bench_name) => benchmarks+ .into_iter()+ .filter(|(key, _val)| key.eq_ignore_ascii_case(bench_name))+ .map(|(key, mut val)| {+ if let Some(subgroup) = &args.subgroup {+ val.retain(|bench| bench.subgroup().eq_ignore_ascii_case(subgroup));+ }+ if let Some(query_number) = &args.query {+ let padded = format!("Q{query_number:0>2}");+ val.retain(|bench| bench.name().eq_ignore_ascii_case(&padded));+ }+ (key, val)+ })+ .collect(),+ None => benchmarks,+ }+}diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.mdnew file mode 100644index 0000000000..e8899641c0--- /dev/null+++ b/benchmarks/sql_benchmarks/README.md@@ -0,0 +1,355 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# SQL Benchmarks++This directory contains a collection of benchmarks each driven by a simple '.benchmark' text file and sql queries+that exercise the DataFusion execution engine against a variety of benchmark suites. The sql benchmark framework+is intentionally simple so that benchmarks and queries can be added or modified without touching the core+engine or requiring recompilation.++The sql benchmarks are organized in sub‑directories that correspond to the benchmark suites that are commonly used+in the community:++| Benchmark Suite | Description |+|-----------------------|--------------------------------------------------------------------|+| `clickbench` | ClickBench benchmark |+| `clickbench extended` | 12 additional, more complex queries against the Clickbench dataset |+| `clickbench_sorted` | ClickBench benchmark using a pre-sorted hits file. |+| `h2o` | The `h2o` benchmark |+| `hj` | Hash join benchmark |+| `imdb` | IMDb benchmark |+| `nlj` | Nested‑loop join benchmark |+| `smj` | Sort‑merge join benchmark |+| `sort tpch` | Sorting benchmarks against the TPC-H lineitem table |+| `taxi` | NYC taxi dataset benchmark |+| `tpcds` | TPC‑DS queries |+| `tpch` | TPC‑H queries |++# Running Benchmarks++The easiest way to run a benchmark is to use the `bench.sh` shell script (up one level from this document)+as it takes care of configuring any required environment variables and can populate any required data files.+However, it is possible to directly run a sql benchmark using the `cargo bench` command. For example:++```shell+BENCH_NAME=tpch cargo bench --bench sql+```++# Benchmark configuration++Sql benchmarks are configured via environment variables. Cargo's bench command and+[criterion](https://github.com/criterion-rs/criterion.rs) (the underlying benchmark framework) have an unfortunate+limitation in that custom command arguments cannot be passed into a benchmark. The alternative is to use environment+variables to pass in arguments which is what is used here.++The SQL benchmarking tool uses the following environment variables:++| Environment Variable | Description |+|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|+| BENCH_NAME | The name of the benchmark suite to run. For example 'imdb'. This should correspond to a directory name in the `sql_benchmarks` directory. |+| BENCH_SUBGROUP | The subgroup with the benchmark suite to run. For example 'window' to run the window subgroup of the h2o benchmark. |+| BENCH_QUERY | A query number to run. |+| BENCH_PERSIST_RESULTS | true/false to persist benchmark results. Results will be persisted in csv format so be cognizant of the size of the results. |+| BENCH_VALIDATE | true/false to validate benchmark results against persisted results or result_query's. If both `BENCH_PERSIST_RESULTS` and `BENCH_VALIDATE` are true, persist mode runs and validation is skipped. |+| SIMULATE_LATENCY | Simulate object store latency to mimic remote storage (e.g. S3). Adds random latency in the range 20-200ms to each object store operation. |+| MEM_POOL_TYPE | The memory pool type to use, should be one of "fair" or "greedy". |+| MEMORY_LIMIT | Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query if there's any, otherwise run with no memory limit. | |++Example – Run the H2O window benchmarks on the 'small' sized CSV data files:++``` bash+BENCH_NAME=h2o BENCH_SUBGROUP=window H2O_BENCH_SIZE=small H20_FILE_TYPE=csv cargo bench --bench sql+```++Some benchmarks use custom environment variables as outlined below:++| Name | Description | Default value |+|------------------------------|--------------------------------------------------------------------------------------------------------------------------|---------------|+| BENCH_SIZE | Used in the tpch, sort-tpch and tpcds benchmarks. The size corresponds to the scale factor. | `1` |+| TPCH_FILE_TYPE | Used in the tpch benchmark to specify which file type to query against. The valid options are `csv`, `parquet` and `mem` | `parquet` |+| H2O_FILE_TYPE | Used in the h2o benchmark to specify which file type to query against. The valid options are `csv` and `parquet` | `csv` |+| CLICKBENCH_TYPE | The type of partitioning for the clickbench benchmark. Valid options are `single` and `partitioned` | `single` |+| H2O_BENCH_SIZE | Used in the h2o benchmark. The valid options are `small`, `medium` and `big` | `small` |+| PREFER_HASH_JOIN | Control datafusion's config option `datafusion.optimizer.prefer_hash_join` | true |+| HASH_JOIN_BUFFERING_CAPACITY | Control datafusion's config option `datafusion.execution.hash_join_buffering_capacity` | 0 |+| BENCH_SORTED | Used in the sort_tpch benchmark to indicate whether the lineitem table should be sorted. | false |+| SORTED_BY | Used in the clickbench_sorted benchmark to indicate the column to sort by. | `EventTime` |+| SORTED_ORDER | Used in the clickbench_sorted benchmark to indicate the sort order of the column. | `ASC` |++## How it works++SQL benchmarks are run via cargo's bench command using [criterion](https://docs.rs/criterion/latest/criterion/)+for running and gathering statistics of each sql being benchmarked.++Each individual benchmark is represented by a `<name>.benchmark` file that contains a number of directives instructing+the tool on how to load data, run initializations, run assertions, run the benchmark, optionally persist and+validate results, and finally run any cleanup if required.++Variables are supported in two forms:++* string substitution based on environment variables (with default values if unset): \${ENV_VAR} and+ \${ENV_VAR:-default}.+* if / else based on whether an environment variable is true or not+ (\${ENV_VAR:-default|true value|false value}). In this form only the value `true` (case-insensitive) selects the+ true branch; any other set value selects the false branch. If ENV_VAR is unset, the valud of `default` is used to+* select the branch.++Comments in files are supported with lines starting with # or --.++Many if not most of the benchmarks are set up using templates to reduce duplication across the .benchmark files. For+example here is one of the benchmark files for the h2o benchmark suite:++```+subgroup groupby++template sql_benchmarks/h2o/h2o.benchmark.template+QUERY_NUMBER=1+QUERY_NUMBER_PADDED=01+```++The template directive above defines the subgroup the benchmark is part of, sets two variables (`QUERY_NUMBER` and+`QUERY_NUMBER_PADDED`) and points to a file containing more directives that are shared across the benchmark suite.++```+load sql_benchmarks/h2o/init/load_${BENCH_SUBGROUP:-groupby}_${BENCH_SIZE:-small}_${BENCH_FILE_TYPE:-csv}.sql++name Q${QUERY_NUMBER_PADDED}+group h2o++run sql_benchmarks/h2o/queries/${BENCH_SUBGROUP:-groupby}/q${QUERY_NUMBER_PADDED}.sql++result sql_benchmarks/h2o/results/${BENCH_SUBGROUP:-groupby}/${BENCH_SIZE:-small}/q${QUERY_NUMBER_PADDED}.csv+```++The above showcases the use of defaults for variables: `${NAME:-default}`++# Directives++<table>+<tr><th>Directive</th><th>Description</th></tr>+<tr>+<td>name</td>+<td>++The name of the benchmark. This will be used as part of the display name used by criterion.<br/><br/>Example:<br/>+<blockquote>name Q${QUERY_NUMBER_PADDED}</blockquote>++The `name` directive also makes the value available to benchmark-file replacements as `BENCH_NAME`. This is separate+from the `BENCH_NAME` environment variable used to select which benchmark group to run.++</td>+</tr>+<tr>+<td>group</td>+<td>++The group name of the benchmark used for grouping benchmarks together.<br/><br/>Example:<br/>+<blockquote>group imdb</blockquote>++</td>+</tr>+<tr>+<td>subgroup</td>+<td>++The sub group name of the benchmark used for filtering to a specific sub group.<br/><br/>Example:<br/>+<blockquote>subgroup window</blockquote>++</td>+</tr>+<tr>+<td>load</td>+<td>++The load directive called during initialization of the benchmark. If a path to a file is provided on the same+line as the load directive that path will be parsed and any sql statements in that file will be executed during+initialization. If no path is specified the next line is required to be the sql statement to execute. <br/> <br/> The+load directive (including any following sql statement) must be followed by a blank line. <br/><br/>Example:<br/>+<blockquote>load sql_benchmarks/h2o/init/load_${BENCH_SUBGROUP:-groupby}_${BENCH_SIZE:-small}_${BENCH_FILE_TYPE:-csv}.sql</blockquote>+or+<blockquote>+load<br/>+CREATE TABLE test AS (SELECT value as key FROM range(1000000) ORDER BY value);+</blockquote>++</td>+</tr>+<tr>+<td>init</td>+<td>++The init directive is called after the load directive prior to benchmark execution. If a path to a file is+provided on the same line as the init directive that path will be parsed and any sql statements in that file will be+executed during the benchmark initialization. If no path is specified the next line is required to be the sql statement+to execute.<br/><br/> The init directive (including any following sql statement) must be followed by a blank+line.<br/><br/>Example:<br/>+<blockquote>+init<br/>+set datafusion.execution.parquet.binary_as_string = true;+</blockquote>++</td>+</tr>+<tr>+<td>run</td>+<td>++The run directive called during execution of the benchmark. If a path to a file is provided on the same line as+the run directive that path will be parsed and any sql statements in that file will be executed during the benchmark+run. If no path is specified the next line is required to be the sql statement to execute. <br/><br/> Multiple+statements are allowed within a single run directive, however a benchmark file may contain only one run directive. When+running with `BENCH_PERSIST_RESULTS` or `BENCH_VALIDATE`, only the last `SELECT` or `WITH` statement from that run+directive will be used for comparison. <br/><br/> The run directive (including any following sql statement) must be+followed by a blank line.<br/><br/>Example:<br/>+<blockquote>run sql_benchmarks/imdb/queries/${QUERY_NUMBER_PADDED}.sql</blockquote>++</td>+</tr>+<tr>+<td>cleanup</td>+<td>++The cleanup directive is called after all other directives and can be used to cleanup after the benchmark -+e.g. to drop tables. If a path to a file is provided on the same line as the cleanup directive that path will be parsed+and any sql statements in that file will be executed during cleanup. If no path is specified the next line is+required to be the sql statement to execute. <br/> <br/> The cleanup directive (including any following sql statement)+must be followed by a blank line. <br/><br/>Example:<br/>+<blockquote>+cleanup<br/>+DROP TABLE test;+</blockquote>++</td>+</tr>+<tr>+<td>expect_plan</td>+<td>++The expect_plan directive will check the physical plan for the string provided on the same line. This+can be used to validate that a particular join was used. <br/> <br/> Example:<br/>+<blockquote>expect_plan NestedLoopJoinExec</blockquote>++</td>+</tr>+<tr>+<td>assert</td>+<td>++The assert directive is run between the init and run directives and can be used to validate system state correctness+prior to running the benchmark sql. The format is+<blockquote>+assert II<br/>+SELECT name, value = 3 FROM information_schema.df_settings WHERE name IN ('datafusion.execution.target_partitions', 'datafusion.execution.planning_concurrency');<br/>+----<br/>+datafusion.execution.planning_concurrency true<br/>+datafusion.execution.target_partitions true<br/>+</blockquote>++The number of I's corresponds to the number of columns in the result. The expected results can be either tab delimited+or pipe delimited.++</td>+</tr>+<tr>+<td>result_query</td>+<td>++The result_query directive is run during the verify phase and can be used to verify a different set of results than any+that might come from queries executed from the `run` directive. The format is the same as the `assert` directive+above.<br/><br/>Example:+<blockquote>+result_query III<br/>+SELECT COUNT(DISTINCT id2), SUM(r4), COUNT(*) FROM answer;<br/>+----<br/>+123 345 45+</blockquote>+++Note that the results of the run query are not automatically stored into a table in datafusion. If you want to+verify a result from queries executed from the `run` directive those queries will have to be saved to a table directly+using `CREATE TABLE AS (..)` or similar.++</td>+</tr>+<tr>+<td>result</td>+<td>++The result directive declares the expected result file used during verification. A path to a file is required on the+same line as the result directive. The file is parsed only during verification, and must be a pipe-delimited CSV file+with a header row. During verification, these expected rows are compared with the rows produced by the last saved+`SELECT` or `WITH` statement from the `run` directive. <br/><br/>Example:<br/>+<blockquote>+result sql_benchmarks/imdb/results/${QUERY_NUMBER_PADDED}.csv+</blockquote>++</td>+</tr>+<tr>+<td>template</td>+<td>++The template directive allows for inclusion of another file in a benchmark file. A path to a file is+required on the same line as the template directive which will be parsed as a benchmark file. Parameters can be passed+to the template file using the format `KEY=value`, one per line after the template directive followed by a blank line.+<br/><br/>Example:<br/>+<blockquote>+template sql_benchmarks/smj/smj.benchmark.template<br/>+QUERY_NUMBER=1<br/>+QUERY_NUMBER_PADDED=01+</blockquote>++</td>+</tr>+<tr>+<td>include</td>+<td>The include directive is similar to the template directive except that it does not support parameters.</td>+</tr>+<tr>+<td>echo</td>+<td>++The echo directive allows for echoing a string to stdout during the execution of the benchmark and may be useful for+debugging.<br/><br/>Example:<br/>+<blockquote>+echo The value for batch size is ${BATCH_SIZE:-8192}+</blockquote>++</td>+</tr>+</table>++# Extending an existing benchmark suite++If you want to add a new query:++* Create a new qXX.sql in the corresponding queries folder of the benchmark.+* Add a new qXX.benchmark that references the appropriate template (clickbench.benchmark.template,+ h2o.benchmark.template,+ etc.).+* (Optional) Add a new entry to the suite’s load script if the data set is different.+* (Optional) Manually create a result csv to be compared against benchmark results during verification.++# Adding a new benchmark suite++* Create a new directory named for the new benchmark suite.+* Within there create a `<name>.benchmark` for each individual benchmark.+* Populate the benchmark with directives as described above. Use the other benchmarks as examples for standardization.+* No rust files need to be updated to run the new benchmark suite.diff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q01.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q01.benchmarknew file mode 100644index 0000000000..d490927df3--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q01.benchmark@@ -0,0 +1,34 @@+name Q01+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select l_returnflag,+ l_linestatus,+ sum(l_quantity) as sum_qty,+ sum(l_extendedprice) as sum_base_price,+ sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,+ sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,+ avg(l_quantity) as avg_qty,+ avg(l_extendedprice) as avg_price,+ avg(l_discount) as avg_disc,+ count(*) as count_order+ from lineitem+ where l_shipdate <= date '1998-12-01' - interval '90' day+ group by l_returnflag,+ l_linestatus+ order by l_returnflag,+ l_linestatus;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q01.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q02.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q02.benchmarknew file mode 100644index 0000000000..6f365248b4--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q02.benchmark@@ -0,0 +1,62 @@+name Q02+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ s_acctbal,+ s_name,+ n_name,+ p_partkey,+ p_mfgr,+ s_address,+ s_phone,+ s_comment+from+ part,+ supplier,+ partsupp,+ nation,+ region+where+ p_partkey = ps_partkey+ and s_suppkey = ps_suppkey+ and p_size = 15+ and p_type like '%BRASS'+ and s_nationkey = n_nationkey+ and n_regionkey = r_regionkey+ and r_name = 'EUROPE'+ and ps_supplycost = (+ select+ min(ps_supplycost)+ from+ partsupp,+ supplier,+ nation,+ region+ where+ p_partkey = ps_partkey+ and s_suppkey = ps_suppkey+ and s_nationkey = n_nationkey+ and n_regionkey = r_regionkey+ and r_name = 'EUROPE'+)+order by+ s_acctbal desc,+ n_name,+ s_name,+ p_partkey+limit 100;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q02.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q03.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q03.benchmarknew file mode 100644index 0000000000..cb16b10c2b--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q03.benchmark@@ -0,0 +1,41 @@+name Q03+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ l_orderkey,+ sum(l_extendedprice * (1 - l_discount)) as revenue,+ o_orderdate,+ o_shippriority+from+ customer,+ orders,+ lineitem+where+ c_mktsegment = 'BUILDING'+ and c_custkey = o_custkey+ and l_orderkey = o_orderkey+ and o_orderdate < date '1995-03-15'+ and l_shipdate > date '1995-03-15'+group by+ l_orderkey,+ o_orderdate,+ o_shippriority+order by+ revenue desc,+ o_orderdate+limit 10;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q03.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q04.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q04.benchmarknew file mode 100644index 0000000000..f2e6f9a558--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q04.benchmark@@ -0,0 +1,39 @@+name Q04+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ o_orderpriority,+ count(*) as order_count+from+ orders+where+ o_orderdate >= '1993-07-01'+ and o_orderdate < date '1993-07-01' + interval '3' month+ and exists (+ select+ *+ from+ lineitem+ where+ l_orderkey = o_orderkey+ and l_commitdate < l_receiptdate+ )+group by+ o_orderpriority+order by+ o_orderpriority;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q04.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q05.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q05.benchmarknew file mode 100644index 0000000000..9b5fbda63b--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q05.benchmark@@ -0,0 +1,37 @@+name Q05+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select n_name,+ sum(l_extendedprice * (1 - l_discount)) as revenue+from customer,+ orders,+ lineitem,+ supplier,+ nation,+ region+where c_custkey = o_custkey+ and l_orderkey = o_orderkey+ and l_suppkey = s_suppkey+ and c_nationkey = s_nationkey+ and s_nationkey = n_nationkey+ and n_regionkey = r_regionkey+ and r_name = 'ASIA'+ and o_orderdate >= date '1994-01-01'+ and o_orderdate < date '1994-01-01' + interval '1' year+group by n_name+order by revenue desc;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q05.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q06.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q06.benchmarknew file mode 100644index 0000000000..761875cbf3--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q06.benchmark@@ -0,0 +1,25 @@+name Q06+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select sum(l_extendedprice * l_discount) as revenue+from lineitem+where l_shipdate >= date '1994-01-01'+ and l_shipdate < date '1994-01-01' + interval '1' year+ and l_discount between 0.06 - 0.01 and 0.06 + 0.01+ and l_quantity < 24;+++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q06.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q07.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q07.benchmarknew file mode 100644index 0000000000..30c4c520de--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q07.benchmark@@ -0,0 +1,57 @@+name Q07+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ supp_nation,+ cust_nation,+ l_year,+ sum(volume) as revenue+from+ (+ select+ n1.n_name as supp_nation,+ n2.n_name as cust_nation,+ extract(year from l_shipdate) as l_year,+ l_extendedprice * (1 - l_discount) as volume+ from+ supplier,+ lineitem,+ orders,+ customer,+ nation n1,+ nation n2+ where+ s_suppkey = l_suppkey+ and o_orderkey = l_orderkey+ and c_custkey = o_custkey+ and s_nationkey = n1.n_nationkey+ and c_nationkey = n2.n_nationkey+ and (+ (n1.n_name = 'FRANCE' and n2.n_name = 'GERMANY')+ or (n1.n_name = 'GERMANY' and n2.n_name = 'FRANCE')+ )+ and l_shipdate between date '1995-01-01' and date '1996-12-31'+ ) as shipping+group by+ supp_nation,+ cust_nation,+ l_year+order by+ supp_nation,+ cust_nation,+ l_year;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q07.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q08.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q08.benchmarknew file mode 100644index 0000000000..86caded4b7--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q08.benchmark@@ -0,0 +1,55 @@+name Q08+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ o_year,+ sum(case+ when nation = 'BRAZIL' then volume+ else 0+ end) / sum(volume) as mkt_share+from+ (+ select+ extract(year from o_orderdate) as o_year,+ l_extendedprice * (1 - l_discount) as volume,+ n2.n_name as nation+ from+ part,+ supplier,+ lineitem,+ orders,+ customer,+ nation n1,+ nation n2,+ region+ where+ p_partkey = l_partkey+ and s_suppkey = l_suppkey+ and l_orderkey = o_orderkey+ and o_custkey = c_custkey+ and c_nationkey = n1.n_nationkey+ and n1.n_regionkey = r_regionkey+ and r_name = 'AMERICA'+ and s_nationkey = n2.n_nationkey+ and o_orderdate between date '1995-01-01' and date '1996-12-31'+ and p_type = 'ECONOMY ANODIZED STEEL'+ ) as all_nations+group by+ o_year+order by+ o_year;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q08.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q09.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q09.benchmarknew file mode 100644index 0000000000..3302cf6f0b--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q09.benchmark@@ -0,0 +1,50 @@+name Q09+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ nation,+ o_year,+ sum(amount) as sum_profit+from+ (+ select+ n_name as nation,+ extract(year from o_orderdate) as o_year,+ l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount+ from+ part,+ supplier,+ lineitem,+ partsupp,+ orders,+ nation+ where+ s_suppkey = l_suppkey+ and ps_suppkey = l_suppkey+ and ps_partkey = l_partkey+ and p_partkey = l_partkey+ and o_orderkey = l_orderkey+ and s_nationkey = n_nationkey+ and p_name like '%green%'+ ) as profit+group by+ nation,+ o_year+order by+ nation,+ o_year desc;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q09.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q10.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q10.benchmarknew file mode 100644index 0000000000..4ef08e3fd2--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q10.benchmark@@ -0,0 +1,50 @@+name Q10+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ c_custkey,+ c_name,+ sum(l_extendedprice * (1 - l_discount)) as revenue,+ c_acctbal,+ n_name,+ c_address,+ c_phone,+ c_comment+from+ customer,+ orders,+ lineitem,+ nation+where+ c_custkey = o_custkey+ and l_orderkey = o_orderkey+ and o_orderdate >= date '1993-10-01'+ and o_orderdate < date '1993-10-01' + interval '3' month+ and l_returnflag = 'R'+ and c_nationkey = n_nationkey+group by+ c_custkey,+ c_name,+ c_acctbal,+ c_phone,+ n_name,+ c_address,+ c_comment+order by+ revenue desc+limit 20;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q10.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q11.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q11.benchmarknew file mode 100644index 0000000000..833799a39d--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q11.benchmark@@ -0,0 +1,45 @@+name Q11+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ ps_partkey,+ sum(ps_supplycost * ps_availqty) as value+from+ partsupp,+ supplier,+ nation+where+ ps_suppkey = s_suppkey+ and s_nationkey = n_nationkey+ and n_name = 'GERMANY'+group by+ ps_partkey having+ sum(ps_supplycost * ps_availqty) > (+ select+ sum(ps_supplycost * ps_availqty) * (0.0001/${BENCH_SIZE:-1})+ from+ partsupp,+ supplier,+ nation+ where+ ps_suppkey = s_suppkey+ and s_nationkey = n_nationkey+ and n_name = 'GERMANY'+ )+order by+ value desc;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q11.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q12.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q12.benchmarknew file mode 100644index 0000000000..37aee848c9--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q12.benchmark@@ -0,0 +1,43 @@+name Q12+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select l_shipmode,+ sum(case+ when o_orderpriority = '1-URGENT'+ or o_orderpriority = '2-HIGH'+ then 1+ else 0+ end) as high_line_count,+ sum(case+ when o_orderpriority <> '1-URGENT'+ and o_orderpriority <> '2-HIGH'+ then 1+ else 0+ end) as low_line_count+from lineitem+ join+ orders+ on+ l_orderkey = o_orderkey+where l_shipmode in ('MAIL', 'SHIP')+ and l_commitdate < l_receiptdate+ and l_shipdate < l_commitdate+ and l_receiptdate >= date '1994-01-01'+ and l_receiptdate < date '1994-01-01' + interval '1' year+group by l_shipmode+order by l_shipmode;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q12.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q13.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q13.benchmarknew file mode 100644index 0000000000..dfb09853d0--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q13.benchmark@@ -0,0 +1,38 @@+name Q13+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ c_count,+ count(*) as custdist+from+ (+ select+ c_custkey,+ count(o_orderkey)+ from+ customer left outer join orders on+ c_custkey = o_custkey+ and o_comment not like '%special%requests%'+ group by+ c_custkey+ ) as c_orders (c_custkey, c_count)+group by+ c_count+order by+ custdist desc,+ c_count desc;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q13.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q14.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q14.benchmarknew file mode 100644index 0000000000..b48d95043f--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q14.benchmark@@ -0,0 +1,28 @@+name Q14+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select 100.00 * sum(case+ when p_type like 'PROMO%'+ then l_extendedprice * (1 - l_discount)+ else 0+ end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue+from lineitem,+ part+where l_partkey = p_partkey+ and l_shipdate >= date '1995-09-01'+ and l_shipdate < date '1995-09-01' + interval '1' month;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q14.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q15.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q15.benchmarknew file mode 100644index 0000000000..0f50fc499d--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q15.benchmark@@ -0,0 +1,49 @@+name Q15+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+create view revenue0 (supplier_no, total_revenue) as+ select+ l_suppkey,+ sum(l_extendedprice * (1 - l_discount))+ from+ lineitem+ where+ l_shipdate >= date '1996-01-01'+ and l_shipdate < date '1996-01-01' + interval '3' month+ group by+ l_suppkey;+select+ s_suppkey,+ s_name,+ s_address,+ s_phone,+ total_revenue+from+ supplier,+ revenue0+where+ s_suppkey = supplier_no+ and total_revenue = (+ select+ max(total_revenue)+ from+ revenue0+ )+order by+ s_suppkey;+drop view revenue0;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q15.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q16.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q16.benchmarknew file mode 100644index 0000000000..3fa6c68e29--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q16.benchmark@@ -0,0 +1,48 @@+name Q16+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ p_brand,+ p_type,+ p_size,+ count(distinct ps_suppkey) as supplier_cnt+from+ partsupp,+ part+where+ p_partkey = ps_partkey+ and p_brand <> 'Brand#45'+ and p_type not like 'MEDIUM POLISHED%'+ and p_size in (49, 14, 23, 45, 19, 3, 36, 9)+ and ps_suppkey not in (+ select+ s_suppkey+ from+ supplier+ where+ s_comment like '%Customer%Complaints%'+)+group by+ p_brand,+ p_type,+ p_size+order by+ supplier_cnt desc,+ p_brand,+ p_type,+ p_size;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q16.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q17.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q17.benchmarknew file mode 100644index 0000000000..a31c837d1e--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q17.benchmark@@ -0,0 +1,35 @@+name Q17+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ sum(l_extendedprice) / 7.0 as avg_yearly+from+ lineitem,+ part+where+ p_partkey = l_partkey+ and p_brand = 'Brand#23'+ and p_container = 'MED BOX'+ and l_quantity < (+ select+ 0.2 * avg(l_quantity)+ from+ lineitem+ where+ l_partkey = p_partkey+);++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q17.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q18.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q18.benchmarknew file mode 100644index 0000000000..149b0efd01--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q18.benchmark@@ -0,0 +1,51 @@+name Q18+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ c_name,+ c_custkey,+ o_orderkey,+ o_orderdate,+ o_totalprice,+ sum(l_quantity)+from+ customer,+ orders,+ lineitem+where+ o_orderkey in (+ select+ l_orderkey+ from+ lineitem+ group by+ l_orderkey having+ sum(l_quantity) > 300+ )+ and c_custkey = o_custkey+ and o_orderkey = l_orderkey+group by+ c_name,+ c_custkey,+ o_orderkey,+ o_orderdate,+ o_totalprice+order by+ o_totalprice desc,+ o_orderdate+limit 100;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q18.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q19.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q19.benchmarknew file mode 100644index 0000000000..f93ad6cb73--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q19.benchmark@@ -0,0 +1,53 @@+name Q19+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ sum(l_extendedprice* (1 - l_discount)) as revenue+from+ lineitem,+ part+where+ (+ p_partkey = l_partkey+ and p_brand = 'Brand#12'+ and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')+ and l_quantity >= 1 and l_quantity <= 1 + 10+ and p_size between 1 and 5+ and l_shipmode in ('AIR', 'AIR REG')+ and l_shipinstruct = 'DELIVER IN PERSON'+ )+ or+ (+ p_partkey = l_partkey+ and p_brand = 'Brand#23'+ and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')+ and l_quantity >= 10 and l_quantity <= 10 + 10+ and p_size between 1 and 10+ and l_shipmode in ('AIR', 'AIR REG')+ and l_shipinstruct = 'DELIVER IN PERSON'+ )+ or+ (+ p_partkey = l_partkey+ and p_brand = 'Brand#34'+ and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')+ and l_quantity >= 20 and l_quantity <= 20 + 10+ and p_size between 1 and 15+ and l_shipmode in ('AIR', 'AIR REG')+ and l_shipinstruct = 'DELIVER IN PERSON'+ );++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q19.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q20.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q20.benchmarknew file mode 100644index 0000000000..123386055b--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q20.benchmark@@ -0,0 +1,55 @@+name Q20+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ s_name,+ s_address+from+ supplier,+ nation+where+ s_suppkey in (+ select+ ps_suppkey+ from+ partsupp+ where+ ps_partkey in (+ select+ p_partkey+ from+ part+ where+ p_name like 'forest%'+ )+ and ps_availqty > (+ select+ 0.5 * sum(l_quantity)+ from+ lineitem+ where+ l_partkey = ps_partkey+ and l_suppkey = ps_suppkey+ and l_shipdate >= date '1994-01-01'+ and l_shipdate < date '1994-01-01' + interval '1' year+ )+ )+ and s_nationkey = n_nationkey+ and n_name = 'CANADA'+order by+ s_name;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q20.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q21.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q21.benchmarknew file mode 100644index 0000000000..24d754a4cb--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q21.benchmark@@ -0,0 +1,58 @@+name Q21+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ s_name,+ count(*) as numwait+from+ supplier,+ lineitem l1,+ orders,+ nation+where+ s_suppkey = l1.l_suppkey+ and o_orderkey = l1.l_orderkey+ and o_orderstatus = 'F'+ and l1.l_receiptdate > l1.l_commitdate+ and exists (+ select+ *+ from+ lineitem l2+ where+ l2.l_orderkey = l1.l_orderkey+ and l2.l_suppkey <> l1.l_suppkey+ )+ and not exists (+ select+ *+ from+ lineitem l3+ where+ l3.l_orderkey = l1.l_orderkey+ and l3.l_suppkey <> l1.l_suppkey+ and l3.l_receiptdate > l3.l_commitdate+ )+ and s_nationkey = n_nationkey+ and n_name = 'SAUDI ARABIA'+group by+ s_name+order by+ numwait desc,+ s_name+limit 100;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q21.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/benchmarks/q22.benchmark b/benchmarks/sql_benchmarks/tpch/benchmarks/q22.benchmarknew file mode 100644index 0000000000..7ef6a78496--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/benchmarks/q22.benchmark@@ -0,0 +1,55 @@+name Q22+group tpch+subgroup sf${BENCH_SIZE:-1}++init sql_benchmarks/tpch/init/set_config.sql++load sql_benchmarks/tpch/init/load_${TPCH_FILE_TYPE:-parquet}.sql++assert I+SELECT COUNT(*) > 0 from lineitem;+----+true++run+select+ cntrycode,+ count(*) as numcust,+ sum(c_acctbal) as totacctbal+from+ (+ select+ substring(c_phone from 1 for 2) as cntrycode,+ c_acctbal+ from+ customer+ where+ substring(c_phone from 1 for 2) in+ ('13', '31', '23', '29', '30', '18', '17')+ and c_acctbal > (+ select+ avg(c_acctbal)+ from+ customer+ where+ c_acctbal > 0.00+ and substring(c_phone from 1 for 2) in+ ('13', '31', '23', '29', '30', '18', '17')+ )+ and not exists (+ select+ *+ from+ orders+ where+ o_custkey = c_custkey+ )+ ) as custsale+group by+ cntrycode+order by+ cntrycode;++result sql_benchmarks/tpch/results/sf${BENCH_SIZE:-1}/q22.csv++cleanup sql_benchmarks/tpch/init/cleanup.sqldiff --git a/benchmarks/sql_benchmarks/tpch/init/cleanup.sql b/benchmarks/sql_benchmarks/tpch/init/cleanup.sqlnew file mode 100644index 0000000000..c8fb66a6a5--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/init/cleanup.sql@@ -0,0 +1,15 @@+DROP TABLE IF EXISTS nation;++DROP TABLE IF EXISTS region;++DROP TABLE IF EXISTS supplier;++DROP TABLE IF EXISTS customer;++DROP TABLE IF EXISTS part;++DROP TABLE IF EXISTS partsupp;++DROP TABLE IF EXISTS orders;++DROP TABLE IF EXISTS lineitem;diff --git a/benchmarks/sql_benchmarks/tpch/init/load_csv.sql b/benchmarks/sql_benchmarks/tpch/init/load_csv.sqlnew file mode 100644index 0000000000..f9a9b2e988--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/init/load_csv.sql@@ -0,0 +1,99 @@+CREATE EXTERNAL TABLE nation+(+ n_nationkey INT,+ n_name CHAR(25),+ n_regionkey INT,+ n_comment VARCHAR(152),+ PRIMARY KEY (n_nationkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/nation/nation.1.csv';++CREATE EXTERNAL TABLE region+(+ r_regionkey INT,+ r_name CHAR(25),+ r_comment VARCHAR(152),+ PRIMARY KEY (r_regionkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/region/region.1.csv';++CREATE EXTERNAL TABLE supplier+(+ s_suppkey INT,+ s_name CHAR(25),+ s_address VARCHAR(40),+ s_nationkey INT,+ s_phone CHAR(15),+ s_acctbal DECIMAL(15, 2),+ s_comment VARCHAR(101),+ PRIMARY KEY (s_suppkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/supplier/supplier.1.csv';++CREATE EXTERNAL TABLE customer+(+ c_custkey INT,+ c_name VARCHAR(25),+ c_address VARCHAR(40),+ c_nationkey INT,+ c_phone CHAR(15),+ c_acctbal DECIMAL(15, 2),+ c_mktsegment CHAR(10),+ c_comment VARCHAR(117),+ PRIMARY KEY (c_custkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/customer/customer.1.csv';++CREATE EXTERNAL TABLE part+(+ p_partkey INT,+ p_name VARCHAR(55),+ p_mfgr CHAR(25),+ p_brand CHAR(10),+ p_type VARCHAR(25),+ p_size INT,+ p_container CHAR(10),+ p_retailprice DECIMAL(15, 2),+ p_comment VARCHAR(23),+ PRIMARY KEY (p_partkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/part/part.1.csv';++CREATE EXTERNAL TABLE partsupp+(+ ps_partkey INT,+ ps_suppkey INT,+ ps_availqty INT,+ ps_supplycost DECIMAL(15, 2),+ ps_comment VARCHAR(199),+ PRIMARY KEY (ps_partkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/partsupp/partsupp.1.csv';++CREATE EXTERNAL TABLE orders+(+ o_orderkey INT,+ o_custkey INT,+ o_orderstatus CHAR(1),+ o_totalprice DECIMAL(15, 2),+ o_orderdate DATE,+ o_orderpriority CHAR(15),+ o_clerk CHAR(15),+ o_shippriority INT,+ o_comment VARCHAR(79),+ PRIMARY KEY (o_orderkey)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/orders/orders.1.csv';++CREATE EXTERNAL TABLE lineitem+(+ l_orderkey INT,+ l_partkey INT,+ l_suppkey INT,+ l_linenumber INT,+ l_quantity DECIMAL(15, 2),+ l_extendedprice DECIMAL(15, 2),+ l_discount DECIMAL(15, 2),+ l_tax DECIMAL(15, 2),+ l_returnflag CHAR(1),+ l_linestatus CHAR(1),+ l_shipdate DATE,+ l_commitdate DATE,+ l_receiptdate DATE,+ l_shipinstruct CHAR(25),+ l_shipmode CHAR(10),+ l_comment VARCHAR(44)+) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/lineitem/lineitem.1.csv';\ No newline at end of filediff --git a/benchmarks/sql_benchmarks/tpch/init/load_mem.sql b/benchmarks/sql_benchmarks/tpch/init/load_mem.sqlnew file mode 100644index 0000000000..57d12c22f0--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/init/load_mem.sql@@ -0,0 +1,31 @@+CREATE EXTERNAL TABLE nation_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';++CREATE EXTERNAL TABLE region_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';++CREATE EXTERNAL TABLE supplier_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';++CREATE EXTERNAL TABLE customer_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';++CREATE EXTERNAL TABLE part_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';++CREATE EXTERNAL TABLE partsupp_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';++CREATE EXTERNAL TABLE orders_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';++CREATE EXTERNAL TABLE lineitem_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';++CREATE TABLE nation as SELECT * FROM nation_raw;++CREATE TABLE region as SELECT * FROM region_raw;++CREATE TABLE supplier as SELECT * FROM supplier_raw;++CREATE TABLE customer as SELECT * FROM customer_raw;++CREATE TABLE part as SELECT * FROM part_raw;++CREATE TABLE partsupp as SELECT * FROM partsupp_raw;++CREATE TABLE orders as SELECT * FROM orders_raw;++CREATE TABLE lineitem as SELECT * FROM lineitem_raw;diff --git a/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sqlnew file mode 100644index 0000000000..172a03d82a--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql@@ -0,0 +1,15 @@+CREATE EXTERNAL TABLE nation STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';++CREATE EXTERNAL TABLE region STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';++CREATE EXTERNAL TABLE supplier STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';++CREATE EXTERNAL TABLE customer STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';++CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';++CREATE EXTERNAL TABLE partsupp STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';++CREATE EXTERNAL TABLE orders STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';++CREATE EXTERNAL TABLE lineitem STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';\ No newline at end of filediff --git a/benchmarks/sql_benchmarks/tpch/init/set_config.sql b/benchmarks/sql_benchmarks/tpch/init/set_config.sqlnew file mode 100644index 0000000000..00457e2bca--- /dev/null+++ b/benchmarks/sql_benchmarks/tpch/init/set_config.sql@@ -0,0 +1,3 @@+set datafusion.optimizer.prefer_hash_join=${PREFER_HASH_JOIN:-true};++set datafusion.execution.hash_join_buffering_capacity=${HASH_JOIN_BUFFERING_CAPACITY:-0};diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rsindex 7e2196e89e..7c598e65d8 100644--- a/benchmarks/src/lib.rs+++ b/benchmarks/src/lib.rs@@ -25,6 +25,7 @@ pub mod nlj;pub mod smj;pub mod sort_pushdown;pub mod sort_tpch;+pub mod sql_benchmark;pub mod tpcds;pub mod tpch;pub mod util;diff --git a/benchmarks/src/sql_benchmark.rs b/benchmarks/src/sql_benchmark.rsnew file mode 100644index 0000000000..34614b1324--- /dev/null+++ b/benchmarks/src/sql_benchmark.rs@@ -0,0 +1,3538 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use arrow::array::{Array, RecordBatch};+use arrow::datatypes::*;+use arrow::error::ArrowError;+use arrow::util::display::{ArrayFormatter, FormatOptions};+use datafusion::dataframe::DataFrameWriteOptions;+use datafusion::datasource::MemTable;+use datafusion::physical_plan::execute_stream;+use datafusion::prelude::{CsvReadOptions, DataFrame, SessionContext};+use datafusion_common::config::CsvOptions;+use datafusion_common::{DataFusionError, Result, exec_datafusion_err};+use futures::StreamExt;+use log::{debug, info, trace, warn};+use regex::Regex;+use std::collections::HashMap;+use std::fmt::Debug;+use std::fs::{self, File, OpenOptions};+use std::io::{BufRead, BufReader};+use std::path::{Path, PathBuf};+use std::sync::{Arc, LazyLock};++/// A collection of benchmark configurations and state used by the DataFusion+/// sql test harness. Each benchmark is defined by a file that can contain+/// directives such as `load`, `run`, `assert`, `result`, etc. The+/// `SqlBenchmark` struct holds the parsed data from that file and+/// the impl provides methods to run, assert, persist, verify and cleanup benchmark+/// results.+#[derive(Debug, Clone)]+pub struct SqlBenchmark {+ /// Human‑readable name of the benchmark.+ name: String,+ /// Top‑level group name (derived from the file path or defined in a benchmark).+ group: String,+ /// Subgroup name, often a logical grouping.+ subgroup: String,+ /// Full path to the benchmark file.+ benchmark_path: PathBuf,+ /// Mapping of placeholder keys to concrete values (e.g. `"BENCHMARK_DIR"`).+ replacement_mapping: HashMap<String, String>,+ /// Expected string that must appear in the physical plan of the queries.+ expect: Vec<String>,+ /// All SQL queries grouped by directive (`load`, `run`, etc.).+ queries: HashMap<QueryDirective, Vec<String>>,+ /// Queries whose results are persisted to disk for later comparison.+ result_queries: Vec<BenchmarkQuery>,+ /// Queries whose results are asserted against an expected table.+ assert_queries: Vec<BenchmarkQuery>,+ /// Flag indicating whether the benchmark has been fully loaded+ is_loaded: bool,+ /// Stores the last run results if needed so they can be compared or persisted.+ last_results: Option<Vec<RecordBatch>>,+ /// echo statements+ echo: Vec<String>,+}++impl SqlBenchmark {+ pub async fn new(+ ctx: &SessionContext,+ full_path: impl AsRef<Path>,+ benchmark_directory: impl AsRef<Path>,+ ) -> Result<Self> {+ let full_path = full_path.as_ref();+ let benchmark_directory = benchmark_directory.as_ref();+ let group_name = parse_group_from_path(full_path, benchmark_directory);+ let mut bm = Self {+ name: String::new(),+ group: group_name,+ subgroup: String::new(),+ benchmark_path: full_path.to_path_buf(),+ replacement_mapping: HashMap::new(),+ expect: vec![],+ queries: HashMap::new(),+ result_queries: vec![],+ assert_queries: vec![],+ is_loaded: false,+ last_results: None,+ echo: vec![],+ };+ insert_replacement(+ &mut bm.replacement_mapping,+ "BENCHMARK_DIR",+ benchmark_directory.to_string_lossy().into_owned(),+ );++ let path = bm.benchmark_path.clone();+ bm.process_file(ctx, &path).await?;++ Ok(bm)+ }++ /// Initializes the benchmark by executing `load` and `init` queries.+ ///+ /// Registers any required tables or sets up state in the provided+ /// `SessionContext` before running queries. This method is idempotent:+ /// calling it multiple times on the same instance returns+ /// immediately after the first successful initialization.+ ///+ /// # Errors+ /// Returns an error if any `load` or `init` query fails, or if the+ /// benchmark file does not contain a `run` query.+ pub async fn initialize(&mut self, ctx: &SessionContext) -> Result<()> {+ if self.is_loaded {+ return Ok(());+ }++ let path = self.benchmark_path.to_string_lossy().into_owned();++ // validate there was a run query+ if !self.queries.contains_key(&QueryDirective::Run) {+ return Err(exec_datafusion_err!(+ "Invalid benchmark file: no \"run\" query specified: {path}"+ ));+ }++ // display any echo's+ self.echo.iter().for_each(|txt| println!("{txt}"));++ let load_queries = self.queries.get(&QueryDirective::Load);++ if let Some(queries) = load_queries {+ for query in queries {+ debug!("Executing load query {query}");+ ctx.sql(query).await?.collect().await?;+ }+ }++ let init_queries = self.queries.get(&QueryDirective::Init);++ if let Some(queries) = init_queries {+ for query in queries {+ debug!("Executing init query {query}");+ ctx.sql(query).await?.collect().await?;+ }+ }++ self.is_loaded = true;++ Ok(())+ }++ /// Executes the `assert` queries and compares actual results against+ /// expected values.+ ///+ /// Each `assert` query must be followed by a result table (separated by+ /// `----`) in the benchmark file. The assertion passes only if the+ /// returned record batches exactly match the expected rows.+ ///+ /// # Errors+ /// Returns an error if any `assert` query fails, or if the actual and+ /// expected results differ in row count or cell values.+ pub async fn assert(&mut self, ctx: &SessionContext) -> Result<()> {+ info!("Running assertions...");++ for assert_query in &self.assert_queries {+ let query = &assert_query.query;++ info!("Executing assert query {query}");++ let result = ctx.sql(query).await?.collect().await?;+ let formatted_actual_results = format_record_batches(&result)?;++ Self::compare_results(+ assert_query,+ &formatted_actual_results,+ &assert_query.expected_result,+ )?;+ }++ Ok(())+ }++ /// Executes the `run` queries, optionally saving results for later+ /// verification. If there are multiple queries only the results for+ /// the last query are saved.+ ///+ /// When `save_results` is `true`, it collects `SELECT`/`WITH` query+ /// results and stores them in `last_results`.+ ///+ /// When `save_results` is `false`, it streams results and counts rows+ /// without buffering them.+ ///+ /// If an 'expect' string is defined this method also validates that+ /// the physical plan contains that string.+ ///+ /// # Errors+ /// Returns an error if a `run` query fails or if expected plan strings+ /// are not found.+ pub async fn run(&mut self, ctx: &SessionContext, save_results: bool) -> Result<()> {+ let run_queries = self+ .queries+ .get(&QueryDirective::Run)+ .ok_or_else(|| exec_datafusion_err!("Run query should be loaded by now"))?;++ let mut result_count = 0;++ let result: Vec<RecordBatch> = {+ let mut local_result = vec![];++ for query in run_queries {+ match save_results {+ true => {+ debug!(+ "Running query (saving results) {}-{}: {query}",+ self.group, self.subgroup+ );++ let df = ctx.sql(query).await?;+ if !self.expect.is_empty() {+ let physical_plan = df.create_physical_plan().await?;+ self.validate_expected_plan(&physical_plan)?;+ }++ let result_schema = Arc::new(df.schema().as_arrow().clone());+ let mut batches = df.collect().await?;+ let trimmed = query.trim_start();++ // save the output for select/with queries+ if starts_with_ignore_ascii_case(trimmed, "select")+ || starts_with_ignore_ascii_case(trimmed, "with")+ {+ if batches.is_empty() {+ batches.push(RecordBatch::new_empty(result_schema));+ }+ let row_count_for_query =+ batches.iter().map(RecordBatch::num_rows).sum::<usize>();+ debug!(+ "Persisting {} batches ({} rows)...",+ batches.len(),+ row_count_for_query+ );++ result_count = row_count_for_query;+ local_result = batches;+ }+ }+ false => {+ debug!(+ "Running query (ignoring results) {}-{}: {query}",+ self.group, self.subgroup+ );++ result_count = self+ .execute_sql_without_result_buffering(query, ctx)+ .await?;+ }+ }+ }++ Ok::<Vec<RecordBatch>, DataFusionError>(local_result)+ }?;++ debug!("Results have {result_count} rows");++ // Store results for verification+ self.last_results = Some(result);++ Ok(())+ }++ /// Calls run and persists results to disk as a CSV file.+ ///+ /// Requires that the benchmark defines a `result` or `result_query`.+ /// Registers the results in a memory table and writes them to disk with+ /// pipe delimiters and a header row.+ ///+ /// # Errors+ /// Returns an error if no results are available or if writing to the+ /// target path fails.+ pub async fn persist(&mut self, ctx: &SessionContext) -> Result<()> {+ self.run(ctx, true).await?;++ // Check if we have result queries to persist for+ if self.result_queries.is_empty() {+ info!("No result paths to persist");+ return Ok(());+ }++ let results = self+ .last_results+ .as_ref()+ .expect("run should store last_results after successful execution");++ let query = &self.result_queries[0];+ let path = query.path.as_ref().ok_or_else(|| {+ exec_datafusion_err!(+ "Unable to persist results from query '{}', no result specified",+ query.query+ )+ })?;++ info!("Persisting results for query to {path}");++ let first_batch = results+ .first()+ .ok_or_else(|| exec_datafusion_err!("Results should be loaded"))?;++ let schema = first_batch.schema();+ let provider = MemTable::try_new(schema, vec![results.clone()])?;++ ctx.register_table("persist_data", Arc::new(provider))?;++ let df = ctx.table("persist_data").await?;+ df.write_csv(+ path,+ DataFrameWriteOptions::new(),+ Some(+ CsvOptions::default()+ .with_delimiter(b'|')+ .with_has_header(true),+ ),+ )+ .await?;++ ctx.deregister_table("persist_data")?;++ Ok(())+ }++ /// Verifies persisted results against expected values.+ ///+ /// Executes the `result_query` or uses the stored last run results, then+ /// compares actual output rows to the expected values defined in the+ /// benchmark file.+ ///+ /// # Errors+ /// Returns an error if no results are available or if the actual and+ /// expected results differ in count or content.+ pub async fn verify(&mut self, ctx: &SessionContext) -> Result<()> {+ // Check if we have result queries to verify+ if self.result_queries.is_empty() {+ return Ok(());+ }++ if self.last_results.is_none() {+ return Err(exec_datafusion_err!(+ "No results available for verification. Run the benchmark first."+ ));+ }++ info!("Verifying results...");++ self.load_expected_result_files(ctx).await?;++ // Get the first result query (assuming only one for now)+ let query = &self.result_queries[0];+ let formatted_actual_results = if !query.query.trim().is_empty() {+ let results = ctx.sql(&query.query).await?.collect().await?;+ format_record_batches(&results)+ } else {+ let actual_results = self+ .last_results+ .as_ref()+ .expect("last_results should be present after successful run");+ format_record_batches(actual_results)+ }?;++ Self::compare_results(query, &formatted_actual_results, &query.expected_result)+ }++ /// Runs `cleanup` queries to reset state after the benchmark run.+ pub async fn cleanup(&mut self, ctx: &SessionContext) -> Result<()> {+ info!("Running cleanup...");++ let cleanup_queries = self.queries.get(&QueryDirective::Cleanup);++ if let Some(queries) = cleanup_queries {+ for query in queries {+ let _ = ctx.sql(query).await?.collect().await?;+ }+ }++ Ok(())+ }++ async fn load_expected_result_files(&mut self, ctx: &SessionContext) -> Result<()> {+ for query in &mut self.result_queries {+ if query.query.trim().is_empty() {+ let Some(path) = query.path.clone() else {+ continue;+ };++ let loaded_query =+ read_query_from_file(ctx, path, &HashMap::new()).await?;+ query.column_count = loaded_query.column_count;+ query.expected_result = loaded_query.expected_result;+ }+ }++ Ok(())+ }++ fn compare_results(+ query: &BenchmarkQuery,+ actual_results: &[Vec<String>],+ expected_results: &[Vec<String>],+ ) -> Result<()> {+ if actual_results.is_empty() && expected_results.is_empty() {+ return Ok(());+ }++ // Compare row count+ if actual_results.len() != expected_results.len() {+ return Err(exec_datafusion_err!(+ "Error in result: expected {} rows but got {} for query {}",+ expected_results.len(),+ actual_results.len(),+ query.query+ ));+ }++ // Compare values+ let zipped = actual_results+ .iter()+ .enumerate()+ .zip(expected_results.iter());++ for ((row_idx, actual), expected) in zipped {+ trace!(+ "row {}\nactual: {actual:?}\nexpected: {expected:?}",+ row_idx + 1+ );++ // Compare column count+ if actual.len() != expected.len() {+ return Err(exec_datafusion_err!(+ "Error in result: expected {} columns but got {} for query {}",+ expected.len(),+ actual.len(),+ query.query+ ));+ }++ for (col_idx, expected_val) in+ expected.iter().enumerate().take(query.column_count)+ {+ // The row-width check above guarantees this index exists.+ let actual_val = &actual[col_idx];++ trace!("actual_val = {actual_val:?}\nexpected_val = {expected_val:?}");++ if (expected_val == "NULL" && actual_val.is_empty())+ || (expected_val == actual_val)+ || (expected_val == "(empty)"+ && (actual_val.is_empty() || actual_val == "NULL"))+ {+ continue;+ }++ return Err(exec_datafusion_err!(+ "Error in result on row {}, column {} running query \"{}\": expected value \+ \"{expected_val}\" but got value \"{actual_val}\" in row: {actual:?}",+ row_idx + 1,+ col_idx + 1,+ query.query+ ));+ }+ }++ Ok(())+ }++ async fn process_file(&mut self, ctx: &SessionContext, path: &Path) -> Result<()> {+ debug!("Processing file {}", path.display());++ let mut replacement_mapping = self.replacement_mapping.clone();+ insert_replacement(+ &mut replacement_mapping,+ "FILE_PATH",+ path.to_string_lossy().into_owned(),+ );++ let mut reader = BenchmarkFileReader::new(path, replacement_mapping)?;+ let mut line = String::with_capacity(1024);+ let mut reader_result = reader.read_line(&mut line);++ while let Some(result) = reader_result {+ match result {+ Ok(_) => {+ if !is_blank_or_comment_line(&line) {+ // boxing required because of recursion+ Box::pin(self.process_line(ctx, &mut reader, &mut line)).await?;+ }+ }+ Err(e) => return Err(e),+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(&mut line);+ }++ Ok(())+ }++ async fn process_line(+ &mut self,+ ctx: &SessionContext,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ ) -> Result<()> {+ // Split the line into directive and arguments.+ let cloned_line = line.trim_start().to_string();+ let splits: Vec<&str> = cloned_line.split_whitespace().collect();++ BenchmarkDirective::select(reader, splits[0])?+ .process(ctx, self, reader, line, &splits)+ .await+ }++ fn process_query(&mut self, splits: &[&str], mut query: String) -> Result<()> {+ debug!("Processing query {query}");++ // Trim and validate.+ query = query.trim().to_string();+ if query.is_empty() {+ return Ok(());+ }++ // remove comments+ query = query+ .lines()+ .filter(|line| !is_comment_line(line))+ .collect::<Vec<_>>()+ .join("\n");++ if query.trim().is_empty() {+ return Ok(());+ }++ query = process_replacements(&query, self.replacement_mapping())?;++ let directive = QueryDirective::parse(splits[0]).ok_or_else(|| {+ exec_datafusion_err!("Invalid query directive: {}", splits[0])+ })?;++ self.queries.entry(directive).or_default().push(query);++ Ok(())+ }++ fn validate_expected_plan(&self, physical_plan: &impl Debug) -> Result<()> {+ if self.expect.is_empty() {+ return Ok(());+ }++ let plan_string = format!("{physical_plan:#?}");++ for exp_str in &self.expect {+ if !plan_string.contains(exp_str) {+ return Err(exec_datafusion_err!(+ "The query physical plan does not contain the expected string '{exp_str}'. Physical plan: {plan_string}"+ ));+ }+ }++ Ok(())+ }++ async fn execute_sql_without_result_buffering(+ &self,+ sql: &str,+ ctx: &SessionContext,+ ) -> Result<usize> {+ let mut row_count = 0;++ let df = ctx.sql(sql).await?;+ let physical_plan = df.create_physical_plan().await?;++ self.validate_expected_plan(&physical_plan)?;+ let mut stream = execute_stream(physical_plan, ctx.task_ctx())?;++ while let Some(batch) = stream.next().await {+ row_count += batch?.num_rows();++ // Evaluate the result and do nothing, the result will be dropped+ // to reduce memory pressure+ }++ Ok(row_count)+ }++ pub fn name(&self) -> &str {+ &self.name+ }++ pub fn group(&self) -> &str {+ &self.group+ }++ pub fn subgroup(&self) -> &str {+ &self.subgroup+ }++ pub fn benchmark_path(&self) -> &Path {+ &self.benchmark_path+ }++ pub fn replacement_mapping(&self) -> &HashMap<String, String> {+ &self.replacement_mapping+ }++ pub fn queries(&self) -> &HashMap<QueryDirective, Vec<String>> {+ &self.queries+ }++ pub fn result_queries(&self) -> &[BenchmarkQuery] {+ &self.result_queries+ }++ pub fn assert_queries(&self) -> &[BenchmarkQuery] {+ &self.assert_queries+ }++ pub fn is_loaded(&self) -> bool {+ self.is_loaded+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum QueryDirective {+ Load,+ Run,+ Init,+ Cleanup,+}++impl QueryDirective {+ fn parse(value: &str) -> Option<Self> {+ if value.eq_ignore_ascii_case("load") {+ Some(Self::Load)+ } else if value.eq_ignore_ascii_case("init") {+ Some(Self::Init)+ } else if value.eq_ignore_ascii_case("run") {+ Some(Self::Run)+ } else if value.eq_ignore_ascii_case("cleanup") {+ Some(Self::Cleanup)+ } else {+ None+ }+ }++ fn as_str(self) -> &'static str {+ match self {+ Self::Load => "load",+ Self::Run => "run",+ Self::Init => "init",+ Self::Cleanup => "cleanup",+ }+ }+}++enum BenchmarkDirective {+ Load,+ Run,+ Init,+ Cleanup,+ Name,+ Group,+ Subgroup,+ Expect,+ Assert,+ ResultQuery,+ Results,+ Template,+ Include,+ Echo,+}++impl BenchmarkDirective {+ fn select(+ reader: &BenchmarkFileReader,+ directive: &str,+ ) -> Result<BenchmarkDirective> {+ if directive.eq_ignore_ascii_case("load") {+ Ok(BenchmarkDirective::Load)+ } else if directive.eq_ignore_ascii_case("run") {+ Ok(BenchmarkDirective::Run)+ } else if directive.eq_ignore_ascii_case("init") {+ Ok(BenchmarkDirective::Init)+ } else if directive.eq_ignore_ascii_case("cleanup") {+ Ok(BenchmarkDirective::Cleanup)+ } else if directive.eq_ignore_ascii_case("name") {+ Ok(BenchmarkDirective::Name)+ } else if directive.eq_ignore_ascii_case("group") {+ Ok(BenchmarkDirective::Group)+ } else if directive.eq_ignore_ascii_case("subgroup") {+ Ok(BenchmarkDirective::Subgroup)+ } else if directive.eq_ignore_ascii_case("expect_plan") {+ Ok(BenchmarkDirective::Expect)+ } else if directive.eq_ignore_ascii_case("assert") {+ Ok(BenchmarkDirective::Assert)+ } else if directive.eq_ignore_ascii_case("result_query") {+ Ok(BenchmarkDirective::ResultQuery)+ } else if directive.eq_ignore_ascii_case("result") {+ Ok(BenchmarkDirective::Results)+ } else if directive.eq_ignore_ascii_case("template") {+ Ok(BenchmarkDirective::Template)+ } else if directive.eq_ignore_ascii_case("include") {+ Ok(BenchmarkDirective::Include)+ } else if directive.eq_ignore_ascii_case("echo") {+ Ok(BenchmarkDirective::Echo)+ } else {+ Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(&format!("Unrecognized command: {directive}"))+ ))+ }+ }++ async fn process(+ &self,+ ctx: &SessionContext,+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ splits: &[&str],+ ) -> Result<()> {+ trace!("-- handling {}", splits[0]);++ match self {+ BenchmarkDirective::Load+ | BenchmarkDirective::Run+ | BenchmarkDirective::Init+ | BenchmarkDirective::Cleanup => {+ Self::process_query_directive(bench, reader, line, splits)+ }+ BenchmarkDirective::Name => Self::process_metadata_value(+ bench,+ reader,+ line,+ "name",+ "BENCH_NAME",+ "name must be followed by a value",+ ),+ BenchmarkDirective::Group => Self::process_metadata_value(+ bench,+ reader,+ line,+ "group",+ "BENCH_GROUP",+ "group must be followed by a value",+ ),+ BenchmarkDirective::Subgroup => Self::process_metadata_value(+ bench,+ reader,+ line,+ "subgroup",+ "BENCH_SUBGROUP",+ "subgroup must be followed by a value",+ ),+ BenchmarkDirective::Expect => Self::process_expect(bench, reader, splits),+ BenchmarkDirective::Assert => {+ Self::process_assert(bench, reader, line, splits)+ }+ BenchmarkDirective::ResultQuery => {+ Self::process_result_query(bench, reader, line, splits)+ }+ BenchmarkDirective::Results => Self::process_results(bench, reader, splits),+ BenchmarkDirective::Template => {+ Self::process_template(ctx, bench, reader, line, splits).await+ }+ BenchmarkDirective::Include => {+ Self::process_include(ctx, bench, reader, splits).await+ }+ BenchmarkDirective::Echo => Self::process_echo(bench, reader, splits),+ }+ }++ fn process_query_directive(+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ splits: &[&str],+ ) -> Result<()> {+ let directive = QueryDirective::parse(splits[0]).ok_or_else(|| {+ exec_datafusion_err!("Invalid query directive: {}", splits[0])+ })?;++ if directive == QueryDirective::Run && bench.queries.contains_key(&directive) {+ return Err(exec_datafusion_err!(+ "Multiple calls to run in the same benchmark file"+ ));+ }++ line.clear();++ // Read the query body until a blank line or EOF.+ let mut query = String::new();+ let mut reader_result = reader.read_line(line);++ loop {+ match reader_result {+ Some(Ok(_)) => {+ if is_comment_line(line) {+ // comment, ignore+ } else if is_blank_line(line) {+ break;+ } else {+ query.push_str(line);+ query.push('\n');+ }+ }+ Some(Err(e)) => return Err(e),+ None => break,+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(line);+ }++ // Optional file parameter.+ if splits.len() > 1 && !splits[1].is_empty() {+ if !query.trim().is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(&format!(+ "{} directive must use either a query file or inline SQL, not both",+ directive.as_str()+ ))+ ));+ }++ debug!("Processing {} file: {}", splits[0], splits[1]);++ let query_file = fs::read_to_string(splits[1]).map_err(|e| {+ exec_datafusion_err!("Failed to read query file {}: {e}", splits[1])+ })?;+ let query_file = query_file.replace("\r\n", "\n");++ // some files have multiple queries, split apart+ for query in split_query_statements(&query_file) {+ bench.process_query(splits, query.to_string())?;+ }+ } else if directive == QueryDirective::Run {+ for query in split_query_statements(&query) {+ bench.process_query(splits, query.to_string())?;+ }+ } else {+ bench.process_query(splits, query)?;+ }++ Ok(())+ }++ fn process_metadata_value(+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &str,+ directive: &str,+ replacement_key: &str,+ message: &str,+ ) -> Result<()> {+ let value =+ directive_value(reader, line.trim_start(), directive, message)?.to_string();++ match directive {+ "name" => bench.name.clone_from(&value),+ "group" => bench.group.clone_from(&value),+ "subgroup" => bench.subgroup.clone_from(&value),+ _ => unreachable!("unsupported metadata directive: {directive}"),+ }++ insert_replacement(+ &mut bench.replacement_mapping,+ replacement_key,+ value.clone(),+ );+ insert_replacement(&mut reader.replacements, replacement_key, value);++ Ok(())+ }++ fn process_expect(+ bench: &mut SqlBenchmark,+ reader: &BenchmarkFileReader,+ splits: &[&str],+ ) -> Result<()> {+ trace!("-- handling {}", splits[0]);++ if splits.len() <= 1 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "expect_plan must be followed by a string to search in the physical plan"+ )+ ));+ }++ bench.expect.push(splits[1..].join(" ").to_string());++ Ok(())+ }++ fn process_assert(+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ splits: &[&str],+ ) -> Result<()> {+ // count the amount of columns based on character count. The actual+ // character used is irrelevant.+ if splits.len() <= 1 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "assert must be followed by a column count (e.g. assert III)"+ )+ ));+ }++ line.clear();++ // read the actual query+ let mut found_break = false;+ let mut sql = String::new();+ let mut reader_result = reader.read_line(line);++ loop {+ match reader_result {+ Some(Ok(_)) => {+ if line.trim() == "----" {+ found_break = true;+ break;+ }+ sql.push('\n');+ sql.push_str(line);+ }+ Some(Err(e)) => return Err(e),+ None => break,+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(line);+ }++ if !found_break {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "assert must be followed by a query and a result (separated by ----)"+ )+ ));+ }++ bench+ .assert_queries+ .push(read_query_from_reader(reader, &sql, splits[1])?);++ Ok(())+ }++ fn process_results(+ bench: &mut SqlBenchmark,+ reader: &BenchmarkFileReader,+ splits: &[&str],+ ) -> Result<()> {+ if splits.len() <= 1 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "result must be followed by a path to a result file"+ )+ ));+ }++ if !bench.result_queries.is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception("multiple results found")+ ));+ }++ let path = process_replacements(splits[1], &bench.replacement_mapping)?;++ bench.result_queries.push(BenchmarkQuery {+ path: Some(path),+ query: String::new(),+ column_count: 0,+ expected_result: vec![],+ });++ Ok(())+ }++ fn process_result_query(+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ splits: &[&str],+ ) -> Result<()> {+ if splits.len() <= 1 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "result_query must be followed by a column count (e.g. result_query III)"+ )+ ));+ }++ line.clear();++ let mut sql = String::new();+ let mut found_break = false;+ let mut reader_result = reader.read_line(line);++ loop {+ match reader_result {+ Some(Ok(_)) => {+ if line.trim() == "----" {+ found_break = true;+ break;+ }+ sql.push_str(line);+ sql.push('\n');+ }+ Some(Err(e)) => return Err(e),+ None => break,+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(line);+ }++ if !found_break {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "result_query must be followed by a query and a result (separated by ----)"+ )+ ));+ }++ let result_check = read_query_from_reader(reader, &sql, splits[1])?;++ if !bench.result_queries.is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception("multiple results found")+ ));+ }+ bench.result_queries.push(result_check);++ Ok(())+ }++ async fn process_template(+ ctx: &SessionContext,+ bench: &mut SqlBenchmark,+ reader: &mut BenchmarkFileReader,+ line: &mut String,+ splits: &[&str],+ ) -> Result<()> {+ if splits.len() != 2 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception("template requires a single template path")+ ));+ }++ // template: update the path to read+ bench.benchmark_path = PathBuf::from(splits[1]);++ line.clear();++ // now read parameters+ let mut reader_result = reader.read_line(line);++ loop {+ match reader_result {+ Some(Ok(_)) => {+ if is_comment_line(line) {+ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(line);+ continue;+ }+ if is_blank_line(line) {+ break;+ }++ let Some((key, value)) = line.trim_start().split_once('=') else {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception(+ "Expected a template parameter in the form of X=Y"+ )+ ));+ };+ insert_replacement(+ &mut bench.replacement_mapping,+ key.trim(),+ value.trim().to_string(),+ );+ }+ Some(Err(e)) => return Err(e),+ None => break,+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(line);+ }++ // restart the load from the template file+ Box::pin(bench.process_file(ctx, Path::new(splits[1]))).await+ }++ async fn process_include(+ ctx: &SessionContext,+ bench: &mut SqlBenchmark,+ reader: &BenchmarkFileReader,+ splits: &[&str],+ ) -> Result<()> {+ if splits.len() != 2 || splits[1].is_empty() {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception("include requires a single argument")+ ));+ }++ Box::pin(bench.process_file(ctx, Path::new(splits[1]))).await+ }++ fn process_echo(+ bench: &mut SqlBenchmark,+ reader: &BenchmarkFileReader,+ splits: &[&str],+ ) -> Result<()> {+ if splits.len() < 2 {+ return Err(exec_datafusion_err!(+ "{}",+ reader.format_exception("Echo requires an argument")+ ));+ }++ bench.echo.push(splits[1..].join(" "));++ Ok(())+ }+}++struct BenchmarkFileReader {+ path: PathBuf,+ reader: BufReader<File>,+ line_nr: usize,+ replacements: HashMap<String, String>,+}++impl BenchmarkFileReader {+ fn new<P: Into<PathBuf>>(+ path: P,+ replacements: HashMap<String, String>,+ ) -> Result<Self> {+ let path = path.into();+ let file = OpenOptions::new().read(true).open(&path)?;++ Ok(Self {+ path,+ reader: BufReader::new(file),+ line_nr: 0,+ replacements,+ })+ }++ /// Read the next line, applying replacements and removing line terminators.+ fn read_line(&mut self, line: &mut String) -> Option<Result<()>> {+ match self.reader.read_line(line) {+ Ok(0) => None,+ Ok(_) => {+ self.line_nr += 1;++ // Trim newline and carriage return without changing other content.+ let trimmed_len = line.trim_end_matches(['\n', '\r']).len();+ line.truncate(trimmed_len);++ match process_replacements(line, &self.replacements) {+ Ok(l) => {+ *line = l;+ Some(Ok(()))+ }+ Err(error) => Some(Err(error)),+ }+ }+ Err(e) => Some(Err(e.into())),+ }+ }++ fn format_exception(&self, msg: &str) -> String {+ format!("{}:{} - {}", self.path.display(), self.line_nr, msg)+ }+}++#[derive(Debug, Clone)]+pub struct BenchmarkQuery {+ path: Option<String>,+ query: String,+ column_count: usize,+ expected_result: Vec<Vec<String>>,+}++// ---- utility function below++fn directive_value<'a>(+ reader: &BenchmarkFileReader,+ line: &'a str,+ directive: &str,+ message: &str,+) -> Result<&'a str> {+ let value = line+ .get(..directive.len())+ .filter(|prefix| prefix.eq_ignore_ascii_case(directive))+ .and_then(|_| line.get(directive.len()..))+ .map(str::trim)+ .filter(|s| !s.is_empty())+ .ok_or_else(|| exec_datafusion_err!("{}", reader.format_exception(message)))?;++ Ok(value)+}++fn parse_group_from_path(path: &Path, benchmark_directory: &Path) -> String {+ let mut group_name = String::new();+ let mut parent = path.parent();++ while let Some(p) = parent {+ if path_ends_with_ignore_ascii_case(p, benchmark_directory) {+ break;+ }++ if let Some(dir_name) = p.file_name() {+ group_name = dir_name.to_string_lossy().into_owned();+ }++ parent = p.parent();+ }++ if group_name.is_empty() {+ warn!("Unable to find group name in path: {}", path.display());+ }++ group_name+}++fn path_ends_with_ignore_ascii_case(path: &Path, suffix: &Path) -> bool {+ let mut path_components = path.components().rev();++ for suffix_component in suffix.components().rev() {+ let Some(path_component) = path_components.next() else {+ return false;+ };++ if !path_component+ .as_os_str()+ .to_string_lossy()+ .eq_ignore_ascii_case(&suffix_component.as_os_str().to_string_lossy())+ {+ return false;+ }+ }++ true+}++fn starts_with_ignore_ascii_case(input: &str, prefix: &str) -> bool {+ input+ .get(..prefix.len())+ .is_some_and(|value| value.eq_ignore_ascii_case(prefix))+}++fn split_query_statements(sql: &str) -> impl Iterator<Item = &str> {+ sql.split("\n\n")+ .flat_map(|query| {+ query+ .split_inclusive(";\n")+ .map(|part| part.trim_end_matches('\n'))+ })+ .filter(|query| !query.trim().is_empty())+}++fn is_blank_line(line: &str) -> bool {+ line.trim().is_empty()+}++fn is_comment_line(line: &str) -> bool {+ let line = line.trim_start();+ line.starts_with('#') || line.starts_with("--")+}++fn is_blank_or_comment_line(line: &str) -> bool {+ is_blank_line(line) || is_comment_line(line)+}++fn insert_replacement(+ replacement_map: &mut HashMap<String, String>,+ key: &str,+ value: String,+) {+ replacement_map.insert(key.to_lowercase(), value);+}++fn replace_all<E>(+ re: &Regex,+ haystack: &str,+ replacement: impl Fn(®ex::Captures) -> Result<String, E>,+) -> Result<String, E> {+ let mut new = String::with_capacity(haystack.len());+ let mut last_match = 0;++ for caps in re.captures_iter(haystack) {+ let m = caps.get(0).unwrap();++ new.push_str(&haystack[last_match..m.start()]);+ new.push_str(&replacement(&caps)?);++ last_match = m.end();+ }++ new.push_str(&haystack[last_match..]);++ Ok(new)+}++static TRUE_FALSE_REPLACEMENT_RE: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r"\$\{(\w+)(?::-([^|}]+))?\|([^|]+)\|([^}]+)}")+ .expect("Regex failed to compile")+});++static VARIABLE_REPLACEMENT_RE: LazyLock<Regex> = LazyLock::new(|| {+ Regex::new(r"\$\{(\w+)(?::-([^}]+))?}").expect("Regex failed to compile")+});++/// Replace all `${KEY}` or `${KEY:-default}` placeholders in a string according to the mapping.+/// Also handles `${KEY:-default|True value|false value}` syntax.+fn process_replacements(+ input: &str,+ replacement_map: &HashMap<String, String>,+) -> Result<String> {+ process_replacements_with_env(input, replacement_map, |key| std::env::var(key).ok())+}++fn process_replacements_with_env(+ input: &str,+ replacement_map: &HashMap<String, String>,+ get_env: impl Fn(&str) -> Option<String>,+) -> Result<String> {+ debug!("processing replacements for line '{input}'");++ // handle ${VAR:-default|true value|false value} syntax+ let replacement = |caps: ®ex::Captures| -> Result<String> {+ let key = &caps[1];+ let default = caps.get(2).map(|m| m.as_str().to_string());+ let true_val = &caps[3];+ let false_val = &caps[4];++ let value = lookup_replacement_value(key, replacement_map, &get_env).or(default);++ match value {+ Some(v) if v.eq_ignore_ascii_case("true") => Ok(true_val.to_string()),+ Some(_) => Ok(false_val.to_string()),+ None => Err(exec_datafusion_err!("Missing value for key '{key}'")),+ }+ };+ let input = replace_all(&TRUE_FALSE_REPLACEMENT_RE, input, replacement)?;++ // handle ${KEY} and ${KEY:-default}`+ let replacement = |caps: ®ex::Captures| -> Result<String> {+ let key = &caps[1];+ let default = caps.get(2);++ if let Some(v) = lookup_replacement_value(key, replacement_map, &get_env) {+ return Ok(v.to_string());+ }++ // use default if it was set+ if let Some(def) = default {+ Ok(def.as_str().to_string())+ } else {+ Err(exec_datafusion_err!("Missing value for key '{key}'"))+ }+ };++ replace_all(&VARIABLE_REPLACEMENT_RE, &input, replacement)+}++fn lookup_replacement_value(+ key: &str,+ replacement_map: &HashMap<String, String>,+ get_env: &impl Fn(&str) -> Option<String>,+) -> Option<String> {+ if let Some(v) = replacement_map.get(&key.to_lowercase()) {+ return Some(v.to_string());+ }++ // look in env variables+ get_env(&key.to_uppercase())+}++fn read_query_from_reader(+ reader: &mut BenchmarkFileReader,+ sql: &str,+ header: &str,+) -> Result<BenchmarkQuery> {+ let column_count = header.len();+ let mut expected_result = vec![];+ let mut line = String::new();+ let mut reader_result = reader.read_line(&mut line);++ loop {+ match reader_result {+ Some(Ok(_)) => {+ if is_comment_line(&line) {+ // comment, ignore+ } else if is_blank_line(&line) {+ break;+ } else {+ let result_splits: Vec<&str> = line.split(['\t', '|']).collect();++ if result_splits.len() != column_count {+ return Err(exec_datafusion_err!(+ "{} {line}",+ reader.format_exception(&format!(+ "expected {} values but got {}",+ column_count,+ result_splits.len(),+ ))+ ));+ }++ expected_result+ .push(result_splits.into_iter().map(|s| s.to_string()).collect());+ }+ }+ Some(Err(e)) => return Err(e),+ None => break,+ }++ // Clear the line buffer for the next iteration.+ line.clear();+ reader_result = reader.read_line(&mut line);+ }++ Ok(BenchmarkQuery {+ path: None,+ query: sql.to_string(),+ column_count,+ expected_result,+ })+}++async fn read_query_from_file(+ ctx: &SessionContext,+ path: impl AsRef<Path>,+ replacement_mapping: &HashMap<String, String>,+) -> Result<BenchmarkQuery> {+ // Process replacements in file path+ let path = path.as_ref().to_string_lossy();+ let path = process_replacements(&path, replacement_mapping)?;+ let df: DataFrame = ctx+ .read_csv(+ path.clone(),+ CsvReadOptions::new()+ .has_header(true)+ .delimiter(b'|')+ .null_regex(Some("NULL".to_string()))+ // we only want string values, we do not want to infer the schema+ .schema_infer_max_records(0),+ )+ .await?;++ // Get schema to determine column count+ let schema = df.schema();+ let column_count = schema.fields().len();++ if column_count == 0 {+ return Err(exec_datafusion_err!(+ "Result file {path} did not contain any columns"+ ));+ }++ // Execute and collect results+ let batches = df.collect().await?;+ // Convert record batches to string vectors+ let expected_result = format_record_batches(&batches)?;++ Ok(BenchmarkQuery {+ path: Some(path),+ query: String::new(),+ column_count,+ expected_result,+ })+}++fn format_record_batches(+ batches: &[RecordBatch],+) -> Result<Vec<Vec<String>>, DataFusionError> {+ let mut expected_result = vec![];+ let arrow_format_options = FormatOptions::default()+ .with_null("NULL")+ .with_display_error(true);++ for batch in batches {+ let schema = batch.schema_ref();++ let formatters = batch+ .columns()+ .iter()+ .zip(schema.fields().iter())+ .map(|(c, field)| make_array_formatter(c, &arrow_format_options, Some(field)))+ .collect::<Result<Vec<_>, ArrowError>>()?;++ for row in 0..batch.num_rows() {+ let mut cells = vec![];+ for formatter in &formatters {+ cells.push(formatter.value(row).to_string());+ }+ expected_result.push(cells);+ }+ }++ Ok(expected_result)+}++fn make_array_formatter<'a>(+ array: &'a dyn Array,+ options: &FormatOptions<'a>,+ field: Option<&'a Field>,+) -> Result<ArrayFormatter<'a>, ArrowError> {+ match options.formatter_factory() {+ None => ArrayFormatter::try_new(array, options),+ Some(formatters) => formatters+ .create_array_formatter(array, options, field)+ .transpose()+ .unwrap_or_else(|| ArrayFormatter::try_new(array, options)),+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use datafusion::prelude::SessionContext;+ use std::fs;+ use std::path::{Path, PathBuf};+ use tempfile::{TempDir, tempdir};++ fn write_test_file(temp_dir: &TempDir, name: &str, contents: &str) -> PathBuf {+ let path = temp_dir.path().join(name);+ fs::write(&path, contents).expect("failed to write benchmark test file");+ path+ }++ async fn parse_benchmark_file(path: &Path) -> Result<SqlBenchmark> {+ let ctx = SessionContext::new();+ let path_string = path.to_string_lossy().into_owned();+ SqlBenchmark::new(&ctx, &path_string, "/tmp").await+ }++ async fn parse_benchmark(contents: &str) -> Result<SqlBenchmark> {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let path = write_test_file(&temp_dir, "parser.benchmark", contents);++ parse_benchmark_file(&path).await+ }++ async fn assert_parse_error(contents: &str, expected_message: &str) {+ let error = parse_benchmark(contents)+ .await+ .expect_err("benchmark parsing should fail");++ let message = error.to_string();+ assert!(+ message.contains(expected_message),+ "expected error containing {expected_message:?}, got {message:?}"+ );+ }++ fn assert_result_error_contains<T: Debug>(result: Result<T>, expected_message: &str) {+ let error = result.expect_err("operation should fail");+ let message = error.to_string();+ assert!(+ message.contains(expected_message),+ "expected error containing {expected_message:?}, got {message:?}"+ );+ }++ fn formatted_last_results(benchmark: &SqlBenchmark) -> Vec<Vec<String>> {+ format_record_batches(+ benchmark+ .last_results+ .as_ref()+ .expect("last results should be set"),+ )+ .expect("results should format")+ }++ fn read_all_files_in_dir(path: &Path) -> String {+ let mut entries = fs::read_dir(path)+ .expect("directory should be readable")+ .filter_map(Result::ok)+ .map(|entry| entry.path())+ .filter(|path| path.is_file())+ .collect::<Vec<_>>();+ entries.sort();++ let mut contents = String::new();+ for path in entries {+ contents+ .push_str(&fs::read_to_string(path).expect("file should be readable"));+ }+ contents+ }++ fn replacement_map(entries: &[(&str, &str)]) -> HashMap<String, String> {+ let mut replacements = HashMap::new();+ for (key, value) in entries {+ insert_replacement(&mut replacements, key, value.to_string());+ }+ replacements+ }++ fn env_map(entries: &[(&str, &str)]) -> HashMap<String, String> {+ entries+ .iter()+ .map(|(key, value)| (key.to_string(), value.to_string()))+ .collect()+ }++ // Replacement tests cover benchmark variable expansion syntax.++ #[test]+ fn process_replacements_replaces_map_values_case_insensitively() {+ let replacements = replacement_map(&[+ ("BENCH_NAME", "tpch"),+ ("QUERY_NUMBER_PADDED", "01"),+ ("format_1", "parquet"),+ ]);++ let actual = process_replacements_with_env(+ "${bench_name}/q${query_number_padded}.${FORMAT_1}",+ &replacements,+ |_| None,+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "tpch/q01.parquet");+ }++ #[test]+ fn process_replacements_uses_env_when_map_value_is_missing() {+ let replacements = HashMap::new();+ let env = env_map(&[("DATA_DIR", "/tmp/data")]);++ let actual = process_replacements_with_env(+ "${data_dir}/lineitem.parquet",+ &replacements,+ |key| env.get(key).cloned(),+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "/tmp/data/lineitem.parquet");+ }++ #[test]+ fn process_replacements_prefers_map_over_env() {+ let replacements = replacement_map(&[("BENCH_SIZE", "10")]);+ let env = env_map(&[("BENCH_SIZE", "100")]);++ let actual =+ process_replacements_with_env("sf${BENCH_SIZE}", &replacements, |key| {+ env.get(key).cloned()+ })+ .expect("replacement should succeed");++ assert_eq!(actual, "sf10");+ }++ #[test]+ fn process_replacements_uses_default_for_missing_variable() {+ let replacements = HashMap::new();++ let actual = process_replacements_with_env(+ "load_${BENCH_SUBGROUP:-groupby}_${FILE_TYPE:-csv}.sql",+ &replacements,+ |_| None,+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_groupby_csv.sql");+ }++ #[test]+ fn process_replacements_reports_missing_variable_without_default() {+ let replacements = HashMap::new();++ let error = process_replacements_with_env("${MISSING}", &replacements, |_| None)+ .expect_err("replacement should fail");++ assert!(+ error+ .to_string()+ .contains("Missing value for key 'MISSING'"),+ "unexpected error: {error}"+ );+ }++ #[test]+ fn process_replacements_applies_true_false_true_branch() {+ let replacements = HashMap::new();+ let env = env_map(&[("USE_PARQUET", "TrUe")]);++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-false|parquet|csv}.sql",+ &replacements,+ |key| env.get(key).cloned(),+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_parquet.sql");+ }++ #[test]+ fn process_replacements_applies_true_false_false_branch() {+ let replacements = HashMap::new();+ let env = env_map(&[("USE_PARQUET", "false")]);++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-true|parquet|csv}.sql",+ &replacements,+ |key| env.get(key).cloned(),+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_csv.sql");+ }++ #[test]+ fn process_replacements_uses_map_for_true_false_branch() {+ let replacements = replacement_map(&[("USE_PARQUET", "true")]);++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-false|parquet|csv}.sql",+ &replacements,+ |_| None,+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_parquet.sql");+ }++ #[test]+ fn process_replacements_prefers_map_over_env_for_true_false_branch() {+ let replacements = replacement_map(&[("USE_PARQUET", "false")]);+ let env = env_map(&[("USE_PARQUET", "true")]);++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-true|parquet|csv}.sql",+ &replacements,+ |key| env.get(key).cloned(),+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_csv.sql");+ }++ #[test]+ fn process_replacements_uses_true_false_default_for_missing_true_value() {+ let replacements = HashMap::new();++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-true|parquet|csv}.sql",+ &replacements,+ |_| None,+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_parquet.sql");+ }++ #[test]+ fn process_replacements_uses_true_false_default_for_missing_false_value() {+ let replacements = HashMap::new();++ let actual = process_replacements_with_env(+ "load_${USE_PARQUET:-false|parquet|csv}.sql",+ &replacements,+ |_| None,+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "load_csv.sql");+ }++ #[test]+ fn process_replacements_reports_missing_true_false_variable_without_default() {+ let replacements = HashMap::new();++ let error = process_replacements_with_env(+ "load_${USE_PARQUET|parquet|csv}.sql",+ &replacements,+ |_| None,+ )+ .expect_err("replacement should fail");++ assert!(+ error+ .to_string()+ .contains("Missing value for key 'USE_PARQUET'"),+ "unexpected error: {error}"+ );+ }++ #[test]+ fn process_replacements_resolves_variables_after_true_false_replacement() {+ let replacements = replacement_map(&[("FILE_TYPE", "parquet")]);+ let env = env_map(&[("USE_TYPED_PATH", "true")]);++ let actual = process_replacements_with_env(+ "${USE_TYPED_PATH:-false|data.${FILE_TYPE}|data.csv}",+ &replacements,+ |key| env.get(key).cloned(),+ )+ .expect("replacement should succeed");++ assert_eq!(actual, "data.parquet");+ }++ #[test]+ fn process_replacements_leaves_unsupported_placeholder_syntax_unchanged() {+ let replacements = HashMap::new();++ let actual =+ process_replacements_with_env("${BAD-KEY:-fallback}", &replacements, |_| {+ None+ })+ .expect("unsupported placeholder should not match replacement regex");++ assert_eq!(actual, "${BAD-KEY:-fallback}");+ }++ // Parser tests cover benchmark directives and parse-time validation.++ #[tokio::test]+ async fn parser_accepts_metadata_expect_echo_and_sql_sections() {+ let benchmark = parse_benchmark(+ r#"+# top-level comments are ignored+name Parser Success+group Parser Group+subgroup Parser Subgroup+expect_plan ProjectionExec with details+echo hello from parser++load+-- query comments are ignored+CREATE TABLE t AS VALUES (1);++init+CREATE VIEW v AS SELECT * FROM t;++run+SELECT * FROM v;++cleanup+DROP VIEW v;+"#,+ )+ .await+ .expect("benchmark should parse");++ assert_eq!(benchmark.name(), "Parser Success");+ assert_eq!(benchmark.group(), "Parser Group");+ assert_eq!(benchmark.subgroup(), "Parser Subgroup");+ assert_eq!(benchmark.expect, vec!["ProjectionExec with details"]);+ assert_eq!(benchmark.echo, vec!["hello from parser"]);+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Load)+ .expect("load query"),+ &vec!["CREATE TABLE t AS VALUES (1);".to_string()]+ );+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Init)+ .expect("init query"),+ &vec!["CREATE VIEW v AS SELECT * FROM t;".to_string()]+ );+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["SELECT * FROM v;".to_string()]+ );+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Cleanup)+ .expect("cleanup query"),+ &vec!["DROP VIEW v;".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_splits_inline_run_block_on_semicolon_newline() {+ let benchmark = parse_benchmark(+ r#"+run+CREATE TABLE t AS SELECT 1 AS value;+SELECT value + 1 AS value FROM t;+DROP TABLE t;+"#,+ )+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec![+ "CREATE TABLE t AS SELECT 1 AS value;".to_string(),+ "SELECT value + 1 AS value FROM t;".to_string(),+ "DROP TABLE t;".to_string(),+ ]+ );+ }++ #[tokio::test]+ async fn parser_accepts_assert_with_expected_rows() {+ let benchmark = parse_benchmark(+ r#"+assert II+select 1, 'one'+----+1|one+2 two+"#,+ )+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .assert_queries()+ .first()+ .expect("assert query should be parsed");++ assert_eq!(query.column_count, 2);+ assert!(query.query.contains("select 1, 'one'"));+ assert_eq!(+ query.expected_result,+ vec![+ vec!["1".to_string(), "one".to_string()],+ vec!["2".to_string(), "two".to_string()]+ ]+ );+ }++ #[tokio::test]+ async fn parser_accepts_result_query_with_expected_rows() {+ let benchmark = parse_benchmark(+ r#"+result_query II+select 1, 'one'+----+1|one+NULL|(empty)+"#,+ )+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .result_queries()+ .first()+ .expect("result query should be parsed");++ assert_eq!(query.path, None);+ assert_eq!(query.column_count, 2);+ assert!(query.query.contains("select 1, 'one'"));+ assert_eq!(+ query.expected_result,+ vec![+ vec!["1".to_string(), "one".to_string()],+ vec!["NULL".to_string(), "(empty)".to_string()]+ ]+ );+ }++ #[tokio::test]+ async fn parser_records_result_file_without_parsing_contents() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path =+ write_test_file(&temp_dir, "result.csv", "col_a|col_b\n1|one\nNULL|two\n");+ let benchmark_path = write_test_file(+ &temp_dir,+ "result.benchmark",+ &format!("result {}\n", result_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .result_queries()+ .first()+ .expect("result file should be parsed");++ assert_eq!(query.path, Some(result_path.to_string_lossy().into_owned()));+ assert_eq!(query.column_count, 0);+ assert!(query.expected_result.is_empty());+ }++ #[tokio::test]+ async fn parser_accepts_include_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let include_path =+ write_test_file(&temp_dir, "include.benchmark", "run\nselect 1\n");++ let benchmark_path = write_test_file(+ &temp_dir,+ "include_driver.benchmark",+ &format!("include {}\n", include_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ let benchmark = result.expect("benchmark should parse");+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["select 1".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_accepts_template_file_with_parameters() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let template_path = write_test_file(+ &temp_dir,+ "template_success.benchmark",+ "# template comments are ignored\nrun\n-- query comments are ignored\nselect '${TABLE_NAME}', '${BENCHMARK_DIR}'\n",+ );++ let benchmark_path = write_test_file(+ &temp_dir,+ "template_success_driver.benchmark",+ &format!(+ "template {}\n# parameter comments are ignored\nTABLE_NAME=orders\n",+ template_path.display()+ ),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ let benchmark = result.expect("benchmark should parse");+ assert_eq!(benchmark.benchmark_path(), template_path.as_path());+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["select 'orders', '/tmp'".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_trims_template_parameter_keys_and_values() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let template_path = write_test_file(+ &temp_dir,+ "template_trim.benchmark",+ "run\nselect '${TABLE_NAME}'\n",+ );++ let benchmark_path = write_test_file(+ &temp_dir,+ "template_trim_driver.benchmark",+ &format!(+ "template {}\n TABLE_NAME = orders \n",+ template_path.display()+ ),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["select 'orders'".to_string()]+ );+ assert_eq!(+ benchmark.replacement_mapping().get("table_name"),+ Some(&"orders".to_string())+ );+ }++ #[tokio::test]+ async fn parser_preserves_expected_result_cell_whitespace() {+ let benchmark = parse_benchmark("assert I\nselect ' x '\n----\n x \n")+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .assert_queries()+ .first()+ .expect("assert query should be parsed");++ assert_eq!(query.expected_result, vec![vec![" x ".to_string()]]);+ }++ #[tokio::test]+ async fn parser_accepts_indented_comments_and_blank_lines() {+ let benchmark =+ parse_benchmark(" # comment\n -- comment\n run\n select 1\n \n")+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["select 1".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_accepts_case_insensitive_query_directives() {+ let benchmark = parse_benchmark("RUN\nselect 1\n")+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["select 1".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_accepts_query_file_and_splits_statements() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "queries.sql",+ "-- leading comment\nSELECT 1 AS value;\nSELECT 2 AS value;\n\n# another comment\nWITH t AS (SELECT 3 AS value) SELECT * FROM t;\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "query_file.benchmark",+ &format!("run {}\n", query_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run queries"),+ &vec![+ "SELECT 1 AS value;".to_string(),+ "SELECT 2 AS value;".to_string(),+ "WITH t AS (SELECT 3 AS value) SELECT * FROM t;".to_string(),+ ]+ );+ }++ #[tokio::test]+ async fn parser_accepts_replacements_in_query_file_path() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path =+ write_test_file(&temp_dir, "queries.sql", "SELECT 5 AS value;\n");+ let template_path = write_test_file(+ &temp_dir,+ "query_file_path_template.benchmark",+ "run ${QUERY_PATH}\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "query_file_path_driver.benchmark",+ &format!(+ "template {}\nQUERY_PATH={}\n",+ template_path.display(),+ query_path.display()+ ),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["SELECT 5 AS value;".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_rejects_inline_sql_when_query_file_is_provided() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path =+ write_test_file(&temp_dir, "queries.sql", "SELECT 1 AS value;\n");+ let benchmark_path = write_test_file(+ &temp_dir,+ "query_file_with_inline_body.benchmark",+ &format!("run {}\nSELECT 999 AS value;\n", query_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(+ result,+ "run directive must use either a query file or inline SQL, not both",+ );+ }++ #[tokio::test]+ async fn parser_rejects_inline_sql_when_load_file_is_provided() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "load.sql",+ "CREATE TABLE t AS SELECT 1 AS value;\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "load_file_with_inline_body.benchmark",+ &format!(+ "load {}\nCREATE TABLE u AS SELECT 2 AS value;\n",+ query_path.display()+ ),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(+ result,+ "load directive must use either a query file or inline SQL, not both",+ );+ }++ #[tokio::test]+ async fn parser_rejects_inline_sql_when_init_file_is_provided() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "init.sql",+ "CREATE VIEW v AS SELECT 1 AS value;\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "init_file_with_inline_body.benchmark",+ &format!(+ "init {}\nCREATE VIEW w AS SELECT 2 AS value;\n",+ query_path.display()+ ),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(+ result,+ "init directive must use either a query file or inline SQL, not both",+ );+ }++ #[tokio::test]+ async fn parser_rejects_inline_sql_when_cleanup_file_is_provided() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(&temp_dir, "cleanup.sql", "DROP TABLE t;\n");+ let benchmark_path = write_test_file(+ &temp_dir,+ "cleanup_file_with_inline_body.benchmark",+ &format!("cleanup {}\nDROP TABLE u;\n", query_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(+ result,+ "cleanup directive must use either a query file or inline SQL, not both",+ );+ }++ #[tokio::test]+ async fn parser_ignores_query_file_with_only_comments_and_blank_lines() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "queries.sql",+ "# comment\n\n-- another comment\n\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "empty_query_file.benchmark",+ &format!("run {}\n", query_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ assert!(!benchmark.queries().contains_key(&QueryDirective::Run));+ }++ #[tokio::test]+ async fn parser_splits_query_file_with_windows_line_endings() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "queries.sql",+ "SELECT 1 AS value;\r\nSELECT 2 AS value;\r\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "windows_query_file.benchmark",+ &format!("run {}\n", query_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run queries"),+ &vec![+ "SELECT 1 AS value;".to_string(),+ "SELECT 2 AS value;".to_string()+ ]+ );+ }++ #[tokio::test]+ async fn parser_rejects_unknown_command() {+ assert_parse_error("wat\n", "Unrecognized command: wat").await;+ }++ #[tokio::test]+ async fn parser_rejects_assert_without_column_count() {+ assert_parse_error(+ "assert\nselect 1\n----\n1\n",+ "assert must be followed by a column count",+ )+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_assert_without_result_separator() {+ assert_parse_error(+ "assert I\nselect 1\n1\n",+ "assert must be followed by a query and a result (separated by ----)",+ )+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_result_query_without_separator() {+ assert_parse_error(+ "result_query I\nselect 1\n1\n",+ "result_query must be followed by a query and a result (separated by ----)",+ )+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_result_query_with_wrong_column_count() {+ assert_parse_error(+ "result_query II\nselect 1\n----\n1\n",+ "expected 2 values but got 1",+ )+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_multiple_result_queries() {+ assert_parse_error(+ "result_query I\nselect 1\n----\n1\n\nresult_query I\nselect 2\n----\n2\n",+ "multiple results found",+ )+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_duplicate_run_directives() {+ assert_parse_error("run\nselect 1\n\nrun\nselect 2\n", "Multiple calls to run")+ .await;+ }++ #[tokio::test]+ async fn parser_accepts_multiple_load_directives() {+ let benchmark = parse_benchmark(+ "load\nCREATE TABLE t AS SELECT 1;\n\nload\nCREATE TABLE u AS SELECT 2;\n",+ )+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Load)+ .expect("load queries"),+ &vec![+ "CREATE TABLE t AS SELECT 1;".to_string(),+ "CREATE TABLE u AS SELECT 2;".to_string(),+ ]+ );+ }++ #[tokio::test]+ async fn parser_accepts_multiple_init_directives() {+ let benchmark = parse_benchmark(+ "init\nCREATE VIEW v AS SELECT 1;\n\ninit\nCREATE VIEW w AS SELECT 2;\n",+ )+ .await;++ let benchmark = benchmark.expect("benchmark should parse");+ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Init)+ .expect("init queries"),+ &vec![+ "CREATE VIEW v AS SELECT 1;".to_string(),+ "CREATE VIEW w AS SELECT 2;".to_string(),+ ]+ );+ }++ #[tokio::test]+ async fn parser_accepts_multiple_cleanup_directives() {+ let benchmark =+ parse_benchmark("cleanup\nDROP TABLE t;\n\ncleanup\nDROP TABLE u;\n")+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Cleanup)+ .expect("cleanup queries"),+ &vec!["DROP TABLE t;".to_string(), "DROP TABLE u;".to_string(),]+ );+ }++ #[tokio::test]+ async fn parser_rejects_missing_query_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let missing_path = temp_dir.path().join("missing.sql");+ let benchmark_path = write_test_file(+ &temp_dir,+ "missing_query_file.benchmark",+ &format!("run {}\n", missing_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(result, "Failed to read query file");+ }++ #[tokio::test]+ async fn parser_rejects_template_with_invalid_parameter_assignment() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let template_path =+ write_test_file(&temp_dir, "template.benchmark", "run\nselect 1\n");++ let benchmark_path = write_test_file(+ &temp_dir,+ "template_driver.benchmark",+ &format!("template {}\nINVALID\n", template_path.display()),+ );++ let ctx = SessionContext::new();+ let benchmark_path_string = benchmark_path.to_string_lossy().into_owned();+ let result = SqlBenchmark::new(&ctx, &benchmark_path_string, "/tmp").await;++ let error = result.expect_err("benchmark parsing should fail");+ let message = error.to_string();+ assert!(+ message.contains("Expected a template parameter in the form of X=Y"),+ "expected template parameter error, got {message:?}"+ );+ }++ #[tokio::test]+ async fn parser_rejects_metadata_and_result_directives_without_values() {+ assert_parse_error("name\n", "name must be followed by a value").await;+ assert_parse_error("group\n", "group must be followed by a value").await;+ assert_parse_error("subgroup\n", "subgroup must be followed by a value").await;+ assert_parse_error(+ "expect_plan\n",+ "expect_plan must be followed by a string to search in the physical plan",+ )+ .await;+ assert_parse_error("echo\n", "Echo requires an argument").await;+ assert_parse_error(+ "result\n",+ "result must be followed by a path to a result file",+ )+ .await;+ assert_parse_error("include\n", "include requires a single argument").await;+ assert_parse_error("template\n", "template requires a single template path")+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_include_and_template_with_too_many_arguments() {+ assert_parse_error("include a b\n", "include requires a single argument").await;+ assert_parse_error("template a b\n", "template requires a single template path")+ .await;+ }++ #[tokio::test]+ async fn parser_rejects_missing_include_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let missing_path = temp_dir.path().join("missing_include.benchmark");+ let benchmark_path = write_test_file(+ &temp_dir,+ "missing_include_driver.benchmark",+ &format!("include {}\n", missing_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(result, "No such file");+ }++ #[tokio::test]+ async fn parser_rejects_missing_template_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let missing_path = temp_dir.path().join("missing_template.benchmark");+ let benchmark_path = write_test_file(+ &temp_dir,+ "missing_template_driver.benchmark",+ &format!("template {}\n", missing_path.display()),+ );++ let result = parse_benchmark_file(&benchmark_path).await;++ assert_result_error_contains(result, "No such file");+ }++ #[tokio::test]+ async fn parser_uses_metadata_values_as_replacements() {+ let benchmark = parse_benchmark(+ r#"+name Q01+group tpch+subgroup sf1++run+SELECT '${BENCH_NAME}', '${BENCH_GROUP}', '${BENCH_SUBGROUP}'+"#,+ )+ .await+ .expect("benchmark should parse");++ assert_eq!(+ benchmark+ .queries()+ .get(&QueryDirective::Run)+ .expect("run query"),+ &vec!["SELECT 'Q01', 'tpch', 'sf1'".to_string()]+ );+ }++ #[tokio::test]+ async fn parser_accepts_replacement_in_result_file_path() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path = write_test_file(&temp_dir, "result.csv", "value\n1\n");+ let template_path = write_test_file(+ &temp_dir,+ "result_path_template.benchmark",+ "result ${RESULT_PATH}\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "result_path_driver.benchmark",+ &format!(+ "template {}\nRESULT_PATH={}\n",+ template_path.display(),+ result_path.display()+ ),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .result_queries()+ .first()+ .expect("result query should be parsed");+ assert_eq!(query.path, Some(result_path.to_string_lossy().into_owned()));+ assert_eq!(query.column_count, 0);+ assert!(query.expected_result.is_empty());+ }++ #[tokio::test]+ async fn parser_rejects_missing_replacement_in_result_file_path() {+ assert_parse_error("result ${MISSING_RESULT_PATH}\n", "Missing value for key")+ .await;+ }++ #[tokio::test]+ async fn parser_accepts_missing_result_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let missing_path = temp_dir.path().join("missing_result.csv");+ let benchmark_path = write_test_file(+ &temp_dir,+ "missing_result_file.benchmark",+ &format!("result {}\n", missing_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .result_queries()+ .first()+ .expect("result file should be parsed");+ assert_eq!(+ query.path,+ Some(missing_path.to_string_lossy().into_owned())+ );+ assert!(query.expected_result.is_empty());+ }++ #[tokio::test]+ async fn parser_accepts_malformed_result_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path = temp_dir.path().join("malformed_result.csv");+ fs::write(&result_path, [0xff]).expect("failed to write malformed result file");+ let benchmark_path = write_test_file(+ &temp_dir,+ "malformed_result_file.benchmark",+ &format!("result {}\n", result_path.display()),+ );++ let benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");++ let query = benchmark+ .result_queries()+ .first()+ .expect("result file should be parsed");+ assert_eq!(query.path, Some(result_path.to_string_lossy().into_owned()));+ assert!(query.expected_result.is_empty());+ }++ // Lifecycle tests cover initialization, assertions, and cleanup execution.++ #[tokio::test]+ async fn initialize_executes_load_before_init_and_is_idempotent() {+ let mut benchmark = parse_benchmark(+ r#"+load+CREATE TABLE t AS SELECT 1 AS value;++load+CREATE TABLE u AS SELECT value + 1 AS value FROM t;++init+CREATE TABLE v AS SELECT value + 1 AS value FROM u;++init+CREATE TABLE initialized AS SELECT value + 1 AS value FROM v;++run+SELECT value FROM initialized;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark+ .initialize(&ctx)+ .await+ .expect("initialize should succeed");+ benchmark+ .initialize(&ctx)+ .await+ .expect("second initialize should be a no-op");++ assert!(benchmark.is_loaded());++ let rows = ctx+ .sql("SELECT value FROM initialized")+ .await+ .expect("query should plan")+ .collect()+ .await+ .expect("query should run");++ assert_eq!(format_record_batches(&rows).unwrap(), vec![vec!["4"]]);+ }++ #[tokio::test]+ async fn initialize_rejects_benchmark_without_run_query() {+ let mut benchmark = parse_benchmark(+ r#"+load+CREATE TABLE t AS SELECT 1 AS value;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.initialize(&ctx).await,+ "Invalid benchmark file: no \"run\" query specified",+ );+ }++ #[tokio::test]+ async fn initialize_propagates_load_query_failures() {+ let mut benchmark = parse_benchmark(+ r#"+load+CREATE TABLE t AS SELECT * FROM missing_load_table;++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.initialize(&ctx).await,+ "missing_load_table",+ );+ }++ #[tokio::test]+ async fn initialize_propagates_init_query_failures() {+ let mut benchmark = parse_benchmark(+ r#"+init+CREATE TABLE t AS SELECT * FROM missing_init_table;++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.initialize(&ctx).await,+ "missing_init_table",+ );+ }++ #[tokio::test]+ async fn cleanup_executes_cleanup_queries() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 1;++cleanup+CREATE TABLE cleanup_marker_a AS SELECT 7 AS value;++cleanup+CREATE TABLE cleanup_marker_b AS SELECT value + 1 AS value FROM cleanup_marker_a;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.cleanup(&ctx).await.expect("cleanup should run");++ let rows = ctx+ .sql("SELECT value FROM cleanup_marker_b")+ .await+ .expect("query should plan")+ .collect()+ .await+ .expect("query should run");+ assert_eq!(format_record_batches(&rows).unwrap(), vec![vec!["8"]]);+ }++ #[tokio::test]+ async fn cleanup_propagates_query_failures() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 1;++cleanup+SELECT * FROM missing_cleanup_table;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.cleanup(&ctx).await,+ "missing_cleanup_table",+ );+ }++ #[tokio::test]+ async fn assert_executes_assert_queries_successfully() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT 1 AS value+----+1++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.assert(&ctx).await.expect("assert should pass");+ }++ #[tokio::test]+ async fn assert_accepts_null_expected_for_empty_actual() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT '' AS value+----+NULL++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.assert(&ctx).await.expect("assert should pass");+ }++ #[tokio::test]+ async fn assert_accepts_empty_marker_for_empty_actual() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT '' AS value+----+(empty)++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.assert(&ctx).await.expect("assert should pass");+ }++ #[tokio::test]+ async fn assert_accepts_empty_marker_for_null_actual() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT CAST(NULL AS VARCHAR) AS value+----+(empty)++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.assert(&ctx).await.expect("assert should pass");+ }++ #[tokio::test]+ async fn assert_succeeds_with_zero_actual_and_expected_rows() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT 1 AS value WHERE false+----++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.assert(&ctx).await.expect("assert should pass");+ }++ #[tokio::test]+ async fn assert_propagates_query_failures() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT * FROM missing_assert_table+----+1++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.assert(&ctx).await,+ "missing_assert_table",+ );+ }++ #[tokio::test]+ async fn assert_reports_row_count_mismatch() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT 1 AS value+----+1+2++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.assert(&ctx).await,+ "expected 2 rows but got 1",+ );+ }++ #[tokio::test]+ async fn assert_reports_column_count_mismatch() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT 1 AS a, 2 AS b+----+1++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.assert(&ctx).await,+ "expected 1 columns but got 2",+ );+ }++ #[tokio::test]+ async fn assert_reports_value_mismatch() {+ let mut benchmark = parse_benchmark(+ r#"+assert I+SELECT 1 AS value+----+2++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.assert(&ctx).await,+ "expected value \"2\" but got value \"1\"",+ );+ }++ // Run tests cover result buffering and physical-plan expectations.++ #[tokio::test]+ async fn run_saves_uppercase_select_results() {+ let mut benchmark = parse_benchmark("run\nSELECT 1 AS value\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_eq!(formatted_last_results(&benchmark), vec![vec!["1"]]);+ }++ #[tokio::test]+ async fn run_saves_with_query_results() {+ let mut benchmark =+ parse_benchmark("run\nWITH t AS (SELECT 3 AS value) SELECT value FROM t\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_eq!(formatted_last_results(&benchmark), vec![vec!["3"]]);+ }++ #[tokio::test]+ async fn run_only_keeps_last_select_or_with_result() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let query_path = write_test_file(+ &temp_dir,+ "queries.sql",+ "SELECT 1 AS value;\nSELECT 2 AS value;\nWITH t AS (SELECT 3 AS value) SELECT value FROM t;\n",+ );+ let benchmark_path = write_test_file(+ &temp_dir,+ "run_file.benchmark",+ &format!("run {}\n", query_path.display()),+ );+ let mut benchmark = parse_benchmark_file(&benchmark_path)+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_eq!(formatted_last_results(&benchmark), vec![vec!["3"]]);+ }++ #[tokio::test]+ async fn run_inline_multi_statement_only_keeps_last_select_or_with_result() {+ let mut benchmark = parse_benchmark(+ "run\nCREATE TABLE t AS SELECT 1 AS value;\nSELECT 2 AS value;\nWITH u AS (SELECT 3 AS value) SELECT value FROM u;\n",+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_eq!(formatted_last_results(&benchmark), vec![vec!["3"]]);+ }++ #[tokio::test]+ async fn run_does_not_save_results_for_non_select_statement() {+ let mut benchmark =+ parse_benchmark("run\nCREATE TABLE run_created AS SELECT 1 AS value;\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert!(+ benchmark+ .last_results+ .as_ref()+ .expect("last results should be set")+ .is_empty()+ );+ }++ #[tokio::test]+ async fn run_propagates_query_failures_when_buffering_results() {+ let mut benchmark = parse_benchmark("run\nSELECT * FROM missing_run_table\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.run(&ctx, true).await,+ "missing_run_table",+ );+ }++ #[tokio::test]+ async fn run_propagates_query_failures_when_streaming_results() {+ let mut benchmark = parse_benchmark("run\nSELECT * FROM missing_stream_table\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.run(&ctx, false).await,+ "missing_stream_table",+ );+ }++ #[tokio::test]+ async fn run_rejects_missing_expect_plan_for_buffered_and_streaming_modes() {+ let ctx = SessionContext::new();+ let benchmark_text = "expect_plan definitely_not_in_plan\nrun\nSELECT 1\n";++ let mut buffered = parse_benchmark(benchmark_text)+ .await+ .expect("benchmark should parse");+ assert_result_error_contains(+ buffered.run(&ctx, true).await,+ "does not contain the expected string 'definitely_not_in_plan'",+ );++ let mut streaming = parse_benchmark(benchmark_text)+ .await+ .expect("benchmark should parse");+ assert_result_error_contains(+ streaming.run(&ctx, false).await,+ "does not contain the expected string 'definitely_not_in_plan'",+ );+ }++ #[tokio::test]+ async fn run_accepts_matching_expect_plan_for_buffered_and_streaming_modes() {+ let ctx = SessionContext::new();+ let benchmark_text = "expect_plan PlaceholderRowExec\nrun\nSELECT 1\n";++ let mut buffered = parse_benchmark(benchmark_text)+ .await+ .expect("benchmark should parse");+ buffered+ .run(&ctx, true)+ .await+ .expect("buffered run should accept matching plan");+ assert_eq!(formatted_last_results(&buffered), vec![vec!["1"]]);++ let mut streaming = parse_benchmark(benchmark_text)+ .await+ .expect("benchmark should parse");+ streaming+ .run(&ctx, false)+ .await+ .expect("streaming run should accept matching plan");+ }++ // Verification tests cover result_query and persisted-result comparison paths.++ #[tokio::test]+ async fn verify_without_result_query_returns_ok() {+ let mut benchmark = parse_benchmark("run\nSELECT 1 AS value\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.verify(&ctx).await.expect("verify should pass");+ }++ #[tokio::test]+ async fn verify_errors_when_benchmark_has_not_run() {+ let mut benchmark = parse_benchmark(+ r#"+result_query I+SELECT 1 AS value+----+1++run+SELECT 1;+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.verify(&ctx).await,+ "No results available for verification. Run the benchmark first.",+ );+ }++ #[tokio::test]+ async fn verify_uses_last_results_for_result_file_entries() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path = write_test_file(&temp_dir, "result.csv", "value\n1\n");+ let mut benchmark = parse_benchmark(&format!(+ "result {}\n\nrun\nSELECT 1 AS value\n",+ result_path.display()+ ))+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");+ benchmark.verify(&ctx).await.expect("verify should pass");+ }++ #[tokio::test]+ async fn verify_uses_last_results_for_zero_row_result_file_entries() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path = write_test_file(&temp_dir, "result.csv", "value\n");+ let mut benchmark = parse_benchmark(&format!(+ "result {}\n\nrun\nSELECT 1 AS value WHERE false\n",+ result_path.display()+ ))+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");+ benchmark.verify(&ctx).await.expect("verify should pass");+ }++ #[tokio::test]+ async fn verify_rejects_missing_result_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let missing_path = temp_dir.path().join("missing_result.csv");+ let mut benchmark = parse_benchmark(&format!(+ "result {}\n\nrun\nSELECT 1 AS value\n",+ missing_path.display()+ ))+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_result_error_contains(benchmark.verify(&ctx).await, "missing_result.csv");+ }++ #[tokio::test]+ async fn verify_rejects_malformed_result_file() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let result_path = temp_dir.path().join("malformed_result.csv");+ fs::write(&result_path, [0xff]).expect("failed to write malformed result file");+ let mut benchmark = parse_benchmark(&format!(+ "result {}\n\nrun\nSELECT 1 AS value\n",+ result_path.display()+ ))+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_result_error_contains(benchmark.verify(&ctx).await, "CSV");+ }++ #[tokio::test]+ async fn verify_executes_result_query_instead_of_last_results() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 100 AS value++result_query I+SELECT 1 AS value+----+1+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");+ benchmark.verify(&ctx).await.expect("verify should pass");+ }++ #[tokio::test]+ async fn verify_propagates_result_query_failures() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 1 AS value++result_query I+SELECT * FROM missing_verify_table+----+1+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ assert_result_error_contains(+ benchmark.verify(&ctx).await,+ "missing_verify_table",+ );+ }++ #[tokio::test]+ async fn verify_reports_result_mismatch_context() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 1 AS value++result_query I+SELECT 1 AS value+----+2+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.run(&ctx, true).await.expect("run should succeed");++ let error = benchmark+ .verify(&ctx)+ .await+ .expect_err("verify should fail");+ let message = error.to_string();+ assert!(+ message.contains("row 1, column 1")+ && message.contains("expected value \"2\"")+ && message.contains("got value \"1\""),+ "unexpected error: {message}"+ );+ }++ // Persistence tests cover CSV writing and persist-time error paths.++ #[tokio::test]+ async fn persist_without_result_query_returns_ok() {+ let mut benchmark = parse_benchmark("run\nSELECT 1 AS value\n")+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ benchmark.persist(&ctx).await.expect("persist should pass");+ }++ #[tokio::test]+ async fn persist_rejects_result_query_without_file_path() {+ let mut benchmark = parse_benchmark(+ r#"+run+SELECT 1 AS value++result_query I+SELECT 1 AS value+----+1+"#,+ )+ .await+ .expect("benchmark should parse");+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.persist(&ctx).await,+ "Unable to persist results from query",+ );+ }++ #[tokio::test]+ async fn persist_rejects_run_without_saved_result_batches() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let output_path = temp_dir.path().join("persisted");+ let mut benchmark =+ parse_benchmark("run\nCREATE TABLE persist_source AS SELECT 1 AS value;\n")+ .await+ .expect("benchmark should parse");+ benchmark.result_queries.push(BenchmarkQuery {+ path: Some(output_path.to_string_lossy().into_owned()),+ query: String::new(),+ column_count: 1,+ expected_result: vec![],+ });+ let ctx = SessionContext::new();++ assert_result_error_contains(+ benchmark.persist(&ctx).await,+ "Results should be loaded",+ );+ }++ #[tokio::test]+ async fn persist_writes_header_and_pipe_delimited_rows() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let output_path = temp_dir.path().join("persisted");+ let mut benchmark = parse_benchmark("run\nSELECT 1 AS a, 'one' AS b\n")+ .await+ .expect("benchmark should parse");+ benchmark.result_queries.push(BenchmarkQuery {+ path: Some(output_path.to_string_lossy().into_owned()),+ query: String::new(),+ column_count: 2,+ expected_result: vec![],+ });+ let ctx = SessionContext::new();++ benchmark.persist(&ctx).await.expect("persist should pass");++ let contents = read_all_files_in_dir(&output_path);+ assert!(+ contents.contains("a|b\n") && contents.contains("1|one\n"),+ "unexpected persisted contents: {contents:?}"+ );+ }++ #[tokio::test]+ async fn persist_writes_header_for_zero_row_select_results() {+ let temp_dir = tempdir().expect("failed to create benchmark test directory");+ let output_path = temp_dir.path().join("persisted_empty");+ let mut benchmark = parse_benchmark("run\nSELECT 1 AS value WHERE false\n")+ .await+ .expect("benchmark should parse");+ benchmark.result_queries.push(BenchmarkQuery {+ path: Some(output_path.to_string_lossy().into_owned()),+ query: String::new(),+ column_count: 1,+ expected_result: vec![],+ });+ let ctx = SessionContext::new();++ benchmark.persist(&ctx).await.expect("persist should pass");++ let contents = read_all_files_in_dir(&output_path);+ assert!(+ contents.contains("value\n"),+ "unexpected persisted contents: {contents:?}"+ );+ }++ // Path helper tests cover group derivation from benchmark file paths.++ #[test]+ fn parse_group_from_path_returns_group_under_benchmark_directory() {+ let group = parse_group_from_path(+ Path::new("sql_benchmarks/tpch/benchmarks/q01.benchmark"),+ Path::new("sql_benchmarks"),+ );++ assert_eq!(group, "tpch");+ }++ #[test]+ fn parse_group_from_path_matches_benchmark_directory_case_insensitively() {+ let group = parse_group_from_path(+ Path::new("/tmp/SQL_BENCHMARKS/Tpch/benchmarks/q01.benchmark"),+ Path::new("sql_benchmarks"),+ );++ assert_eq!(group, "Tpch");+ }++ #[test]+ fn parse_group_from_path_handles_relative_and_absolute_paths() {+ let relative = parse_group_from_path(+ Path::new("sql_benchmarks/h2o/q01.benchmark"),+ Path::new("sql_benchmarks"),+ );+ let absolute = parse_group_from_path(+ Path::new("/tmp/sql_benchmarks/imdb/q01.benchmark"),+ Path::new("sql_benchmarks"),+ );++ assert_eq!(relative, "h2o");+ assert_eq!(absolute, "imdb");+ }++ #[test]+ fn parse_group_from_path_pins_fallback_for_paths_outside_benchmark_directory() {+ let group = parse_group_from_path(+ Path::new("outside/group/q01.benchmark"),+ Path::new("sql_benchmarks"),+ );++ assert_eq!(group, "outside");+ }++ #[test]+ fn path_ends_with_ignore_ascii_case_matches_component_suffixes() {+ assert!(path_ends_with_ignore_ascii_case(+ Path::new("/tmp/SQL_BENCHMARKS"),+ Path::new("sql_benchmarks")+ ));+ assert!(!path_ends_with_ignore_ascii_case(+ Path::new("/tmp/sql_benchmarks_extra"),+ Path::new("sql_benchmarks")+ ));+ }+}diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rsindex a50a5268c0..a3e6d2a4c5 100644--- a/benchmarks/src/util/options.rs+++ b/benchmarks/src/util/options.rs@@ -37,38 +37,38 @@ use super::latency_object_store::LatencyObjectStore;#[derive(Debug, Args, Clone)]pub struct CommonOpt {/// Number of iterations of each test run- #[arg(short = 'i', long = "iterations", default_value = "3")]+ #[arg(short = 'i', long = "iterations", default_value = "3", env)]pub iterations: usize,/// Number of partitions to process in parallel. Defaults to number of available cores.- #[arg(short = 'n', long = "partitions")]+ #[arg(short = 'n', long = "partitions", env)]pub partitions: Option<usize>,/// Batch size when reading CSV or Parquet files- #[arg(short = 's', long = "batch-size")]+ #[arg(short = 's', long = "batch-size", env)]pub batch_size: Option<usize>,/// The memory pool type to use, should be one of "fair" or "greedy"- #[arg(long = "mem-pool-type", default_value = "fair")]+ #[arg(long = "mem-pool-type", default_value = "fair", env)]pub mem_pool_type: String,/// Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query/// if there's any, otherwise run with no memory limit.- #[arg(long = "memory-limit", value_parser = parse_capacity_limit)]+ #[arg(long = "memory-limit", value_parser = parse_capacity_limit, env)]pub memory_limit: Option<usize>,/// The amount of memory to reserve for sort spill operations. DataFusion's default value will be used/// if not specified.- #[arg(long = "sort-spill-reservation-bytes", value_parser = parse_capacity_limit)]+ #[arg(long = "sort-spill-reservation-bytes", value_parser = parse_capacity_limit, env)]pub sort_spill_reservation_bytes: Option<usize>,/// Activate debug mode to see more details- #[arg(short, long)]+ #[arg(short, long, env)]pub debug: bool,/// Simulate object store latency to mimic remote storage (e.g. S3)./// Adds random latency in the range 20-200ms to each object store operation.- #[arg(long = "simulate-latency")]+ #[arg(long = "simulate-latency", env)]pub simulate_latency: bool,}