Skip to content

Apache DataFusion SQL Query Engine

Respect DATA_DIR location for sql benchmarks (#21961)

Bruce Ritchiedcd364ad9325
diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
index 7aa0418e1d..4e7d743997 100755
--- a/benchmarks/bench.sh
+++ b/benchmarks/bench.sh
@@ -691,6 +691,7 @@ run_tpch() {
debug_run env BENCH_NAME=tpch \
BENCH_SIZE="${SCALE_FACTOR}" \
+ DATA_DIR="${DATA_DIR}" \
PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \
TPCH_FILE_TYPE="${FORMAT}" \
SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
@@ -709,6 +710,7 @@ run_tpch_mem() {
debug_run env BENCH_NAME=tpch \
BENCH_SIZE="${SCALE_FACTOR}" \
+ DATA_DIR="${DATA_DIR}" \
TPCH_FILE_TYPE="mem" \
PREFER_HASH_JOIN="${PREFER_HASH_JOIN}" \
SIMULATE_LATENCY="${SIMULATE_LATENCY}" \
diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.md
index e8899641c0..5bbf2ba81b 100644
--- a/benchmarks/sql_benchmarks/README.md
+++ b/benchmarks/sql_benchmarks/README.md
@@ -68,9 +68,10 @@ The SQL benchmarking tool uses the following environment variables:
| BENCH_QUERY | A query number to run. |
| BENCH_PERSIST_RESULTS | true/false to persist benchmark results. Results will be persisted in csv format so be cognizant of the size of the results. |
| BENCH_VALIDATE | true/false to validate benchmark results against persisted results or result_query's. If both `BENCH_PERSIST_RESULTS` and `BENCH_VALIDATE` are true, persist mode runs and validation is skipped. |
+| DATA_DIR | Root directory for benchmark data loaded by SQL benchmark files. When unset, uses `data` (relative to the benchmarks/ directory). |
| SIMULATE_LATENCY | Simulate object store latency to mimic remote storage (e.g. S3). Adds random latency in the range 20-200ms to each object store operation. |
| MEM_POOL_TYPE | The memory pool type to use, should be one of "fair" or "greedy". |
-| MEMORY_LIMIT | Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query if there's any, otherwise run with no memory limit. | |
+| MEMORY_LIMIT | Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query if there's any, otherwise run with no memory limit. |
Example – Run the H2O window benchmarks on the 'small' sized CSV data files:
diff --git a/benchmarks/sql_benchmarks/tpch/init/load_csv.sql b/benchmarks/sql_benchmarks/tpch/init/load_csv.sql
index f9a9b2e988..d3594a7262 100644
--- a/benchmarks/sql_benchmarks/tpch/init/load_csv.sql
+++ b/benchmarks/sql_benchmarks/tpch/init/load_csv.sql
@@ -5,7 +5,7 @@ CREATE EXTERNAL TABLE nation
n_regionkey INT,
n_comment VARCHAR(152),
PRIMARY KEY (n_nationkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/nation/nation.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/nation/nation.1.csv';
CREATE EXTERNAL TABLE region
(
@@ -13,7 +13,7 @@ CREATE EXTERNAL TABLE region
r_name CHAR(25),
r_comment VARCHAR(152),
PRIMARY KEY (r_regionkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/region/region.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/region/region.1.csv';
CREATE EXTERNAL TABLE supplier
(
@@ -25,7 +25,7 @@ CREATE EXTERNAL TABLE supplier
s_acctbal DECIMAL(15, 2),
s_comment VARCHAR(101),
PRIMARY KEY (s_suppkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/supplier/supplier.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/supplier/supplier.1.csv';
CREATE EXTERNAL TABLE customer
(
@@ -38,7 +38,7 @@ CREATE EXTERNAL TABLE customer
c_mktsegment CHAR(10),
c_comment VARCHAR(117),
PRIMARY KEY (c_custkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/customer/customer.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/customer/customer.1.csv';
CREATE EXTERNAL TABLE part
(
@@ -52,7 +52,7 @@ CREATE EXTERNAL TABLE part
p_retailprice DECIMAL(15, 2),
p_comment VARCHAR(23),
PRIMARY KEY (p_partkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/part/part.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/part/part.1.csv';
CREATE EXTERNAL TABLE partsupp
(
@@ -62,7 +62,7 @@ CREATE EXTERNAL TABLE partsupp
ps_supplycost DECIMAL(15, 2),
ps_comment VARCHAR(199),
PRIMARY KEY (ps_partkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/partsupp/partsupp.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/partsupp/partsupp.1.csv';
CREATE EXTERNAL TABLE orders
(
@@ -76,7 +76,7 @@ CREATE EXTERNAL TABLE orders
o_shippriority INT,
o_comment VARCHAR(79),
PRIMARY KEY (o_orderkey)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/orders/orders.1.csv';
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/orders/orders.1.csv';
CREATE EXTERNAL TABLE lineitem
(
@@ -96,4 +96,4 @@ CREATE EXTERNAL TABLE lineitem
l_shipinstruct CHAR(25),
l_shipmode CHAR(10),
l_comment VARCHAR(44)
-) STORED AS CSV LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/csv/lineitem/lineitem.1.csv';
\ No newline at end of file
+) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/lineitem/lineitem.1.csv';
diff --git a/benchmarks/sql_benchmarks/tpch/init/load_mem.sql b/benchmarks/sql_benchmarks/tpch/init/load_mem.sql
index 57d12c22f0..16e5c0b493 100644
--- a/benchmarks/sql_benchmarks/tpch/init/load_mem.sql
+++ b/benchmarks/sql_benchmarks/tpch/init/load_mem.sql
@@ -1,18 +1,18 @@
-CREATE EXTERNAL TABLE nation_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';
+CREATE EXTERNAL TABLE nation_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';
-CREATE EXTERNAL TABLE region_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';
+CREATE EXTERNAL TABLE region_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';
-CREATE EXTERNAL TABLE supplier_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';
+CREATE EXTERNAL TABLE supplier_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';
-CREATE EXTERNAL TABLE customer_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';
+CREATE EXTERNAL TABLE customer_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';
-CREATE EXTERNAL TABLE part_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';
+CREATE EXTERNAL TABLE part_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';
-CREATE EXTERNAL TABLE partsupp_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';
+CREATE EXTERNAL TABLE partsupp_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';
-CREATE EXTERNAL TABLE orders_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';
+CREATE EXTERNAL TABLE orders_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';
-CREATE EXTERNAL TABLE lineitem_raw STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';
+CREATE EXTERNAL TABLE lineitem_raw STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';
CREATE TABLE nation as SELECT * FROM nation_raw;
diff --git a/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql
index 172a03d82a..9423e38a44 100644
--- a/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql
+++ b/benchmarks/sql_benchmarks/tpch/init/load_parquet.sql
@@ -1,15 +1,15 @@
-CREATE EXTERNAL TABLE nation STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';
+CREATE EXTERNAL TABLE nation STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/nation/nation.1.parquet';
-CREATE EXTERNAL TABLE region STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';
+CREATE EXTERNAL TABLE region STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/region/region.1.parquet';
-CREATE EXTERNAL TABLE supplier STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';
+CREATE EXTERNAL TABLE supplier STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/supplier/supplier.1.parquet';
-CREATE EXTERNAL TABLE customer STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';
+CREATE EXTERNAL TABLE customer STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/customer/customer.1.parquet';
-CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';
+CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/part/part.1.parquet';
-CREATE EXTERNAL TABLE partsupp STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';
+CREATE EXTERNAL TABLE partsupp STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/partsupp/partsupp.1.parquet';
-CREATE EXTERNAL TABLE orders STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';
+CREATE EXTERNAL TABLE orders STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/orders/orders.1.parquet';
-CREATE EXTERNAL TABLE lineitem STORED AS PARQUET LOCATION 'data/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';
\ No newline at end of file
+CREATE EXTERNAL TABLE lineitem STORED AS PARQUET LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/lineitem/lineitem.1.parquet';
diff --git a/benchmarks/src/sql_benchmark.rs b/benchmarks/src/sql_benchmark.rs
index 34614b1324..fc6da24b8a 100644
--- a/benchmarks/src/sql_benchmark.rs
+++ b/benchmarks/src/sql_benchmark.rs
@@ -2230,6 +2230,74 @@ NULL|(empty)
);
}
+ #[tokio::test]
+ async fn parser_applies_data_dir_replacement_in_load_query_file() {
+ let temp_dir = tempdir().expect("failed to create benchmark test directory");
+ let data_dir = temp_dir.path().join("non_default_data");
+ let csv_dir = data_dir.join("tpch_sf1/csv/generated");
+ fs::create_dir_all(&csv_dir).expect("failed to create generated data directory");
+ fs::write(csv_dir.join("generated.1.csv"), "value\n42\n")
+ .expect("failed to write generated csv file");
+
+ let load_path = write_test_file(
+ &temp_dir,
+ "load_generated_csv.sql",
+ "CREATE EXTERNAL TABLE generated(value INT) STORED AS CSV LOCATION '${DATA_DIR:-data}/tpch_sf${BENCH_SIZE:-1}/csv/generated/generated.1.csv' OPTIONS ('format.has_header' 'true');\n",
+ );
+ let template_path = write_test_file(
+ &temp_dir,
+ "load_file_template.benchmark",
+ &format!(
+ "load {}\n\nrun\nSELECT value FROM generated;\n",
+ load_path.display()
+ ),
+ );
+ let benchmark_path = write_test_file(
+ &temp_dir,
+ "load_file_driver.benchmark",
+ &format!(
+ "template {}\nDATA_DIR={}\n",
+ template_path.display(),
+ data_dir.display()
+ ),
+ );
+
+ let ctx = SessionContext::new();
+ let path_string = benchmark_path.to_string_lossy().into_owned();
+ let mut benchmark = SqlBenchmark::new(&ctx, &path_string, "/tmp")
+ .await
+ .expect("benchmark should parse");
+
+ let load_queries = benchmark
+ .queries()
+ .get(&QueryDirective::Load)
+ .expect("load queries");
+ assert_eq!(load_queries.len(), 1);
+ assert!(
+ load_queries.iter().all(|query| !query.contains("${")),
+ "all placeholders should be replaced: {load_queries:?}"
+ );
+ let expected_location = format!(
+ "LOCATION '{}/tpch_sf1/csv/generated/generated.1.csv'",
+ data_dir.display()
+ );
+ assert!(
+ load_queries[0].contains(&expected_location),
+ "all load locations should use the non-default DATA_DIR: {load_queries:?}"
+ );
+
+ benchmark
+ .initialize(&ctx)
+ .await
+ .expect("benchmark should load generated csv file");
+ benchmark
+ .run(&ctx, true)
+ .await
+ .expect("benchmark should read generated csv file");
+
+ assert_eq!(formatted_last_results(&benchmark), vec![vec!["42"]]);
+ }
+
#[tokio::test]
async fn parser_rejects_inline_sql_when_query_file_is_provided() {
let temp_dir = tempdir().expect("failed to create benchmark test directory");