Apache DataFusion SQL Query Engine
bench: add ASOF join benchmarks (#23833)
Xuanwo04ca5aced902
diff --git a/benchmarks/README.md b/benchmarks/README.mdindex 489f2d485a..df96b1aeeb 100644--- a/benchmarks/README.md+++ b/benchmarks/README.md@@ -923,6 +923,21 @@ cargo run --release --bin dfbench -- h2o --join-paths ./benchmarks/data/h2o/J1_1# Micro-Benchmarks+## ASOF Join++This benchmark exercises ASOF joins across relative input sizes, available+input ordering, equality-group cardinality and skew, match direction, and+payload width. One keyed workload reads pre-sorted Parquet inputs from+`benchmarks/data/asof_join` with declared ordering so it measures the join+without including an input sort.++### Example Run++```bash+./bench.sh data asof_join+./bench.sh run asof_join+```+## Nested Loop JoinThis benchmark focuses on the performance of queries with nested loop joins, minimizing other overheads such as scanning data sources or evaluating predicates.diff --git a/benchmarks/bench.sh b/benchmarks/bench.shindex df33d2aec7..082651656e 100755--- a/benchmarks/bench.sh+++ b/benchmarks/bench.sh@@ -160,6 +160,7 @@ imdb: Join Order Benchmark (JOB) using the IMDB dataset conver# Micro-Benchmarks (specific operators and features)cancellation: How long cancelling a query takes+asof_join: ASOF join workloads varying size, ordering, grouping, match direction, and payload widthnlj: Benchmark for simple nested loop joins, testing various join scenarioshj: Benchmark for simple hash joins, testing various join scenariossmj: Benchmark for simple sort merge joins, testing various join scenarios@@ -238,6 +239,7 @@ main() {data_clickbench_1data_clickbench_partitioneddata_imdb+ data_asof_join# nlj uses range() function, no data generation needed;;tpch)@@ -269,6 +271,9 @@ main() {# Data is generated inline by the suite's load SQL (COPY).echo "parquet_row_filter_skip: no external data to generate";;+ asof_join)+ data_asof_join+ ;;tpcds)data_tpcds;;@@ -479,6 +484,7 @@ main() {run_h2o_join "BIG" "PARQUET" "join"run_imdbrun_external_aggr+ run_asof_joinrun_nljrun_hjrun_tpcds@@ -512,6 +518,9 @@ main() {parquet_row_filter_skip)run_parquet_row_filter_skip;;+ asof_join)+ run_asof_join+ ;;tpcds)run_tpcds;;@@ -1632,6 +1641,51 @@ run_topk_sorted_tpch() {$CARGO_COMMAND --bin dfbench -- sort-tpch --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" --sorted --limit 100 ${QUERY_ARG} ${LATENCY_ARG}}+# Generates the pre-sorted Parquet inputs for the ASOF join benchmark.+data_asof_join() {+ ASOF_DIR="${DATA_DIR}/asof_join"+ LEFT_FILE="${ASOF_DIR}/q07_left.parquet"+ RIGHT_FILE="${ASOF_DIR}/q07_right.parquet"++ if [ -f "${LEFT_FILE}" ] && [ -f "${RIGHT_FILE}" ]; then+ echo "ASOF join benchmark data already exists at ${ASOF_DIR}"+ return+ fi++ mkdir -p "${ASOF_DIR}"+ echo "Generating ASOF join benchmark data at ${ASOF_DIR}..."+ (+ cd "${DATAFUSION_DIR}"+ debug_run $CARGO_COMMAND -p datafusion-cli -- -c "+ COPY (+ SELECT value / 10 AS group_key,+ value % 10 + 1 AS ts,+ value AS payload+ FROM range(100000)+ ORDER BY group_key, ts+ )+ TO '${LEFT_FILE}' STORED AS PARQUET;++ COPY (+ SELECT value / 10 AS group_key,+ value % 10 AS ts,+ value AS payload+ FROM range(100000)+ ORDER BY group_key, ts+ )+ TO '${RIGHT_FILE}' STORED AS PARQUET;+ "+ )+}++# Runs the ASOF join benchmark+run_asof_join() {+ RESULTS_FILE="${RESULTS_DIR}/asof_join.json"+ echo "RESULTS_FILE: ${RESULTS_FILE}"+ echo "Running ASOF join benchmark..."+ debug_run $CARGO_COMMAND --bin benchmark_runner -- asof_join --iterations 5 --path "${DATA_DIR}" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}+}+# Runs the nlj benchmarkrun_nlj() {RESULTS_FILE="${RESULTS_DIR}/nlj.json"diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.mdindex b8bf8adede..c5a4136e8e 100644--- a/benchmarks/sql_benchmarks/README.md+++ b/benchmarks/sql_benchmarks/README.md@@ -29,6 +29,7 @@ in the community:| Benchmark Suite | Description ||-----------------------|--------------------------------------------------------------------|+| `asof_join` | ASOF join benchmarks across size, ordering, grouping, match direction, and payload width || `clickbench` | ClickBench benchmark || `clickbench extended` | 12 additional, more complex queries against the Clickbench dataset || `clickbench_sorted` | ClickBench benchmark using a pre-sorted hits file. |diff --git a/benchmarks/sql_benchmarks/asof_join/asof_join.suite b/benchmarks/sql_benchmarks/asof_join/asof_join.suitenew file mode 100644index 0000000000..36198cea09--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/asof_join.suite@@ -0,0 +1,21 @@+# Workloads vary these axes:+# - relative left and right input sizes+# - required ordering already available or included in the measured query+# - no equality key, many balanced groups, or heavily skewed groups+# - latest right row at or before, or earliest right row at or after, each left row+# - narrow integer or wide UTF-8 payloads++description = "ASOF join SQL benchmarks"++query_pattern = "q{QUERY_ID_PADDED}.benchmark"++[path_replacements]+DATA_DIR = "../../data"++[[examples]]+command = "cargo run --release --bin benchmark_runner -- asof_join"+description = "Run all ASOF join queries."++[[examples]]+command = "cargo run --release --bin benchmark_runner -- asof_join --query 7"+description = "Run the pre-sorted keyed ASOF join query."diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmarknew file mode 100644index 0000000000..fdbea8bd45--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q01.benchmark@@ -0,0 +1,17 @@+name Q01+group asof_join++expect_plan AsOfJoinExec++run+-- Left: 1M rows; right: 10K rows; no equality key; ascending timestamps.+-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).+WITH left_input AS (+ SELECT value AS ts, value AS payload FROM range(1000000)+),+right_input AS (+ SELECT value AS ts, value AS payload FROM range(10000)+)+SELECT l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts);diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmarknew file mode 100644index 0000000000..0c857192c4--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q02.benchmark@@ -0,0 +1,25 @@+name Q02+group asof_join++expect_plan AsOfJoinExec++run+-- Left and right: 1M rows each; 10K balanced equality groups; integer payloads.+-- Inputs are not ordered by (key, ts), so required ordering is included.+-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).+WITH left_input AS (+ SELECT value % 10000 AS key,+ value / 10000 + 1 AS ts,+ value AS payload+ FROM range(1000000)+),+right_input AS (+ SELECT value % 10000 AS key,+ value / 10000 AS ts,+ value AS payload+ FROM range(1000000)+)+SELECT l.key, l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)+ON l.key = r.key;diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmarknew file mode 100644index 0000000000..af051fc106--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q03.benchmark@@ -0,0 +1,26 @@+name Q03+group asof_join++expect_plan AsOfJoinExec++run+-- Left and right: 250K rows each; 10K balanced equality groups.+-- Inputs are not ordered by (key, ts), so required ordering is included.+-- Matches the latest right row at or before each left timestamp and materializes+-- 256-byte UTF-8 payloads from both inputs.+WITH left_input AS (+ SELECT value % 10000 AS key,+ value / 10000 + 1 AS ts,+ repeat('x', 256) AS payload+ FROM range(250000)+),+right_input AS (+ SELECT value % 10000 AS key,+ value / 10000 AS ts,+ repeat('y', 256) AS payload+ FROM range(250000)+)+SELECT l.key, l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)+ON l.key = r.key;diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmarknew file mode 100644index 0000000000..1c58063bbf--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q04.benchmark@@ -0,0 +1,18 @@+name Q04+group asof_join++expect_plan AsOfJoinExec++run+-- Left and right: 500K rows each; no equality key; ascending source timestamps.+-- Matches the earliest right row at or after each left timestamp (l.ts <= r.ts),+-- so the measured query includes the required descending ordering.+WITH left_input AS (+ SELECT value AS ts, value AS payload FROM range(500000)+),+right_input AS (+ SELECT value AS ts, value AS payload FROM range(500000)+)+SELECT l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts <= r.ts);diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmarknew file mode 100644index 0000000000..b1d10c66df--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q05.benchmark@@ -0,0 +1,18 @@+name Q05+group asof_join++expect_plan AsOfJoinExec++run+-- Left: 100K rows; right: 1M rows; no equality key; ascending timestamps.+-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).+-- Exercises a right input that is ten times larger than the left input.+WITH left_input AS (+ SELECT value AS ts, value AS payload FROM range(100000)+),+right_input AS (+ SELECT value AS ts, value AS payload FROM range(1000000)+)+SELECT l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts);diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmarknew file mode 100644index 0000000000..c6cb12a45e--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q06.benchmark@@ -0,0 +1,25 @@+name Q06+group asof_join++expect_plan AsOfJoinExec++run+-- Left and right: 1M rows each; one equality group contains 95% of rows.+-- Inputs are not ordered by (key, ts), so required ordering is included.+-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).+WITH left_input AS (+ SELECT CASE WHEN value % 100 < 95 THEN 0 ELSE value % 16 + 1 END AS key,+ value / 100 + 1 AS ts,+ value AS payload+ FROM range(1000000)+),+right_input AS (+ SELECT CASE WHEN value % 100 < 95 THEN 0 ELSE value % 16 + 1 END AS key,+ value / 100 AS ts,+ value AS payload+ FROM range(1000000)+)+SELECT l.key, l.ts, l.payload, r.payload AS right_payload+FROM left_input l+ASOF JOIN right_input r MATCH_CONDITION (l.ts >= r.ts)+ON l.key = r.key;diff --git a/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmark b/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmarknew file mode 100644index 0000000000..7d878f468b--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/benchmarks/q07.benchmark@@ -0,0 +1,25 @@+name Q07+group asof_join++load sql_benchmarks/asof_join/init/load_q07.sql++assert II+SELECT+ (SELECT count(*) FROM asof_left_sorted) AS left_rows,+ (SELECT count(*) FROM asof_right_sorted) AS right_rows+----+100000|100000++expect_plan AsOfJoinExec++run+-- Left and right: 100K rows each; 10K balanced equality groups; integer payloads.+-- Both Parquet inputs declare (group_key, ts) ordering, so the measured query does not+-- include the input sorts required by the other keyed workloads.+-- Matches the latest right row at or before each left timestamp (l.ts >= r.ts).+SELECT l.group_key, l.ts, l.payload, r.payload AS right_payload+FROM asof_left_sorted l+ASOF JOIN asof_right_sorted r MATCH_CONDITION (l.ts >= r.ts)+ON l.group_key = r.group_key;++cleanup sql_benchmarks/asof_join/init/cleanup_q07.sqldiff --git a/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sql b/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sqlnew file mode 100644index 0000000000..85887a15ab--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/init/cleanup_q07.sql@@ -0,0 +1,2 @@+DROP TABLE IF EXISTS asof_left_sorted;+DROP TABLE IF EXISTS asof_right_sorted;diff --git a/benchmarks/sql_benchmarks/asof_join/init/load_q07.sql b/benchmarks/sql_benchmarks/asof_join/init/load_q07.sqlnew file mode 100644index 0000000000..9869e95ba8--- /dev/null+++ b/benchmarks/sql_benchmarks/asof_join/init/load_q07.sql@@ -0,0 +1,17 @@+CREATE EXTERNAL TABLE asof_left_sorted (+ group_key BIGINT,+ ts BIGINT,+ payload BIGINT+)+STORED AS PARQUET+LOCATION '${DATA_DIR:-data}/asof_join/q07_left.parquet'+WITH ORDER (group_key ASC NULLS FIRST, ts ASC NULLS FIRST);++CREATE EXTERNAL TABLE asof_right_sorted (+ group_key BIGINT,+ ts BIGINT,+ payload BIGINT+)+STORED AS PARQUET+LOCATION '${DATA_DIR:-data}/asof_join/q07_right.parquet'+WITH ORDER (group_key ASC NULLS FIRST, ts ASC NULLS FIRST);