Apache DataFusion SQL Query Engine
feat(benchmarks): add sql benchmark sort pushdown SQL suite (#26245)
Bruce Ritchieb5e5c0e925e8
diff --git a/benchmarks/sql_benchmarks/README.md b/benchmarks/sql_benchmarks/README.mdindex 7866f9ebd6..55a1cc5898 100644--- a/benchmarks/sql_benchmarks/README.md+++ b/benchmarks/sql_benchmarks/README.md@@ -41,6 +41,7 @@ in the community:| `push_down_topk` | `ORDER BY ... LIMIT` over outer joins (TPC-H data); exercises pushing a TopK through a join || `pwmj` | Piecewise merge join: classic `INNER`/`LEFT`/`RIGHT`/`FULL` joins over `lhs.key < rhs.key`. Subgroups (`--subgroup`) are match regimes: `no_match`, `null_heavy`, `half_match`, `selective`, `all_match`. Generated with `range()`, no data needed. || `smj` | Sort‑merge join benchmark |+| [`sort_pushdown`](sort_pushdown/README.md) | SQL counterparts of the five sort pushdown variants, using existing Parquet data and query files. || `sort tpch` | Sorting benchmarks against the TPC-H lineitem table || `taxi` | NYC taxi dataset benchmark || `tpcds` | TPC‑DS queries |diff --git a/benchmarks/sql_benchmarks/sort_pushdown/README.md b/benchmarks/sql_benchmarks/sort_pushdown/README.mdnew file mode 100644index 0000000000..9adb5533cb--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/README.md@@ -0,0 +1,46 @@+# Sort pushdown SQL benchmarks++Run the existing sort pushdown queries through `benchmark_runner`. This suite+uses the same Parquet files, with queries inlined in each `.benchmark` file.+The `dfbench` implementation and its commands remain available for comparison.++From `benchmarks/`, prepare data with the existing generators:++```bash+./bench.sh data sort_pushdown+./bench.sh data sort_pushdown_inexact+```++The second command also generates the overlap dataset and requires PyArrow.+The SQL suite does not generate or rewrite data.++| SQL subgroup | Existing benchmark | Dataset directory | Queries |+|--------------------|----------------------------------|---------------------------------|---------|+| `baseline` | `sort_pushdown` | `sort_pushdown` | 1–8 |+| `sorted` | `sort_pushdown_sorted` | `sort_pushdown` | 1–8 |+| `inexact` | `sort_pushdown_inexact` | `sort_pushdown_inexact` | 1–4 |+| `inexact_unsorted` | `sort_pushdown_inexact_unsorted` | `sort_pushdown_inexact` | 1–6 |+| `inexact_overlap` | `sort_pushdown_inexact_overlap` | `sort_pushdown_inexact_overlap` | 1–4 |++Run the standalone SQL suite directly:++```bash+cargo run --release --bin benchmark_runner -- sort_pushdown --iterations 5+```++Use `--subgroup` and `--query` to select a variant and original query number:++```bash+cargo run --release --bin benchmark_runner -- sort_pushdown \+ --subgroup sorted --query 2 --iterations 5+```++Omit `--subgroup` to run all 30 cases. `--path` points to the parent directory+containing the dataset directories in the table above; it defaults to+`benchmarks/data`. Each dataset must contain a `lineitem/` directory.++The suite forces statistics collection, as the legacy runner does. The three+inexact subgroups also enable Parquet filter pushdown, matching their `bench.sh`+commands. The ordered subgroups declare `l_orderkey ASC NULLS LAST`, including+for queries that request descending output. Other settings use the common+runner options and environment variables.diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q01.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q01.benchmarknew file mode 100644index 0000000000..57e8188ce6--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q01.benchmark@@ -0,0 +1,14 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=01+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Sort elimination: ORDER BY sort key ASC (full scan)+-- With --sorted: SortExec removed, sequential scan in file order+-- Without --sorted: full SortExec required+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q02.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q02.benchmarknew file mode 100644index 0000000000..f6b393d2c4--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q02.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=02+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Sort elimination + limit pushdown+-- With --sorted: SortExec removed + limit pushed to DataSourceExec+-- Without --sorted: TopK sort over all data+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q03.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q03.benchmarknew file mode 100644index 0000000000..8017c41e71--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q03.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=03+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Sort elimination: wide projection (all columns)+-- Tests sort elimination benefit with larger row payload+SELECT *+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q04.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q04.benchmarknew file mode 100644index 0000000000..c0ffe5715b--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q04.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=04+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Sort elimination + limit: wide projection+SELECT *+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q05.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q05.benchmarknew file mode 100644index 0000000000..680e50f0f8--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q05.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=05+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Reverse scan: ORDER BY DESC LIMIT (narrow projection)+-- With --sorted: reverse_row_groups=true + TopK + stats init + cumulative prune+-- Without --sorted: full TopK sort over all data+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q06.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q06.benchmarknew file mode 100644index 0000000000..020a3c1016--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q06.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=06+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Reverse scan: ORDER BY DESC LIMIT larger fetch (narrow projection)+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q07.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q07.benchmarknew file mode 100644index 0000000000..19282746c0--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q07.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=07+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Reverse scan: wide projection + DESC LIMIT+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q08.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q08.benchmarknew file mode 100644index 0000000000..7c9cf54a53--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/baseline/q08.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=08+VARIANT=baseline+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=unordered++run+-- Reverse scan: wide projection + DESC LIMIT larger fetch+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q01.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q01.benchmarknew file mode 100644index 0000000000..c9ced97035--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q01.benchmark@@ -0,0 +1,16 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=01+VARIANT=inexact+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=ordered++run+-- Inexact path: TopK + DESC LIMIT on ASC-declared file.+-- With RG reorder, the first RG read contains the highest max value,+-- so TopK's threshold tightens quickly and subsequent RGs get filtered+-- efficiently via dynamic filter pushdown.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q02.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q02.benchmarknew file mode 100644index 0000000000..509bbd6188--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q02.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=02+VARIANT=inexact+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=ordered++run+-- Inexact path: TopK + DESC LIMIT with larger fetch (1000).+-- Larger LIMIT means more row_replacements; RG reorder reduces the+-- total replacement count by tightening the threshold faster.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q03.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q03.benchmarknew file mode 100644index 0000000000..9ddc4c3b39--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q03.benchmark@@ -0,0 +1,16 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=03+VARIANT=inexact+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=ordered++run+-- Inexact path: wide projection (all columns) + DESC LIMIT.+-- Shows the row-level filter benefit: with a tight threshold from the+-- first RG, subsequent RGs skip decoding non-sort columns for filtered+-- rows — bigger wins for wide tables.+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q04.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q04.benchmarknew file mode 100644index 0000000000..381236aeb9--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact/q04.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=04+VARIANT=inexact+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=ordered++run+-- Inexact path: wide projection + DESC LIMIT with larger fetch.+-- Combines wide-row row-level filter benefit with larger LIMIT to+-- demonstrate cumulative gains from RG reorder.+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q01.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q01.benchmarknew file mode 100644index 0000000000..03c84615f5--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q01.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=01+VARIANT=inexact_overlap+DATASET=sort_pushdown_inexact_overlap+SETTINGS=inexact+ORDERING=ordered++run+-- Overlapping RGs: TopK + DESC LIMIT on file with partially overlapping+-- row groups (simulates streaming data with network jitter).+-- RG reorder places highest-max RG first for fastest threshold convergence.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q02.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q02.benchmarknew file mode 100644index 0000000000..e50a1565e2--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q02.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=02+VARIANT=inexact_overlap+DATASET=sort_pushdown_inexact_overlap+SETTINGS=inexact+ORDERING=ordered++run+-- Overlapping RGs: DESC LIMIT with larger fetch.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q03.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q03.benchmarknew file mode 100644index 0000000000..b9017038a9--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q03.benchmark@@ -0,0 +1,14 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=03+VARIANT=inexact_overlap+DATASET=sort_pushdown_inexact_overlap+SETTINGS=inexact+ORDERING=ordered++run+-- Overlapping RGs: wide projection + DESC LIMIT.+-- Row-level filter benefit: tight threshold skips decoding non-sort columns.+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q04.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q04.benchmarknew file mode 100644index 0000000000..9872d51db8--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_overlap/q04.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=04+VARIANT=inexact_overlap+DATASET=sort_pushdown_inexact_overlap+SETTINGS=inexact+ORDERING=ordered++run+-- Overlapping RGs: wide projection + DESC LIMIT larger fetch.+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q01.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q01.benchmarknew file mode 100644index 0000000000..7f264c978e--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q01.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=01+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: TopK + ASC LIMIT on file without declared ordering.+-- Tests RG reorder benefit when no WITH ORDER is declared — the+-- Unsupported path in try_pushdown_sort triggers RG reorder.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q02.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q02.benchmarknew file mode 100644index 0000000000..e8b848fec3--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q02.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=02+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: TopK + ASC LIMIT with larger fetch.+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q03.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q03.benchmarknew file mode 100644index 0000000000..6e6d2f865b--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q03.benchmark@@ -0,0 +1,14 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=03+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: wide projection + ASC LIMIT.+-- Shows row-level filter benefit when RG reorder tightens TopK threshold.+SELECT *+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q04.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q04.benchmarknew file mode 100644index 0000000000..3013172eeb--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q04.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=04+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: wide projection + ASC LIMIT with larger fetch.+SELECT *+FROM lineitem+ORDER BY l_orderkey+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q05.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q05.benchmarknew file mode 100644index 0000000000..89d59301f8--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q05.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=05+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: DESC LIMIT (no declared ordering = no reverse scan).+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q06.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q06.benchmarknew file mode 100644index 0000000000..bbad068ca4--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/inexact_unsorted/q06.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=06+VARIANT=inexact_unsorted+DATASET=sort_pushdown_inexact+SETTINGS=inexact+ORDERING=unordered++run+-- Unsupported path: wide projection + DESC LIMIT.+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q01.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q01.benchmarknew file mode 100644index 0000000000..9aa7096042--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q01.benchmark@@ -0,0 +1,14 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=01+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Sort elimination: ORDER BY sort key ASC (full scan)+-- With --sorted: SortExec removed, sequential scan in file order+-- Without --sorted: full SortExec required+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q02.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q02.benchmarknew file mode 100644index 0000000000..652b0de0d0--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q02.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=02+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Sort elimination + limit pushdown+-- With --sorted: SortExec removed + limit pushed to DataSourceExec+-- Without --sorted: TopK sort over all data+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q03.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q03.benchmarknew file mode 100644index 0000000000..7eff43cc71--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q03.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=03+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Sort elimination: wide projection (all columns)+-- Tests sort elimination benefit with larger row payload+SELECT *+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q04.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q04.benchmarknew file mode 100644index 0000000000..7238005988--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q04.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=04+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Sort elimination + limit: wide projection+SELECT *+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q05.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q05.benchmarknew file mode 100644index 0000000000..fcb7edc5b6--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q05.benchmark@@ -0,0 +1,15 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=05+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Reverse scan: ORDER BY DESC LIMIT (narrow projection)+-- With --sorted: reverse_row_groups=true + TopK + stats init + cumulative prune+-- Without --sorted: full TopK sort over all data+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q06.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q06.benchmarknew file mode 100644index 0000000000..095ba183dc--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q06.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=06+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Reverse scan: ORDER BY DESC LIMIT larger fetch (narrow projection)+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q07.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q07.benchmarknew file mode 100644index 0000000000..e7c28c202d--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q07.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=07+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Reverse scan: wide projection + DESC LIMIT+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 100diff --git a/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q08.benchmark b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q08.benchmarknew file mode 100644index 0000000000..dd13c09fba--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/benchmarks/sorted/q08.benchmark@@ -0,0 +1,13 @@+template sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template+QPAD=08+VARIANT=sorted+DATASET=sort_pushdown+SETTINGS=standard+ORDERING=ordered++run+-- Reverse scan: wide projection + DESC LIMIT larger fetch+SELECT *+FROM lineitem+ORDER BY l_orderkey DESC+LIMIT 1000diff --git a/benchmarks/sql_benchmarks/sort_pushdown/load/inexact.sql b/benchmarks/sql_benchmarks/sort_pushdown/load/inexact.sqlnew file mode 100644index 0000000000..ad6a789e3f--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/load/inexact.sql@@ -0,0 +1,3 @@+-- Match the legacy runner and the inexact bench.sh commands.+SET datafusion.execution.collect_statistics = true;+SET datafusion.execution.parquet.pushdown_filters = true;diff --git a/benchmarks/sql_benchmarks/sort_pushdown/load/ordered.sql b/benchmarks/sql_benchmarks/sort_pushdown/load/ordered.sqlnew file mode 100644index 0000000000..a1e79c5404--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/load/ordered.sql@@ -0,0 +1,4 @@+CREATE EXTERNAL TABLE lineitem+STORED AS PARQUET+LOCATION '${DATA_DIR}/${DATASET}/lineitem/'+WITH ORDER (l_orderkey ASC NULLS LAST);diff --git a/benchmarks/sql_benchmarks/sort_pushdown/load/standard.sql b/benchmarks/sql_benchmarks/sort_pushdown/load/standard.sqlnew file mode 100644index 0000000000..b9e841e815--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/load/standard.sql@@ -0,0 +1,2 @@+-- Match the legacy runner's unconditional statistics collection.+SET datafusion.execution.collect_statistics = true;diff --git a/benchmarks/sql_benchmarks/sort_pushdown/load/unordered.sql b/benchmarks/sql_benchmarks/sort_pushdown/load/unordered.sqlnew file mode 100644index 0000000000..96f20319ad--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/load/unordered.sql@@ -0,0 +1,3 @@+CREATE EXTERNAL TABLE lineitem+STORED AS PARQUET+LOCATION '${DATA_DIR}/${DATASET}/lineitem/';diff --git a/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template b/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.templatenew file mode 100644index 0000000000..ad357edb72--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.benchmark.template@@ -0,0 +1,10 @@+name Q${QPAD}+group sort_pushdown+subgroup ${VARIANT}++load sql_benchmarks/sort_pushdown/load/${SETTINGS}.sql++load sql_benchmarks/sort_pushdown/load/${ORDERING}.sql++cleanup+DROP TABLE lineitem;diff --git a/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.suite b/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.suitenew file mode 100644index 0000000000..87a6b789df--- /dev/null+++ b/benchmarks/sql_benchmarks/sort_pushdown/sort_pushdown.suite@@ -0,0 +1,18 @@+description = "Sort pushdown on existing TPC-H lineitem Parquet data"++query_pattern = "q{QUERY_ID_PADDED}.benchmark"++[path_replacements]+DATA_DIR = "../../data"++[[examples]]+command = "cargo run --release --bin benchmark_runner -- sort_pushdown --subgroup baseline --iterations 5"+description = "Run the original eight queries without declared ordering."++[[examples]]+command = "cargo run --release --bin benchmark_runner -- sort_pushdown --subgroup sorted --query 2 --iterations 5"+description = "Run the ascending LIMIT query with declared ascending ordering."++[[examples]]+command = "cargo run --release --bin benchmark_runner -- sort_pushdown --subgroup inexact --iterations 5"+description = "Run DESC TopK queries on the existing inexact dataset."