Apache DataFusion SQL Query Engine
feat: add sort pushdown benchmark and SLT tests (#21213)
Qi Zhu80a8bb8a7be1
diff --git a/benchmarks/bench.sh b/benchmarks/bench.shindex 0fc6ede3b3..b8c9ff5c8e 100755--- a/benchmarks/bench.sh+++ b/benchmarks/bench.sh@@ -106,6 +106,10 @@ clickbench_partitioned: ClickBench queries against partitioned (100 files) parquclickbench_pushdown: ClickBench queries against partitioned (100 files) parquet w/ filter_pushdown enabledclickbench_extended: ClickBench \"inspired\" queries against a single parquet (DataFusion specific)+# Sort Pushdown Benchmarks+sort_pushdown: Sort pushdown baseline (no WITH ORDER) on TPC-H data (SF=1)+sort_pushdown_sorted: Sort pushdown with WITH ORDER — tests sort elimination on non-overlapping files+# Sorted Data Benchmarks (ORDER BY Optimization)clickbench_sorted: ClickBench queries on pre-sorted data using prefer_existing_sort (tests sort elimination optimization)@@ -309,6 +313,10 @@ main() {# same data as for tpchdata_tpch "1" "parquet";;+ sort_pushdown|sort_pushdown_sorted)+ # same data as for tpch+ data_tpch "1" "parquet"+ ;;sort_tpch)# same data as for tpchdata_tpch "1" "parquet"@@ -509,6 +517,12 @@ main() {external_aggr)run_external_aggr;;+ sort_pushdown)+ run_sort_pushdown+ ;;+ sort_pushdown_sorted)+ run_sort_pushdown_sorted+ ;;sort_tpch)run_sort_tpch "1";;@@ -1070,6 +1084,22 @@ run_external_aggr() {debug_run $CARGO_COMMAND --bin external_aggr -- benchmark --partitions 4 --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" ${QUERY_ARG}}+# Runs the sort pushdown benchmark (without WITH ORDER)+run_sort_pushdown() {+ TPCH_DIR="${DATA_DIR}/tpch_sf1"+ RESULTS_FILE="${RESULTS_DIR}/sort_pushdown.json"+ echo "Running sort pushdown benchmark (no WITH ORDER)..."+ debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}+}++# Runs the sort pushdown benchmark with WITH ORDER (enables sort elimination)+run_sort_pushdown_sorted() {+ TPCH_DIR="${DATA_DIR}/tpch_sf1"+ RESULTS_FILE="${RESULTS_DIR}/sort_pushdown_sorted.json"+ echo "Running sort pushdown benchmark (with WITH ORDER)..."+ debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path "${TPCH_DIR}" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}+}+# Runs the sort integration benchmarkrun_sort_tpch() {SCALE_FACTOR=$1diff --git a/benchmarks/queries/sort_pushdown/q1.sql b/benchmarks/queries/sort_pushdown/q1.sqlnew file mode 100644index 0000000000..f5f51a5d40--- /dev/null+++ b/benchmarks/queries/sort_pushdown/q1.sql@@ -0,0 +1,6 @@+-- Sort elimination: ORDER BY sort key ASC (full scan)+-- With --sorted: SortExec removed, sequential scan in file order+-- Without --sorted: full SortExec required+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/queries/sort_pushdown/q2.sql b/benchmarks/queries/sort_pushdown/q2.sqlnew file mode 100644index 0000000000..29a0e127cb--- /dev/null+++ b/benchmarks/queries/sort_pushdown/q2.sql@@ -0,0 +1,7 @@+-- Sort elimination + limit pushdown+-- With --sorted: SortExec removed + limit pushed to DataSourceExec+-- Without --sorted: TopK sort over all data+SELECT l_orderkey, l_partkey, l_suppkey+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/queries/sort_pushdown/q3.sql b/benchmarks/queries/sort_pushdown/q3.sqlnew file mode 100644index 0000000000..e11b48659a--- /dev/null+++ b/benchmarks/queries/sort_pushdown/q3.sql@@ -0,0 +1,5 @@+-- Sort elimination: wide projection (all columns)+-- Tests sort elimination benefit with larger row payload+SELECT *+FROM lineitem+ORDER BY l_orderkeydiff --git a/benchmarks/queries/sort_pushdown/q4.sql b/benchmarks/queries/sort_pushdown/q4.sqlnew file mode 100644index 0000000000..99500c3719--- /dev/null+++ b/benchmarks/queries/sort_pushdown/q4.sql@@ -0,0 +1,5 @@+-- Sort elimination + limit: wide projection+SELECT *+FROM lineitem+ORDER BY l_orderkey+LIMIT 100diff --git a/benchmarks/src/bin/dfbench.rs b/benchmarks/src/bin/dfbench.rsindex 7e21890519..3b1f54291e 100644--- a/benchmarks/src/bin/dfbench.rs+++ b/benchmarks/src/bin/dfbench.rs@@ -34,7 +34,8 @@ static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc;static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;use datafusion_benchmarks::{- cancellation, clickbench, h2o, hj, imdb, nlj, smj, sort_tpch, tpcds, tpch,+ cancellation, clickbench, h2o, hj, imdb, nlj, smj, sort_pushdown, sort_tpch, tpcds,+ tpch,};#[derive(Debug, Parser)]@@ -53,6 +54,7 @@ enum Options {Imdb(imdb::RunOpt),Nlj(nlj::RunOpt),Smj(smj::RunOpt),+ SortPushdown(sort_pushdown::RunOpt),SortTpch(sort_tpch::RunOpt),Tpch(tpch::RunOpt),Tpcds(tpcds::RunOpt),@@ -72,6 +74,7 @@ pub async fn main() -> Result<()> {Options::Imdb(opt) => Box::pin(opt.run()).await,Options::Nlj(opt) => opt.run().await,Options::Smj(opt) => opt.run().await,+ Options::SortPushdown(opt) => opt.run().await,Options::SortTpch(opt) => opt.run().await,Options::Tpch(opt) => Box::pin(opt.run()).await,Options::Tpcds(opt) => Box::pin(opt.run()).await,diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rsindex a3bc221840..7e2196e89e 100644--- a/benchmarks/src/lib.rs+++ b/benchmarks/src/lib.rs@@ -23,6 +23,7 @@ pub mod hj;pub mod imdb;pub mod nlj;pub mod smj;+pub mod sort_pushdown;pub mod sort_tpch;pub mod tpcds;pub mod tpch;diff --git a/benchmarks/src/sort_pushdown.rs b/benchmarks/src/sort_pushdown.rsnew file mode 100644index 0000000000..e7fce1921e--- /dev/null+++ b/benchmarks/src/sort_pushdown.rs@@ -0,0 +1,282 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++//! Benchmark for sort pushdown optimization.+//!+//! Tests performance of sort elimination when files are non-overlapping and+//! internally sorted (declared via `--sorted` / `WITH ORDER`).+//!+//! Queries are loaded from external SQL files under `queries/sort_pushdown/`+//! so they can also be run directly with `datafusion-cli`.+//!+//! # Usage+//!+//! ```text+//! # Prepare sorted TPCH lineitem data (SF=1)+//! ./bench.sh data sort_pushdown+//!+//! # Baseline (no WITH ORDER, full SortExec)+//! ./bench.sh run sort_pushdown+//!+//! # With sort elimination (WITH ORDER, SortExec removed)+//! ./bench.sh run sort_pushdown_sorted+//! ```++use clap::Args;+use futures::StreamExt;+use std::path::PathBuf;+use std::sync::Arc;++use datafusion::datasource::TableProvider;+use datafusion::datasource::file_format::parquet::ParquetFormat;+use datafusion::datasource::listing::{+ ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl,+};+use datafusion::error::Result;+use datafusion::execution::SessionStateBuilder;+use datafusion::physical_plan::display::DisplayableExecutionPlan;+use datafusion::physical_plan::{displayable, execute_stream};+use datafusion::prelude::*;+use datafusion_common::DEFAULT_PARQUET_EXTENSION;+use datafusion_common::instant::Instant;++use crate::util::{BenchmarkRun, CommonOpt, QueryResult, print_memory_stats};++/// Default path to query files, relative to the benchmark root+const SORT_PUSHDOWN_QUERY_DIR: &str = "queries/sort_pushdown";++#[derive(Debug, Args)]+pub struct RunOpt {+ /// Common options+ #[command(flatten)]+ common: CommonOpt,++ /// Sort pushdown query number (1-4). If not specified, runs all queries+ #[arg(short, long)]+ pub query: Option<usize>,++ /// Path to data files (lineitem). Only parquet format is supported.+ #[arg(required = true, short = 'p', long = "path")]+ path: PathBuf,++ /// Path to JSON benchmark result to be compared using `compare.py`+ #[arg(short = 'o', long = "output")]+ output_path: Option<PathBuf>,++ /// Path to directory containing query SQL files (q1.sql, q2.sql, ...).+ /// Defaults to `queries/sort_pushdown/` relative to current directory.+ #[arg(long = "queries-path")]+ queries_path: Option<PathBuf>,++ /// Mark the first column (l_orderkey) as sorted via WITH ORDER.+ /// When set, enables sort elimination for matching queries.+ #[arg(short = 't', long = "sorted")]+ sorted: bool,+}++impl RunOpt {+ const TABLES: [&'static str; 1] = ["lineitem"];++ fn queries_dir(&self) -> PathBuf {+ self.queries_path+ .clone()+ .unwrap_or_else(|| PathBuf::from(SORT_PUSHDOWN_QUERY_DIR))+ }++ fn load_query(&self, query_id: usize) -> Result<String> {+ let path = self.queries_dir().join(format!("q{query_id}.sql"));+ std::fs::read_to_string(&path).map_err(|e| {+ datafusion_common::DataFusionError::Execution(format!(+ "Failed to read query file {}: {e}",+ path.display()+ ))+ })+ }++ fn available_queries(&self) -> Vec<usize> {+ let dir = self.queries_dir();+ let mut ids = Vec::new();+ if let Ok(entries) = std::fs::read_dir(&dir) {+ for entry in entries.flatten() {+ let name = entry.file_name();+ let name = name.to_string_lossy();+ if let Some(rest) = name.strip_prefix('q')+ && let Some(num_str) = rest.strip_suffix(".sql")+ && let Ok(id) = num_str.parse::<usize>()+ {+ ids.push(id);+ }+ }+ }+ ids.sort();+ ids+ }++ pub async fn run(&self) -> Result<()> {+ let mut benchmark_run = BenchmarkRun::new();++ let query_ids = match self.query {+ Some(query_id) => vec![query_id],+ None => self.available_queries(),+ };++ for query_id in query_ids {+ benchmark_run.start_new_case(&format!("{query_id}"));++ let query_results = self.benchmark_query(query_id).await;+ match query_results {+ Ok(query_results) => {+ for iter in query_results {+ benchmark_run.write_iter(iter.elapsed, iter.row_count);+ }+ }+ Err(e) => {+ benchmark_run.mark_failed();+ eprintln!("Query {query_id} failed: {e}");+ }+ }+ }++ benchmark_run.maybe_write_json(self.output_path.as_ref())?;+ benchmark_run.maybe_print_failures();+ Ok(())+ }++ async fn benchmark_query(&self, query_id: usize) -> Result<Vec<QueryResult>> {+ let sql = self.load_query(query_id)?;++ let config = self.common.config()?;+ let rt = self.common.build_runtime()?;+ let state = SessionStateBuilder::new()+ .with_config(config)+ .with_runtime_env(rt)+ .with_default_features()+ .build();+ let ctx = SessionContext::from(state);++ self.register_tables(&ctx).await?;++ let mut millis = vec![];+ let mut query_results = vec![];+ for i in 0..self.iterations() {+ let start = Instant::now();++ let row_count = self.execute_query(&ctx, sql.as_str()).await?;++ let elapsed = start.elapsed();+ let ms = elapsed.as_secs_f64() * 1000.0;+ millis.push(ms);++ println!(+ "Query {query_id} iteration {i} took {ms:.1} ms and returned {row_count} rows"+ );+ query_results.push(QueryResult { elapsed, row_count });+ }++ let avg = millis.iter().sum::<f64>() / millis.len() as f64;+ println!("Query {query_id} avg time: {avg:.2} ms");++ print_memory_stats();++ Ok(query_results)+ }++ async fn register_tables(&self, ctx: &SessionContext) -> Result<()> {+ for table in Self::TABLES {+ let table_provider = self.get_table(ctx, table).await?;+ ctx.register_table(table, table_provider)?;+ }+ Ok(())+ }++ async fn execute_query(&self, ctx: &SessionContext, sql: &str) -> Result<usize> {+ let debug = self.common.debug;+ let plan = ctx.sql(sql).await?;+ let (state, plan) = plan.into_parts();++ if debug {+ println!("=== Logical plan ===\n{plan}\n");+ }++ let plan = state.optimize(&plan)?;+ if debug {+ println!("=== Optimized logical plan ===\n{plan}\n");+ }+ let physical_plan = state.create_physical_plan(&plan).await?;+ if debug {+ println!(+ "=== Physical plan ===\n{}\n",+ displayable(physical_plan.as_ref()).indent(true)+ );+ }++ let mut row_count = 0;+ let mut stream = execute_stream(physical_plan.clone(), state.task_ctx())?;+ while let Some(batch) = stream.next().await {+ row_count += batch?.num_rows();+ }++ if debug {+ println!(+ "=== Physical plan with metrics ===\n{}\n",+ DisplayableExecutionPlan::with_metrics(physical_plan.as_ref())+ .indent(true)+ );+ }++ Ok(row_count)+ }++ async fn get_table(+ &self,+ ctx: &SessionContext,+ table: &str,+ ) -> Result<Arc<dyn TableProvider>> {+ let path = self.path.to_str().unwrap();+ let state = ctx.state();+ let path = format!("{path}/{table}");+ let format = Arc::new(+ ParquetFormat::default()+ .with_options(ctx.state().table_options().parquet.clone()),+ );+ let extension = DEFAULT_PARQUET_EXTENSION;++ let options = ListingOptions::new(format)+ .with_file_extension(extension)+ .with_collect_stat(true); // Always collect statistics for sort pushdown++ let table_path = ListingTableUrl::parse(path)?;+ let schema = options.infer_schema(&state, &table_path).await?;+ let options = if self.sorted {+ let key_column_name = schema.fields()[0].name();+ options+ .with_file_sort_order(vec![vec![col(key_column_name).sort(true, false)]])+ } else {+ options+ };++ let config = ListingTableConfig::new(table_path)+ .with_listing_options(options)+ .with_schema(schema);++ Ok(Arc::new(ListingTable::try_new(config)?))+ }++ fn iterations(&self) -> usize {+ self.common.iterations+ }+}diff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltindex 5f0a93a5fa..c6d5df0d0d 100644--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -1644,3 +1644,460 @@ RESET datafusion.execution.parquet.enable_page_index;statement okRESET datafusion.optimizer.enable_sort_pushdown;+++###############################################################+# Statistics-based file sorting and sort elimination tests+###############################################################++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;++statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+SET datafusion.execution.collect_statistics = true;++# Test A: Non-overlapping files with matching within-file ordering → Sort eliminated++statement ok+CREATE TABLE ta_src_a(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++statement ok+CREATE TABLE ta_src_b(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE ta_src_c(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900), (10, 1000);++query I+COPY (SELECT * FROM ta_src_a ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/ta_nonoverlap/file_a.parquet';+----+3++query I+COPY (SELECT * FROM ta_src_b ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/ta_nonoverlap/file_b.parquet';+----+3++query I+COPY (SELECT * FROM ta_src_c ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/ta_nonoverlap/file_c.parquet';+----+4++statement ok+CREATE EXTERNAL TABLE ta_sorted(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/ta_nonoverlap/'+WITH ORDER (id ASC);++# Test A.1: SortExec eliminated — Exact pushdown+query TT+EXPLAIN SELECT * FROM ta_sorted ORDER BY id ASC;+----+logical_plan+01)Sort: ta_sorted.id ASC NULLS LAST+02)--TableScan: ta_sorted projection=[id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/ta_nonoverlap/file_a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/ta_nonoverlap/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/ta_nonoverlap/file_c.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++query II+SELECT * FROM ta_sorted ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900+10 1000++# Cleanup Test A+statement ok+DROP TABLE ta_src_a;++statement ok+DROP TABLE ta_src_b;++statement ok+DROP TABLE ta_src_c;++statement ok+DROP TABLE ta_sorted;+++# Test B: Overlapping files → statistics-based reorder, SortExec retained++statement ok+CREATE TABLE tb_src_x(id INT, value INT) AS VALUES (1, 10), (2, 20), (3, 30), (4, 40), (5, 50);++statement ok+CREATE TABLE tb_src_y(id INT, value INT) AS VALUES (3, 31), (4, 41), (5, 51), (6, 61), (7, 71), (8, 81);++statement ok+CREATE TABLE tb_src_z(id INT, value INT) AS VALUES (6, 62), (7, 72), (8, 82), (9, 92), (10, 102);++query I+COPY (SELECT * FROM tb_src_x ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet';+----+5++query I+COPY (SELECT * FROM tb_src_y ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet';+----+6++query I+COPY (SELECT * FROM tb_src_z ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet';+----+5++statement ok+CREATE EXTERNAL TABLE tb_overlap(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tb_overlap/'+WITH ORDER (id ASC);++# Test B.1: Multi-column DESC sort — statistics fallback sorts files [z, y, x]+query TT+EXPLAIN SELECT * FROM tb_overlap ORDER BY id DESC, value DESC LIMIT 5;+----+logical_plan+01)Sort: tb_overlap.id DESC NULLS FIRST, tb_overlap.value DESC NULLS FIRST, fetch=5+02)--TableScan: tb_overlap projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=5), expr=[id@0 DESC, value@1 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_y.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tb_overlap/file_z.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ]++query II+SELECT * FROM tb_overlap ORDER BY id DESC, value DESC LIMIT 5;+----+10 102+9 92+8 82+8 81+7 72++# Cleanup Test B+statement ok+DROP TABLE tb_src_x;++statement ok+DROP TABLE tb_src_y;++statement ok+DROP TABLE tb_src_z;++statement ok+DROP TABLE tb_overlap;+++# Test C: Non-overlapping files with LIMIT — sort elimination + limit pushdown++statement ok+CREATE TABLE tc_src_a(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++statement ok+CREATE TABLE tc_src_b(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE tc_src_c(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900), (10, 1000);++query I+COPY (SELECT * FROM tc_src_a ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tc_limit/file_a.parquet';+----+3++query I+COPY (SELECT * FROM tc_src_b ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tc_limit/file_b.parquet';+----+3++query I+COPY (SELECT * FROM tc_src_c ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/tc_limit/file_c.parquet';+----+4++statement ok+CREATE EXTERNAL TABLE tc_limit(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/tc_limit/'+WITH ORDER (id ASC);++# Test C.1: ASC LIMIT — sort eliminated, limit pushed down+query TT+EXPLAIN SELECT * FROM tc_limit ORDER BY id ASC LIMIT 3;+----+logical_plan+01)Sort: tc_limit.id ASC NULLS LAST, fetch=3+02)--TableScan: tc_limit projection=[id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet]]}, projection=[id, value], limit=3, output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++query II+SELECT * FROM tc_limit ORDER BY id ASC LIMIT 3;+----+1 100+2 200+3 300++# Test C.2: DESC LIMIT — reverse scan path, SortExec stays+query TT+EXPLAIN SELECT * FROM tc_limit ORDER BY id DESC LIMIT 3;+----+logical_plan+01)Sort: tc_limit.id DESC NULLS FIRST, fetch=3+02)--TableScan: tc_limit projection=[id, value]+physical_plan+01)SortExec: TopK(fetch=3), expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_c.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/tc_limit/file_a.parquet]]}, projection=[id, value], file_type=parquet, predicate=DynamicFilter [ empty ], reverse_row_groups=true++query II+SELECT * FROM tc_limit ORDER BY id DESC LIMIT 3;+----+10 1000+9 900+8 800++# Cleanup Test C+statement ok+DROP TABLE tc_src_a;++statement ok+DROP TABLE tc_src_b;++statement ok+DROP TABLE tc_src_c;++statement ok+DROP TABLE tc_limit;+++# Test D: Multi-group case with target_partitions=2++statement ok+CREATE TABLE td_src_a(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++statement ok+CREATE TABLE td_src_b(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE td_src_c(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900), (10, 1000);++query I+COPY (SELECT * FROM td_src_a ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/td_multi/file_a.parquet';+----+3++query I+COPY (SELECT * FROM td_src_b ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/td_multi/file_b.parquet';+----+3++query I+COPY (SELECT * FROM td_src_c ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/td_multi/file_c.parquet';+----+4++statement ok+SET datafusion.execution.target_partitions = 2;++statement ok+CREATE EXTERNAL TABLE td_multi(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/td_multi/'+WITH ORDER (id ASC);++# Test D.1: With 2 partitions, files split across 2 groups.+# Each group's sort is eliminated; SortPreservingMergeExec merges groups.+query TT+EXPLAIN SELECT * FROM td_multi ORDER BY id ASC;+----+logical_plan+01)Sort: td_multi.id ASC NULLS LAST+02)--TableScan: td_multi projection=[id, value]+physical_plan+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/td_multi/file_a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/td_multi/file_b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/td_multi/file_c.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++query II+SELECT * FROM td_multi ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900+10 1000++# Cleanup Test D+statement ok+DROP TABLE td_src_a;++statement ok+DROP TABLE td_src_b;++statement ok+DROP TABLE td_src_c;++statement ok+DROP TABLE td_multi;++# Restore target_partitions=1 for remaining tests+statement ok+SET datafusion.execution.target_partitions = 1;++# ===========================================================+# Test E: Inferred ordering from Parquet metadata (no WITH ORDER)+# Parquet files written with ORDER BY have sorting_columns in metadata.+# DataFusion should automatically infer the ordering and eliminate Sort.+# ===========================================================++# Create sorted parquet files — COPY with ORDER BY writes sorting_columns metadata+statement ok+CREATE TABLE te_src_a(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++statement ok+CREATE TABLE te_src_b(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE te_src_c(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900), (10, 1000);++query I+COPY (SELECT * FROM te_src_a ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/te_inferred/file_a.parquet';+----+3++query I+COPY (SELECT * FROM te_src_b ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/te_inferred/file_b.parquet';+----+3++query I+COPY (SELECT * FROM te_src_c ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/te_inferred/file_c.parquet';+----+4++# Create external table WITHOUT "WITH ORDER" — ordering should be inferred+# from Parquet sorting_columns metadata+statement ok+CREATE EXTERNAL TABLE te_inferred(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/te_inferred/';++# Test E.1: Ordering should be inferred and SortExec eliminated+# Even without WITH ORDER, the optimizer detects that files are sorted+# (from Parquet sorting_columns metadata) and non-overlapping.+# SortExec is completely eliminated — just DataSourceExec with output_ordering.+query TT+EXPLAIN SELECT * FROM te_inferred ORDER BY id ASC;+----+logical_plan+01)Sort: te_inferred.id ASC NULLS LAST+02)--TableScan: te_inferred projection=[id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_b.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_c.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test E.2: Verify result correctness+query II+SELECT * FROM te_inferred ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900+10 1000++# Test E.3: LIMIT should also work with inferred ordering+query II+SELECT * FROM te_inferred ORDER BY id ASC LIMIT 3;+----+1 100+2 200+3 300++# Test E.4: Inferred ordering with multiple partitions+# With target_partitions=2, files split into 2 groups.+# Each group has SortExec eliminated, SPM merges the sorted streams.+statement ok+DROP TABLE te_inferred;++statement ok+SET datafusion.execution.target_partitions = 2;++statement ok+CREATE EXTERNAL TABLE te_inferred_multi(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/te_inferred/';++query TT+EXPLAIN SELECT * FROM te_inferred_multi ORDER BY id ASC;+----+logical_plan+01)Sort: te_inferred_multi.id ASC NULLS LAST+02)--TableScan: te_inferred_multi projection=[id, value]+physical_plan+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_a.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_b.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/te_inferred/file_c.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Verify correctness with multi partition+query II+SELECT * FROM te_inferred_multi ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900+10 1000++# Cleanup Test E+statement ok+DROP TABLE te_src_a;++statement ok+DROP TABLE te_src_b;++statement ok+DROP TABLE te_src_c;++statement ok+DROP TABLE te_inferred_multi;++# Reset settings (SLT runner uses target_partitions=4, not system default)+statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+SET datafusion.execution.collect_statistics = true;++statement ok+SET datafusion.optimizer.enable_sort_pushdown = true;