Apache DataFusion SQL Query Engine
fix: validate inter-file ordering in eq_properties() (#20329)
Adrian Garcia Badaracco53b0ffb93d5f
diff --git a/datafusion/core/tests/physical_optimizer/partition_statistics.rs b/datafusion/core/tests/physical_optimizer/partition_statistics.rsindex 5d168c0641..fa021ed3dc 100644--- a/datafusion/core/tests/physical_optimizer/partition_statistics.rs+++ b/datafusion/core/tests/physical_optimizer/partition_statistics.rs@@ -826,7 +826,7 @@ mod test {let plan_string = get_plan_string(&aggregate_exec_partial).swap_remove(0);assert_snapshot!(plan_string,- @"AggregateExec: mode=Partial, gby=[id@0 as id, 1 + id@0 as expr], aggr=[COUNT(c)], ordering_mode=Sorted"+ @"AggregateExec: mode=Partial, gby=[id@0 as id, 1 + id@0 as expr], aggr=[COUNT(c)]");let p0_statistics = aggregate_exec_partial.partition_statistics(Some(0))?;diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex 8c64ce7842..c3e5cabce7 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -736,7 +736,7 @@ impl DataSource for FileScanConfig {let schema = self.file_source.table_schema().table_schema();let mut eq_properties = EquivalenceProperties::new_with_orderings(Arc::clone(schema),- self.output_ordering.clone(),+ self.validated_output_ordering(),).with_constraints(self.constraints.clone());@@ -926,6 +926,40 @@ impl DataSource for FileScanConfig {}impl FileScanConfig {+ /// Returns only the output orderings that are validated against actual+ /// file group statistics.+ ///+ /// For example, individual files may be ordered by `col1 ASC`,+ /// but if we have files with these min/max statistics in a single partition / file group:+ ///+ /// - file1: min(col1) = 10, max(col1) = 20+ /// - file2: min(col1) = 5, max(col1) = 15+ ///+ /// Because reading file1 followed by file2 would produce out-of-order output (there is overlap+ /// in the ranges), we cannot retain `col1 ASC` as a valid output ordering.+ ///+ /// Similarly this would not be a valid order (non-overlapping ranges but not ordered):+ ///+ /// - file1: min(col1) = 20, max(col1) = 30+ /// - file2: min(col1) = 10, max(col1) = 15+ ///+ /// On the other hand if we had:+ ///+ /// - file1: min(col1) = 5, max(col1) = 15+ /// - file2: min(col1) = 16, max(col1) = 25+ ///+ /// Then we know that reading file1 followed by file2 will produce ordered output,+ /// so `col1 ASC` would be retained.+ ///+ /// Note that we are checking for ordering *within* *each* file group / partition,+ /// files in different partitions are read independently and do not affect each other's ordering.+ /// Merging of the multiple partition streams into a single ordered stream is handled+ /// upstream e.g. by `SortPreservingMergeExec`.+ fn validated_output_ordering(&self) -> Vec<LexOrdering> {+ let schema = self.file_source.table_schema().table_schema();+ validate_orderings(&self.output_ordering, schema, &self.file_groups, None)+ }+/// Get the file schema (schema of the files without partition columns)pub fn file_schema(&self) -> &SchemaRef {self.file_source.table_schema().file_schema()@@ -1300,6 +1334,51 @@ fn ordered_column_indices_from_projection(.collect::<Option<Vec<usize>>>()}+/// Check whether a given ordering is valid for all file groups by verifying+/// that files within each group are sorted according to their min/max statistics.+///+/// For single-file (or empty) groups, the ordering is trivially valid.+/// For multi-file groups, we check that the min/max statistics for the sort+/// columns are in order and non-overlapping (or touching at boundaries).+///+/// `projection` maps projected column indices back to table-schema indices+/// when validating after projection; pass `None` when validating at+/// table-schema level.+fn is_ordering_valid_for_file_groups(+ file_groups: &[FileGroup],+ ordering: &LexOrdering,+ schema: &SchemaRef,+ projection: Option<&[usize]>,+) -> bool {+ file_groups.iter().all(|group| {+ if group.len() <= 1 {+ return true; // single-file groups are trivially sorted+ }+ match MinMaxStatistics::new_from_files(ordering, schema, projection, group.iter())+ {+ Ok(stats) => stats.is_sorted(),+ Err(_) => false, // can't prove sorted → reject+ }+ })+}++/// Filters orderings to retain only those valid for all file groups,+/// verified via min/max statistics.+fn validate_orderings(+ orderings: &[LexOrdering],+ schema: &SchemaRef,+ file_groups: &[FileGroup],+ projection: Option<&[usize]>,+) -> Vec<LexOrdering> {+ orderings+ .iter()+ .filter(|ordering| {+ is_ordering_valid_for_file_groups(file_groups, ordering, schema, projection)+ })+ .cloned()+ .collect()+}+/// The various listing tables does not attempt to read all files/// concurrently, instead they will read files in sequence within a/// partition. This is an important property as it allows plans to@@ -1366,52 +1445,47 @@ fn get_projected_output_ordering(let projected_orderings =project_orderings(&base_config.output_ordering, projected_schema);- let mut all_orderings = vec![];- for new_ordering in projected_orderings {- // Check if any file groups are not sorted- if base_config.file_groups.iter().any(|group| {- if group.len() <= 1 {- // File groups with <= 1 files are always sorted- return false;- }-- let Some(indices) = base_config- .file_source- .projection()- .as_ref()- .map(|p| ordered_column_indices_from_projection(p))- else {- // Can't determine if ordered without a simple projection- return true;- };-- let statistics = match MinMaxStatistics::new_from_files(- &new_ordering,+ let indices = base_config+ .file_source+ .projection()+ .as_ref()+ .map(|p| ordered_column_indices_from_projection(p));++ match indices {+ Some(Some(indices)) => {+ // Simple column projection — validate with statistics+ validate_orderings(+ &projected_orderings,projected_schema,- indices.as_deref(),- group.iter(),- ) {- Ok(statistics) => statistics,- Err(e) => {- log::trace!("Error fetching statistics for file group: {e}");- // we can't prove that it's ordered, so we have to reject it- return true;- }- };-- !statistics.is_sorted()- }) {- debug!(- "Skipping specified output ordering {:?}. \- Some file groups couldn't be determined to be sorted: {:?}",- base_config.output_ordering[0], base_config.file_groups- );- continue;+ &base_config.file_groups,+ Some(indices.as_slice()),+ )+ }+ None => {+ // No projection — validate with statistics (no remapping needed)+ validate_orderings(+ &projected_orderings,+ projected_schema,+ &base_config.file_groups,+ None,+ )+ }+ Some(None) => {+ // Complex projection (expressions, not simple columns) — can't+ // determine column indices for statistics. Still valid if all+ // file groups have at most one file.+ if base_config.file_groups.iter().all(|g| g.len() <= 1) {+ projected_orderings+ } else {+ debug!(+ "Skipping specified output orderings. \+ Some file groups couldn't be determined to be sorted: {:?}",+ base_config.file_groups+ );+ vec![]+ }}-- all_orderings.push(new_ordering);}- all_orderings}/// Convert type to a type suitable for use as a `ListingTable`diff --git a/datafusion/datasource/src/statistics.rs b/datafusion/datasource/src/statistics.rsindex 2f34ca032e..b1a56e096c 100644--- a/datafusion/datasource/src/statistics.rs+++ b/datafusion/datasource/src/statistics.rs@@ -266,11 +266,12 @@ impl MinMaxStatistics {}/// Check if the min/max statistics are in order and non-overlapping+ /// (or touching at boundaries)pub fn is_sorted(&self) -> bool {self.max_by_sort_order.iter().zip(self.min_by_sort_order.iter().skip(1))- .all(|(max, next_min)| max < next_min)+ .all(|(max, next_min)| max <= next_min)}}diff --git a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.sltindex 5a559bdb94..fd3a40ca17 100644--- a/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt+++ b/datafusion/sqllogictest/test_files/parquet_sorted_statistics.slt@@ -274,4 +274,4 @@ logical_plan02)--TableScan: test_table projection=[constant_col]physical_plan01)SortPreservingMergeExec: [constant_col@0 ASC NULLS LAST]-02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet]]}, projection=[constant_col], file_type=parquet+02)--DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=A/0.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=B/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_sorted_statistics/test_table/partition_col=C/2.parquet]]}, projection=[constant_col], output_ordering=[constant_col@0 ASC NULLS LAST], file_type=parquetdiff --git a/datafusion/sqllogictest/test_files/sort_pushdown.slt b/datafusion/sqllogictest/test_files/sort_pushdown.sltindex 58d9915a24..7b741579cb 100644--- a/datafusion/sqllogictest/test_files/sort_pushdown.slt+++ b/datafusion/sqllogictest/test_files/sort_pushdown.slt@@ -851,7 +851,545 @@ LIMIT 3;5 42 -3+# Test 4: Reversed filesystem order with inferred ordering+# Create 3 parquet files with non-overlapping id ranges, named so filesystem+# order is OPPOSITE to data order. Each file is internally sorted by id ASC.+# Force target_partitions=1 so all files end up in one file group, which is+# where the inter-file ordering bug manifests.+# Without inter-file validation, the optimizer would incorrectly trust the+# inferred ordering and remove SortExec.++# Save current target_partitions and set to 1 to force single file group+statement ok+SET datafusion.execution.target_partitions = 1;++statement ok+CREATE TABLE reversed_high(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900);++statement ok+CREATE TABLE reversed_mid(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE reversed_low(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++query I+COPY (SELECT * FROM reversed_high ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/reversed/a_high.parquet';+----+3++query I+COPY (SELECT * FROM reversed_mid ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/reversed/b_mid.parquet';+----+3++query I+COPY (SELECT * FROM reversed_low ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/reversed/c_low.parquet';+----+3++# External table with NO "WITH ORDER" — relies on inferred ordering from parquet metadata+statement ok+CREATE EXTERNAL TABLE reversed_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/reversed/';++# Test 4.1: SortExec must be present because files are not in inter-file order+query TT+EXPLAIN SELECT * FROM reversed_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: reversed_parquet.id ASC NULLS LAST+02)--TableScan: reversed_parquet projection=[id, value]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet]]}, projection=[id, value], file_type=parquet++# Test 4.2: Results must be correct+query II+SELECT * FROM reversed_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900++# Test 5: Overlapping files with inferred ordering+# Create files with overlapping id ranges++statement ok+CREATE TABLE overlap_x(id INT, value INT) AS VALUES (1, 100), (3, 300), (5, 500);++statement ok+CREATE TABLE overlap_y(id INT, value INT) AS VALUES (2, 200), (4, 400), (6, 600);++query I+COPY (SELECT * FROM overlap_x ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/overlap/file_x.parquet';+----+3++query I+COPY (SELECT * FROM overlap_y ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/overlap/file_y.parquet';+----+3++statement ok+CREATE EXTERNAL TABLE overlap_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/overlap/';++# Test 5.1: SortExec must be present because files have overlapping ranges+query TT+EXPLAIN SELECT * FROM overlap_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: overlap_parquet.id ASC NULLS LAST+02)--TableScan: overlap_parquet projection=[id, value]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_x.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/overlap/file_y.parquet]]}, projection=[id, value], file_type=parquet++# Test 5.2: Results must be correct+query II+SELECT * FROM overlap_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600++# Test 6: WITH ORDER + reversed filesystem order+# Same file setup as Test 4 but explicitly declaring ordering via WITH ORDER.+# Even with WITH ORDER, the optimizer should detect that inter-file order is wrong+# and keep SortExec.++statement ok+CREATE EXTERNAL TABLE reversed_with_order_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/reversed/'+WITH ORDER (id ASC);++# Test 6.1: SortExec must be present despite WITH ORDER+query TT+EXPLAIN SELECT * FROM reversed_with_order_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: reversed_with_order_parquet.id ASC NULLS LAST+02)--TableScan: reversed_with_order_parquet projection=[id, value]+physical_plan+01)SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet]]}, projection=[id, value], file_type=parquet++# Test 6.2: Results must be correct+query II+SELECT * FROM reversed_with_order_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900++# Test 7: Correctly ordered multi-file single group (positive case)+# Files are in CORRECT inter-file order within a single group.+# The validation should PASS and SortExec should be eliminated.++statement ok+CREATE TABLE correct_low(id INT, value INT) AS VALUES (1, 100), (2, 200), (3, 300);++statement ok+CREATE TABLE correct_mid(id INT, value INT) AS VALUES (4, 400), (5, 500), (6, 600);++statement ok+CREATE TABLE correct_high(id INT, value INT) AS VALUES (7, 700), (8, 800), (9, 900);++query I+COPY (SELECT * FROM correct_low ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/correct/a_low.parquet';+----+3++query I+COPY (SELECT * FROM correct_mid ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/correct/b_mid.parquet';+----+3++query I+COPY (SELECT * FROM correct_high ORDER BY id ASC)+TO 'test_files/scratch/sort_pushdown/correct/c_high.parquet';+----+3++statement ok+CREATE EXTERNAL TABLE correct_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/correct/'+WITH ORDER (id ASC);++# Test 7.1: SortExec should be ELIMINATED — files are in correct inter-file order+query TT+EXPLAIN SELECT * FROM correct_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: correct_parquet.id ASC NULLS LAST+02)--TableScan: correct_parquet projection=[id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test 7.2: Results must be correct+query II+SELECT * FROM correct_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900++# Test 7.3: DESC query on correctly ordered ASC files should still use SortExec+# Note: reverse_row_groups=true reverses the file list in the plan display+query TT+EXPLAIN SELECT * FROM correct_parquet ORDER BY id DESC;+----+logical_plan+01)Sort: correct_parquet.id DESC NULLS FIRST+02)--TableScan: correct_parquet projection=[id, value]+physical_plan+01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet]]}, projection=[id, value], file_type=parquet, reverse_row_groups=true++query II+SELECT * FROM correct_parquet ORDER BY id DESC;+----+9 900+8 800+7 700+6 600+5 500+4 400+3 300+2 200+1 100++# Test 8: DESC ordering with files in wrong inter-file DESC order+# Create files internally sorted by id DESC, but named so filesystem order+# is WRONG for DESC ordering (low values first in filesystem order).++statement ok+CREATE TABLE desc_low(id INT, value INT) AS VALUES (3, 300), (2, 200), (1, 100);++statement ok+CREATE TABLE desc_high(id INT, value INT) AS VALUES (9, 900), (8, 800), (7, 700);++query I+COPY (SELECT * FROM desc_low ORDER BY id DESC)+TO 'test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet';+----+3++query I+COPY (SELECT * FROM desc_high ORDER BY id DESC)+TO 'test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet';+----+3++statement ok+CREATE EXTERNAL TABLE desc_reversed_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/desc_reversed/'+WITH ORDER (id DESC);++# Test 8.1: SortExec must be present — files are in wrong inter-file DESC order+# (a_low has 1-3, b_high has 7-9; for DESC, b_high should come first)+query TT+EXPLAIN SELECT * FROM desc_reversed_parquet ORDER BY id DESC;+----+logical_plan+01)Sort: desc_reversed_parquet.id DESC NULLS FIRST+02)--TableScan: desc_reversed_parquet projection=[id, value]+physical_plan+01)SortExec: expr=[id@0 DESC], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/a_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/desc_reversed/b_high.parquet]]}, projection=[id, value], file_type=parquet++# Test 8.2: Results must be correct+query II+SELECT * FROM desc_reversed_parquet ORDER BY id DESC;+----+9 900+8 800+7 700+3 300+2 200+1 100++# Test 9: Multi-column sort key validation+# Files have (category, id) ordering. Files share a boundary value on category='B'+# so column-level min/max statistics overlap on the primary key column.+# The validation conservatively rejects this because column-level stats can't+# precisely represent row-level boundaries for multi-column keys.++statement ok+CREATE TABLE multi_col_a(category VARCHAR, id INT, value INT) AS VALUES+('A', 1, 10), ('A', 2, 20), ('B', 1, 30);++statement ok+CREATE TABLE multi_col_b(category VARCHAR, id INT, value INT) AS VALUES+('B', 2, 40), ('C', 1, 50), ('C', 2, 60);++query I+COPY (SELECT * FROM multi_col_a ORDER BY category ASC, id ASC)+TO 'test_files/scratch/sort_pushdown/multi_col/a_first.parquet';+----+3++query I+COPY (SELECT * FROM multi_col_b ORDER BY category ASC, id ASC)+TO 'test_files/scratch/sort_pushdown/multi_col/b_second.parquet';+----+3++statement ok+CREATE EXTERNAL TABLE multi_col_parquet(category VARCHAR, id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/multi_col/'+WITH ORDER (category ASC, id ASC);++# Test 9.1: SortExec is present — validation conservatively rejects because+# column-level stats overlap on category='B' across both files+query TT+EXPLAIN SELECT * FROM multi_col_parquet ORDER BY category ASC, id ASC;+----+logical_plan+01)Sort: multi_col_parquet.category ASC NULLS LAST, multi_col_parquet.id ASC NULLS LAST+02)--TableScan: multi_col_parquet projection=[category, id, value]+physical_plan+01)SortExec: expr=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/a_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col/b_second.parquet]]}, projection=[category, id, value], file_type=parquet++# Test 9.2: Results must be correct+query TII+SELECT * FROM multi_col_parquet ORDER BY category ASC, id ASC;+----+A 1 10+A 2 20+B 1 30+B 2 40+C 1 50+C 2 60++# Test 9.3: Multi-column sort with non-overlapping primary key across files+# When files don't overlap on the primary column, validation succeeds.++statement ok+CREATE TABLE multi_col_x(category VARCHAR, id INT, value INT) AS VALUES+('A', 1, 10), ('A', 2, 20);++statement ok+CREATE TABLE multi_col_y(category VARCHAR, id INT, value INT) AS VALUES+('B', 1, 30), ('B', 2, 40);++query I+COPY (SELECT * FROM multi_col_x ORDER BY category ASC, id ASC)+TO 'test_files/scratch/sort_pushdown/multi_col_clean/x_first.parquet';+----+2++query I+COPY (SELECT * FROM multi_col_y ORDER BY category ASC, id ASC)+TO 'test_files/scratch/sort_pushdown/multi_col_clean/y_second.parquet';+----+2++statement ok+CREATE EXTERNAL TABLE multi_col_clean_parquet(category VARCHAR, id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/multi_col_clean/'+WITH ORDER (category ASC, id ASC);++# Test 9.3a: SortExec should be eliminated — non-overlapping primary column+query TT+EXPLAIN SELECT * FROM multi_col_clean_parquet ORDER BY category ASC, id ASC;+----+logical_plan+01)Sort: multi_col_clean_parquet.category ASC NULLS LAST, multi_col_clean_parquet.id ASC NULLS LAST+02)--TableScan: multi_col_clean_parquet projection=[category, id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col_clean/x_first.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/multi_col_clean/y_second.parquet]]}, projection=[category, id, value], output_ordering=[category@0 ASC NULLS LAST, id@1 ASC NULLS LAST], file_type=parquet++# Test 9.3b: Results must be correct+query TII+SELECT * FROM multi_col_clean_parquet ORDER BY category ASC, id ASC;+----+A 1 10+A 2 20+B 1 30+B 2 40++# Test 10: Correctly ordered files WITH ORDER (positive counterpart to Test 6)+# Files in correct_parquet are in correct ASC order — WITH ORDER should pass validation+# and SortExec should be eliminated.++statement ok+CREATE EXTERNAL TABLE correct_with_order_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/correct/'+WITH ORDER (id ASC);++# Test 10.1: SortExec should be ELIMINATED — files are in correct order+query TT+EXPLAIN SELECT * FROM correct_with_order_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: correct_with_order_parquet.id ASC NULLS LAST+02)--TableScan: correct_with_order_parquet projection=[id, value]+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/a_low.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/b_mid.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/correct/c_high.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test 10.2: Results must be correct+query II+SELECT * FROM correct_with_order_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900++# Test 11: Multiple file groups (target_partitions > 1) — each group has one file+# When files are spread across separate partitions (one file per group), each+# partition is trivially sorted and SortPreservingMergeExec handles the merge.++# Restore higher target_partitions so files go into separate groups+statement ok+SET datafusion.execution.target_partitions = 4;++statement ok+CREATE EXTERNAL TABLE multi_partition_parquet(id INT, value INT)+STORED AS PARQUET+LOCATION 'test_files/scratch/sort_pushdown/reversed/'+WITH ORDER (id ASC);++# Test 11.1: With separate partitions, each file is trivially sorted.+# SortPreservingMergeExec merges, no SortExec needed per-partition.+query TT+EXPLAIN SELECT * FROM multi_partition_parquet ORDER BY id ASC;+----+logical_plan+01)Sort: multi_partition_parquet.id ASC NULLS LAST+02)--TableScan: multi_partition_parquet projection=[id, value]+physical_plan+01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]+02)--DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/a_high.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/b_mid.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/sort_pushdown/reversed/c_low.parquet]]}, projection=[id, value], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet++# Test 11.2: Results must be correct+query II+SELECT * FROM multi_partition_parquet ORDER BY id ASC;+----+1 100+2 200+3 300+4 400+5 500+6 600+7 700+8 800+9 900++# Restore target_partitions to 1 for remaining cleanup+statement ok+SET datafusion.execution.target_partitions = 2;+# Cleanup+statement ok+DROP TABLE reversed_high;++statement ok+DROP TABLE reversed_mid;++statement ok+DROP TABLE reversed_low;++statement ok+DROP TABLE reversed_parquet;++statement ok+DROP TABLE overlap_x;++statement ok+DROP TABLE overlap_y;++statement ok+DROP TABLE overlap_parquet;++statement ok+DROP TABLE reversed_with_order_parquet;++statement ok+DROP TABLE correct_low;++statement ok+DROP TABLE correct_mid;++statement ok+DROP TABLE correct_high;++statement ok+DROP TABLE correct_parquet;++statement ok+DROP TABLE desc_low;++statement ok+DROP TABLE desc_high;++statement ok+DROP TABLE desc_reversed_parquet;++statement ok+DROP TABLE multi_col_a;++statement ok+DROP TABLE multi_col_b;++statement ok+DROP TABLE multi_col_parquet;++statement ok+DROP TABLE multi_col_x;++statement ok+DROP TABLE multi_col_y;++statement ok+DROP TABLE multi_col_clean_parquet;++statement ok+DROP TABLE correct_with_order_parquet;++statement ok+DROP TABLE multi_partition_parquet;+statement okDROP TABLE timestamp_data;