Apache DataFusion SQL Query Engine
fix: SortPreservingMergeExec drops input orderings when merging partitions (#26654)
Harshil Mitalebf485818985
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex e865fef98f..febc494bfd 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -172,6 +172,11 @@ impl SortPreservingMergeExec {};let mut eq_properties = input.equivalence_properties().clone();+ // Merging multiple partitions only preserves the merge key ordering.+ // Other input orderings hold only within each partition:+ if input_partitions > 1 {+ eq_properties.clear_orderings();+ }eq_properties.clear_per_partition_constants();eq_properties.add_ordering(ordering);PlanProperties::new(diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.sltindex 3a2c7ccb84..0d48421a83 100644--- a/datafusion/sqllogictest/test_files/order.slt+++ b/datafusion/sqllogictest/test_files/order.slt@@ -2276,3 +2276,60 @@ SELECT i, -i AS n FROM (-128 -128-128 -128NULL NULL++# SortPreservingMergeExec only guarantees its merge key ordering. Each file+# below is sorted by both `a` and `b`, but merging them on `a` does not keep+# them sorted by `b`, so the outer sort must stay.+statement ok+set datafusion.execution.target_partitions = 2;++query I+COPY (VALUES (1, 10), (2, 20))+TO 'test_files/scratch/order/spm_orderings/1.csv'+OPTIONS ('format.has_header' 'true');+----+2++query I+COPY (VALUES (1, 1), (3, 2))+TO 'test_files/scratch/order/spm_orderings/2.csv'+OPTIONS ('format.has_header' 'true');+----+2++statement ok+CREATE EXTERNAL TABLE spm_orderings (+ a BIGINT,+ b BIGINT+)+STORED AS CSV+WITH ORDER (a ASC)+WITH ORDER (b ASC)+LOCATION 'test_files/scratch/order/spm_orderings/'+OPTIONS ('format.has_header' 'true');++query II+SELECT * FROM (SELECT a, b FROM spm_orderings ORDER BY a LIMIT 10) ORDER BY b;+----+1 1+3 2+1 10+2 20++query TT+EXPLAIN SELECT * FROM (SELECT a, b FROM spm_orderings ORDER BY a LIMIT 10) ORDER BY b;+----+logical_plan+01)Sort: spm_orderings.b ASC NULLS LAST+02)--Sort: spm_orderings.a ASC NULLS LAST, fetch=10+03)----TableScan: spm_orderings projection=[a, b]+physical_plan+01)SortExec: expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]+02)--SortPreservingMergeExec: [a@0 ASC NULLS LAST], fetch=10+03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/spm_orderings/1.csv], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/order/spm_orderings/2.csv]]}, projection=[a, b], limit=10, output_orderings=[[a@0 ASC NULLS LAST], [b@1 ASC NULLS LAST]], file_type=csv, has_header=true++statement ok+DROP TABLE spm_orderings;++statement ok+set datafusion.execution.target_partitions = 4;