Apache DataFusion SQL Query Engine
Removed incorrect union check in enforce_sorting and updated tests (#18661)
Gene Bordegaray1833093ceb91
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex e2f3ece1e4..45641ed0fa 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -3135,19 +3135,18 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_reparti) -> Result<()> {assert_snapshot!(union_with_mix_of_presorted_and_explicitly_resorted_inputs_impl(false).await?,- @r#"+ @r"AggregateExec: mode=Final, gby=[id@0 as id], aggr=[], ordering_mode=Sorted- SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]- CoalescePartitionsExec- AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[]- UnionExec- DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+ SortPreservingMergeExec: [id@0 ASC NULLS LAST]+ AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[], ordering_mode=Sorted+ UnionExec+ DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet+ SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false]DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet- "#);+ ");Ok(())}-#[ignore] // See https://github.com/apache/datafusion/issues/18380#[tokio::test]// Test with `repartition_sorts` enabled to preserve pre-sorted partitions and avoid resortingasync fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartition_sorts_true(@@ -3164,52 +3163,6 @@ async fn union_with_mix_of_presorted_and_explicitly_resorted_inputs_with_repartiDataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet"#);- // 💥 Doesn't pass, and generates this plan:- //- // AggregateExec: mode=Final, gby=[id@0 as id], aggr=[], ordering_mode=Sorted- // SortPreservingMergeExec: [id@0 ASC NULLS LAST]- // SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]- // AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[]- // UnionExec- // DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet- // DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet- //- //- // === Excerpt from the verbose explain ===- //- // +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+- // | plan_type | plan |- // +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+- // | initial_physical_plan | AggregateExec: mode=Final, gby=[id@0 as id], aggr=[], ordering_mode=Sorted |- // | | AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[], ordering_mode=Sorted |- // | | UnionExec |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet |- // | | SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet |- // ...- // | physical_plan after EnforceDistribution | OutputRequirementExec: order_by=[], dist_by=Unspecified |- // | | AggregateExec: mode=Final, gby=[id@0 as id], aggr=[], ordering_mode=Sorted |- // | | SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] |- // | | CoalescePartitionsExec |- // | | AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[], ordering_mode=Sorted |- // | | UnionExec |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet |- // | | SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[false] |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet |- // | | |- // | physical_plan after CombinePartialFinalAggregate | SAME TEXT AS ABOVE- // | | |- // | physical_plan after EnforceSorting | OutputRequirementExec: order_by=[], dist_by=Unspecified |- // | | AggregateExec: mode=Final, gby=[id@0 as id], aggr=[], ordering_mode=Sorted |- // | | SortPreservingMergeExec: [id@0 ASC NULLS LAST] |- // | | SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true] |- // | | AggregateExec: mode=Partial, gby=[id@0 as id], aggr=[] |- // | | UnionExec |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], output_ordering=[id@0 ASC NULLS LAST], file_type=parquet |- // | | DataSourceExec: file_groups={1 group: [[{testdata}/alltypes_tiny_pages.parquet]]}, projection=[id], file_type=parquet |- // ...- // +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-Ok(())}diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rsindex c0cfa46733..caeaeb3d69 100644--- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs@@ -664,21 +664,13 @@ async fn test_union_inputs_different_sorted7() -> Result<()> {// Union has unnecessarily fine ordering below it. We should be able to replace them with absolutely necessary ordering.let test = EnforceSortingTest::new(physical_plan).with_repartition_sorts(true);assert_snapshot!(test.run(), @r"- Input Plan:+ Input / Optimized Plan:SortPreservingMergeExec: [nullable_col@0 ASC]UnionExecSortExec: expr=[nullable_col@0 ASC, non_nullable_col@1 ASC], preserve_partitioning=[false]DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquetSortExec: expr=[nullable_col@0 ASC, non_nullable_col@1 ASC], preserve_partitioning=[false]DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet-- Optimized Plan:- SortPreservingMergeExec: [nullable_col@0 ASC]- UnionExec- SortExec: expr=[nullable_col@0 ASC], preserve_partitioning=[false]- DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet- SortExec: expr=[nullable_col@0 ASC], preserve_partitioning=[false]- DataSourceExec: file_groups={1 group: [[x]]}, projection=[nullable_col, non_nullable_col], file_type=parquet");// Union preserves the inputs ordering, and we should not change any of the SortExecs under UnionExecdiff --git a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/enforce_sorting/mod.rsindex 28d187bbf8..69593697e2 100644--- a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs+++ b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs@@ -49,7 +49,7 @@ use crate::enforce_sorting::sort_pushdown::{use crate::output_requirements::OutputRequirementExec;use crate::utils::{add_sort_above, add_sort_above_with_check, is_coalesce_partitions, is_limit,- is_repartition, is_sort, is_sort_preserving_merge, is_union, is_window,+ is_repartition, is_sort, is_sort_preserving_merge, is_window,};use crate::PhysicalOptimizerRule;@@ -516,10 +516,7 @@ pub fn ensure_sorting();child = update_sort_ctx_children_data(child, true)?;}- } else if physical_ordering.is_none()- || !plan.maintains_input_order()[idx]- || is_union(plan)- {+ } else if physical_ordering.is_none() || !plan.maintains_input_order()[idx] {// We have a `SortExec` whose effect may be neutralized by another// order-imposing operator, remove this sort:child = update_child_to_remove_unnecessary_sort(idx, child, plan)?;