Apache DataFusion SQL Query Engine
Insta for enforce_distrubution (easy ones) (#18248)
Dmitrii Blaginine432d5561e5e
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 63111f4380..db011c4be4 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -66,9 +66,52 @@ use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::{- get_plan_string, DisplayAs, DisplayFormatType, ExecutionPlanProperties,+ displayable, get_plan_string, DisplayAs, DisplayFormatType, ExecutionPlanProperties,PlanProperties, Statistics,};+use insta::Settings;++/// Helper function to replace only the first occurrence of a regex pattern in a plan+/// Returns (captured_group_1, modified_string)+fn hide_first(+ plan: &dyn ExecutionPlan,+ regex: &str,+ replacement: &str,+) -> (String, String) {+ let plan_str = displayable(plan).indent(true).to_string();+ let pattern = regex::Regex::new(regex).unwrap();++ if let Some(captures) = pattern.captures(&plan_str) {+ let full_match = captures.get(0).unwrap();+ let captured_value = captures+ .get(1)+ .map(|m| m.as_str().to_string())+ .unwrap_or_default();+ let pos = full_match.start();+ let end_pos = full_match.end();+ let mut result = String::with_capacity(plan_str.len());+ result.push_str(&plan_str[..pos]);+ result.push_str(replacement);+ result.push_str(&plan_str[end_pos..]);+ (captured_value, result)+ } else {+ (String::new(), plan_str)+ }+}++macro_rules! assert_plan {+ ($plan: expr, @ $expected:literal) => {+ insta::assert_snapshot!(+ displayable($plan.as_ref()).indent(true).to_string(),+ @ $expected+ )+ };+ ($plan: expr, $another_plan: expr) => {+ let plan1 = displayable($plan.as_ref()).indent(true).to_string();+ let plan2 = displayable($another_plan.as_ref()).indent(true).to_string();+ assert_eq!(plan1, plan2);+ }+}/// Models operators like BoundedWindowExec that require an input/// ordering but is easy to construct@@ -352,22 +395,6 @@ fn ensure_distribution_helper(ensure_distribution(distribution_context, &config).map(|item| item.data.plan)}-/// Test whether plan matches with expected plan-macro_rules! plans_matches_expected {- ($EXPECTED_LINES: expr, $PLAN: expr) => {- let physical_plan = $PLAN;- let actual = get_plan_string(&physical_plan);-- let expected_plan_lines: Vec<&str> = $EXPECTED_LINES- .iter().map(|s| *s).collect();-- assert_eq!(- expected_plan_lines, actual,- "\n**Original Plan Mismatch\n\nexpected:\n\n{expected_plan_lines:#?}\nactual:\n\n{actual:#?}\n\n"- );- }-}-fn test_suite_default_config_options() -> ConfigOptions {let mut config = ConfigOptions::new();@@ -442,6 +469,7 @@ impl TestConfig {self}+ // This be deleted in https://github.com/apache/datafusion/pull/18185/// Perform a series of runs using the current [`TestConfig`],/// assert the expected plan result,/// and return the result plan (for potential subsequent runs).@@ -517,20 +545,79 @@ impl TestConfig {Ok(optimized)}-}-macro_rules! assert_plan_txt {- ($EXPECTED_LINES: expr, $PLAN: expr) => {- let expected_lines: Vec<&str> = $EXPECTED_LINES.iter().map(|s| *s).collect();- // Now format correctly- let actual_lines = get_plan_string(&$PLAN);+ /// Perform a series of runs using the current [`TestConfig`],+ /// assert the expected plan result,+ /// and return the result plan (for potential subsequent runs).+ fn try_to_plan(+ &self,+ plan: Arc<dyn ExecutionPlan>,+ optimizers_to_run: &[Run],+ ) -> Result<Arc<dyn ExecutionPlan>> {+ // Add the ancillary output requirements operator at the start:+ let optimizer = OutputRequirements::new_add_mode();+ let mut optimized = optimizer.optimize(plan.clone(), &self.config)?;- assert_eq!(- &expected_lines, &actual_lines,- "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",- expected_lines, actual_lines- );- };+ // This file has 2 rules that use tree node, apply these rules to original plan consecutively+ // After these operations tree nodes should be in a consistent state.+ // This code block makes sure that these rules doesn't violate tree node integrity.+ {+ let adjusted = if self.config.optimizer.top_down_join_key_reordering {+ // Run adjust_input_keys_ordering rule+ let plan_requirements =+ PlanWithKeyRequirements::new_default(plan.clone());+ let adjusted = plan_requirements+ .transform_down(adjust_input_keys_ordering)+ .data()+ .and_then(check_integrity)?;+ // TODO: End state payloads will be checked here.+ adjusted.plan+ } else {+ // Run reorder_join_keys_to_inputs rule+ plan.clone()+ .transform_up(|plan| {+ Ok(Transformed::yes(reorder_join_keys_to_inputs(plan)?))+ })+ .data()?+ };++ // Then run ensure_distribution rule+ DistributionContext::new_default(adjusted)+ .transform_up(|distribution_context| {+ ensure_distribution(distribution_context, &self.config)+ })+ .data()+ .and_then(check_integrity)?;+ // TODO: End state payloads will be checked here.+ }++ for run in optimizers_to_run {+ optimized = match run {+ Run::Distribution => {+ let optimizer = EnforceDistribution::new();+ optimizer.optimize(optimized, &self.config)?+ }+ Run::Sorting => {+ let optimizer = EnforceSorting::new();+ optimizer.optimize(optimized, &self.config)?+ }+ };+ }++ // Remove the ancillary output requirements operator when done:+ let optimizer = OutputRequirements::new_remove_mode();+ let optimized = optimizer.optimize(optimized, &self.config)?;++ Ok(optimized)+ }++ fn to_plan(+ &self,+ plan: Arc<dyn ExecutionPlan>,+ optimizers_to_run: &[Run],+ ) -> Arc<dyn ExecutionPlan> {+ self.try_to_plan(plan, optimizers_to_run).unwrap()+ }}#[test]@@ -556,6 +643,8 @@ fn multi_hash_joins() -> Result<()> {JoinType::RightAnti,];+ let settings = Settings::clone_current();+// Join on (a == b1)let join_on = vec![(Arc::new(Column::new_with_schema("a", &schema()).unwrap()) as _,@@ -564,11 +653,17 @@ fn multi_hash_joins() -> Result<()> {for join_type in join_types {let join = hash_join_exec(left.clone(), right.clone(), &join_on, &join_type);- let join_plan = |shift| -> String {- format!("{}HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(a@0, b1@1)]", " ".repeat(shift))- };- let join_plan_indent2 = join_plan(2);- let join_plan_indent4 = join_plan(4);++ let mut settings = settings.clone();+ settings.add_filter(+ // join_type={} replace with join_type=... to avoid snapshot name issue+ format!("join_type={join_type}").as_str(),+ "join_type=...",+ );++ insta::allow_duplicates! {+ settings.bind( || {+match join_type {JoinType::Inner@@ -589,50 +684,58 @@ fn multi_hash_joins() -> Result<()> {&top_join_on,&join_type,);- let top_join_plan =- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(a@0, c@2)]");- let expected = match join_type {+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);++ match join_type {// Should include 3 RepartitionExecs- JoinType::Inner | JoinType::Left | JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => vec![- top_join_plan.as_str(),- &join_plan_indent2,- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ],+ JoinType::Inner | JoinType::Left | JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {++ assert_plan!(plan_distrib, @r"+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, c@2)]+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ },// Should include 4 RepartitionExecs- _ => vec![- top_join_plan.as_str(),- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- &join_plan_indent4,- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ],+ _ => {+ assert_plan!(plan_distrib, @r"+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, c@2)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ },};- let test_config = TestConfig::default();- test_config.run(&expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected, top_join, &SORT_DISTRIB_DISTRIB)?;++ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);}JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {}}++match join_type {JoinType::Inner| JoinType::Left@@ -650,55 +753,64 @@ fn multi_hash_joins() -> Result<()> {let top_join =hash_join_exec(join, parquet_exec(), &top_join_on, &join_type);- let top_join_plan = match join_type {- JoinType::RightSemi | JoinType::RightAnti =>- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(b1@1, c@2)]"),- _ =>- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(b1@6, c@2)]"),- };- let expected = match join_type {+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);++ match join_type {// Should include 3 RepartitionExecs- JoinType::Inner | JoinType::Right | JoinType::RightSemi | JoinType::RightAnti =>- vec![- top_join_plan.as_str(),- &join_plan_indent2,- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ],+ JoinType::Inner | JoinType::Right => {+ assert_plan!(parquet_exec(), @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet");+ },+ // Should include 3 RepartitionExecs but have a different "on"+ JoinType::RightSemi | JoinType::RightAnti => {+ assert_plan!(plan_distrib, @r"+ HashJoinExec: mode=Partitioned, join_type=..., on=[(b1@1, c@2)]+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");++ }+// Should include 4 RepartitionExecs- _ =>- vec![- top_join_plan.as_str(),- " RepartitionExec: partitioning=Hash([b1@6], 10), input_partitions=10",- &join_plan_indent4,- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ],+ _ => {+ assert_plan!(plan_distrib, @r"+ HashJoinExec: mode=Partitioned, join_type=..., on=[(b1@6, c@2)]+ RepartitionExec: partitioning=Hash([b1@6], 10), input_partitions=10+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");++ },};- let test_config = TestConfig::default();- test_config.run(&expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected, top_join, &SORT_DISTRIB_DISTRIB)?;++ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);}JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {}}++ });+ }}Ok(())@@ -737,23 +849,27 @@ fn multi_joins_after_alias() -> Result<()> {);// Output partition need to respect the Alias and should not introduce additional RepartitionExec- let expected = &[- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, c@2)]",- " ProjectionExec: expr=[a@0 as a1, a@0 as a2]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, c@2)]+ ProjectionExec: expr=[a@0 as a1, a@0 as a2]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);// Join on (a2 == c)let top_join_on = vec![(@@ -764,23 +880,27 @@ fn multi_joins_after_alias() -> Result<()> {let top_join = hash_join_exec(projection, right, &top_join_on, &JoinType::Inner);// Output partition need to respect the Alias and should not introduce additional RepartitionExec- let expected = &[- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a2@1, c@2)]",- " ProjectionExec: expr=[a@0 as a1, a@0 as a2]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a2@1, c@2)]+ ProjectionExec: expr=[a@0 as a1, a@0 as a2]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -816,26 +936,29 @@ fn multi_joins_after_multi_alias() -> Result<()> {// The Column 'a' has different meaning now after the two Projections// The original Output partition can not satisfy the Join requirements and need to add an additional RepartitionExec- let expected = &[- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, c@2)]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " ProjectionExec: expr=[c1@0 as a]",- " ProjectionExec: expr=[c@2 as c1]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, c@2)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ ProjectionExec: expr=[c1@0 as a]+ ProjectionExec: expr=[c@2 as c1]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -861,22 +984,26 @@ fn join_after_agg_alias() -> Result<()> {let join = hash_join_exec(left, right.clone(), &join_on, &JoinType::Inner);// Only two RepartitionExecs added- let expected = &[- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, a2@0)]",- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",- " RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a2], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, a2@0)]+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]+ RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a2], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -914,23 +1041,27 @@ fn hash_join_key_ordering() -> Result<()> {let join = hash_join_exec(left, right.clone(), &join_on, &JoinType::Inner);// Only two RepartitionExecs added- let expected = &[- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b1@1, b@0), (a1@0, a@1)]",- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b1@1, b@0), (a1@0, a@1)]+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1034,30 +1165,35 @@ fn multi_hash_join_key_ordering() -> Result<()> {Arc::new(FilterExec::try_new(predicate, top_join)?);// The bottom joins' join key ordering is adjusted based on the top join. And the top join should not introduce additional RepartitionExec- let expected = &[- "FilterExec: c@6 > 1",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(B@2, b1@6), (C@3, c@2), (AA@1, a1@5)]",- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]",- " RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]",- " RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, filter_top_join.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, filter_top_join, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib =+ test_config.to_plan(filter_top_join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(+ plan_distrib,+ @r"+ FilterExec: c@6 > 1+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(B@2, b1@6), (C@3, c@2), (AA@1, a1@5)]+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]+ RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]+ RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ "+ );+ let plan_sort = test_config.to_plan(filter_top_join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1168,34 +1304,34 @@ fn reorder_join_keys_to_left_input() -> Result<()> {&top_join_on,&join_type,);- let top_join_plan =- format!("HashJoinExec: mode=Partitioned, join_type={:?}, on=[(AA@1, a1@5), (B@2, b1@6), (C@3, c@2)]", &join_type);- let reordered = reorder_join_keys_to_inputs(top_join)?;+ let reordered = reorder_join_keys_to_inputs(top_join).unwrap();// The top joins' join key ordering is adjusted based on the children inputs.- let expected = &[- top_join_plan.as_str(),- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1), (c@2, c1@2)]",- " RepartitionExec: partitioning=Hash([a@0, b@1, c@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([a1@0, b1@1, c1@2], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]",- " RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-- assert_plan_txt!(expected, reordered);+ let (captured_join_type, modified_plan) =+ hide_first(reordered.as_ref(), r"join_type=(\w+)", "join_type=...");+ assert_eq!(captured_join_type, join_type.to_string());++ insta::allow_duplicates! {insta::assert_snapshot!(modified_plan, @r"+HashJoinExec: mode=Partitioned, join_type=..., on=[(AA@1, a1@5), (B@2, b1@6), (C@3, c@2)]+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1), (c@2, c1@2)]+ RepartitionExec: partitioning=Hash([a@0, b@1, c@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([a1@0, b1@1, c1@2], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]+ RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");}}Ok(())@@ -1302,34 +1438,32 @@ fn reorder_join_keys_to_right_input() -> Result<()> {&top_join_on,&join_type,);- let top_join_plan =- format!("HashJoinExec: mode=Partitioned, join_type={:?}, on=[(C@3, c@2), (B@2, b1@6), (AA@1, a1@5)]", &join_type);- let reordered = reorder_join_keys_to_inputs(top_join)?;+ let reordered = reorder_join_keys_to_inputs(top_join).unwrap();// The top joins' join key ordering is adjusted based on the children inputs.- let expected = &[- top_join_plan.as_str(),- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]",- " RepartitionExec: partitioning=Hash([a@0, b@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([a1@0, b1@1], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]",- " RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-- assert_plan_txt!(expected, reordered);+ let (_, plan_str) =+ hide_first(reordered.as_ref(), r"join_type=(\w+)", "join_type=...");+ insta::allow_duplicates! {insta::assert_snapshot!(plan_str, @r"+HashJoinExec: mode=Partitioned, join_type=..., on=[(C@3, c@2), (B@2, b1@6), (AA@1, a1@5)]+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]+ RepartitionExec: partitioning=Hash([a@0, b@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([a1@0, b1@1], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]+ RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");}}Ok(())@@ -1670,52 +1804,52 @@ fn smj_join_key_ordering() -> Result<()> {// Test: run EnforceDistribution, then EnforceSort.// Only two RepartitionExecs added- let expected = &[- "SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]",- " SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]",- " ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]",- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]",- " ProjectionExec: expr=[a@1 as a2, b@0 as b2]",- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib, @r"+SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]+ SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]",- " RepartitionExec: partitioning=Hash([b3@1, a3@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b3@1 ASC, a3@0 ASC",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]",- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " RepartitionExec: partitioning=Hash([b2@1, a2@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b2@1 ASC, a2@0 ASC",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " ProjectionExec: expr=[a@1 as a2, b@0 as b2]",- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, join, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort, @r"+SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]+ RepartitionExec: partitioning=Hash([b3@1, a3@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b3@1 ASC, a3@0 ASC+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ RepartitionExec: partitioning=Hash([b2@1, a2@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b2@1 ASC, a2@0 ASC+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");Ok(())}@@ -1744,13 +1878,14 @@ fn merge_does_not_need_sort() -> Result<()> {//// The optimizer should not add an additional SortExec as the// data is already sorted- let expected = &[- "SortPreservingMergeExec: [a@0 ASC]",- " CoalesceBatchesExec: target_batch_size=4096",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, exec.clone(), &DISTRIB_DISTRIB_SORT)?;+ let plan_distrib = test_config.to_plan(exec.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [a@0 ASC]+ CoalesceBatchesExec: target_batch_size=4096+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");// Test: result IS DIFFERENT, if EnforceSorting is run first://@@ -1758,13 +1893,14 @@ fn merge_does_not_need_sort() -> Result<()> {// (according to flag: PREFER_EXISTING_SORT)// hence in this case ordering lost during CoalescePartitionsExec and re-introduced with// SortExec at the top.- let expected_first_sort_enforcement = &[- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " CoalesceBatchesExec: target_batch_size=4096",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, exec, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(exec, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ CoalesceBatchesExec: target_batch_size=4096+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");Ok(())}@@ -1790,25 +1926,26 @@ fn union_to_interleave() -> Result<()> {aggregate_exec_with_alias(plan, vec![("a1".to_string(), "a2".to_string())]);// Only two RepartitionExecs added, no final RepartitionExec required- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",- " AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]",- " InterleaveExec",- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]+ AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]+ InterleaveExec+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1834,28 +1971,29 @@ fn union_not_to_interleave() -> Result<()> {aggregate_exec_with_alias(plan, vec![("a1".to_string(), "a2".to_string())]);// Only two RepartitionExecs added, no final RepartitionExec required- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",- " RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=20",- " AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]",- " UnionExec",- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-// TestConfig: Prefer existing union.let test_config = TestConfig::default().with_prefer_existing_union();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]+ RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=20+ AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]+ UnionExec+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1865,17 +2003,18 @@ fn added_repartition_to_single_partition() -> Result<()> {let alias = vec![("a".to_string(), "a".to_string())];let plan = aggregate_exec_with_alias(parquet_exec(), alias);- let expected = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(&expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1885,18 +2024,19 @@ fn repartition_deepest_node() -> Result<()> {let alias = vec![("a".to_string(), "a".to_string())];let plan = aggregate_exec_with_alias(filter_exec(parquet_exec()), alias);- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1905,19 +2045,20 @@ fn repartition_deepest_node() -> Result<()> {fn repartition_unsorted_limit() -> Result<()> {let plan = limit_exec(filter_exec(parquet_exec()));- let expected = &[- "GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // nothing sorts the data, so the local limit doesn't require sorted data either- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ // nothing sorts the data, so the local limit doesn't require sorted data either+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1932,17 +2073,18 @@ fn repartition_sorted_limit() -> Result<()> {.into();let plan = limit_exec(sort_exec(sort_key, parquet_exec()));- let expected = &[- "GlobalLimitExec: skip=0, fetch=100",- " LocalLimitExec: fetch=100",- // data is sorted so can't repartition here- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // data is sorted so can't repartition here+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1960,19 +2102,20 @@ fn repartition_sorted_limit_with_filter() -> Result<()> {sort_key,);- let expected = &[- "SortRequiredExec: [c@2 ASC]",- " FilterExec: c@2 = 0",- // We can use repartition here, ordering requirement by SortRequiredExec- // is still satisfied.- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortRequiredExec: [c@2 ASC]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // We can use repartition here, ordering requirement by SortRequiredExec+ // is still satisfied.+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -1985,26 +2128,28 @@ fn repartition_ignores_limit() -> Result<()> {alias,);- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // repartition should happen prior to the filter to maximize parallelism- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- " LocalLimitExec: fetch=100",- // Expect no repartition to happen for local limit- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // repartition should happen prior to the filter to maximize parallelism+ // Expect no repartition to happen for local limit (DataSourceExec)++ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2013,19 +2158,20 @@ fn repartition_ignores_limit() -> Result<()> {fn repartition_ignores_union() -> Result<()> {let plan = union_exec(vec![parquet_exec(); 5]);- let expected = &[- "UnionExec",- // Expect no repartition of DataSourceExec- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Expect no repartition of DataSourceExec+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2041,15 +2187,15 @@ fn repartition_through_sort_preserving_merge() -> Result<()> {.into();let plan = sort_preserving_merge_exec(sort_key, parquet_exec());- // need resort as the data was not sorted correctly- let expected = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2068,24 +2214,25 @@ fn repartition_ignores_sort_preserving_merge() -> Result<()> {parquet_exec_multiple_sorted(vec![sort_key]),);+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);// Test: run EnforceDistribution, then EnforceSort- //+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [c@2 ASC]+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");// should not sort (as the data was already sorted)// should not repartition, since increased parallelism is not beneficial for SortPReservingMerge- let expected = &[- "SortPreservingMergeExec: [c@2 ASC]",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");Ok(())}@@ -2105,27 +2252,29 @@ fn repartition_ignores_sort_preserving_merge_with_union() -> Result<()> {]);let plan = sort_preserving_merge_exec(sort_key, input);+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);// Test: run EnforceDistribution, then EnforceSort.+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [c@2 ASC]+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");//// should not repartition / sort (as the data was already sorted)- let expected = &[- "SortPreservingMergeExec: [c@2 ASC]",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;// test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");Ok(())}@@ -2149,16 +2298,17 @@ fn repartition_does_not_destroy_sort() -> Result<()> {// TestConfig: Prefer existing sort.let test_config = TestConfig::default().with_prefer_existing_sort();+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortRequiredExec: [d@3 ASC]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet+");// during repartitioning ordering is preserved- let expected = &[- "SortRequiredExec: [d@3 ASC]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet",- ];-- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2187,22 +2337,25 @@ fn repartition_does_not_destroy_sort_more_complex() -> Result<()> {let input2 = filter_exec(parquet_exec());let plan = union_exec(vec![input1, input2]);+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+UnionExec+ SortRequiredExec: [c@2 ASC]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // union input 1: no repartitioning+ // union input 2: should repartition+ //// should not repartition below the SortRequired as that// branch doesn't benefit from increased parallelism- let expected = &[- "UnionExec",- // union input 1: no repartitioning- " SortRequiredExec: [c@2 ASC]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- // union input 2: should repartition- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2227,28 +2380,28 @@ fn repartition_transitively_with_projection() -> Result<()> {.into();let plan = sort_preserving_merge_exec(sort_key, proj);- // Test: run EnforceDistribution, then EnforceSort.- let expected = &[- "SortPreservingMergeExec: [sum@0 ASC]",- " SortExec: expr=[sum@0 ASC], preserve_partitioning=[true]",- // Since this projection is not trivial, increasing parallelism is beneficial- " ProjectionExec: expr=[a@0 + b@1 as sum]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [sum@0 ASC]+ SortExec: expr=[sum@0 ASC], preserve_partitioning=[true]+ ProjectionExec: expr=[a@0 + b@1 as sum]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[sum@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- // Since this projection is not trivial, increasing parallelism is beneficial- " ProjectionExec: expr=[a@0 + b@1 as sum]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[sum@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ ProjectionExec: expr=[a@0 + b@1 as sum]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Since this projection is not trivial, increasing parallelism is beneficialOk(())}@@ -2275,16 +2428,18 @@ fn repartition_ignores_transitively_with_projection() -> Result<()> {sort_key,);- let expected = &[- "SortRequiredExec: [c@2 ASC]",- // Since this projection is trivial, increasing parallelism is not beneficial- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortRequiredExec: [c@2 ASC]+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ // Since this projection is trivial, increasing parallelism is not beneficial++ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2310,16 +2465,17 @@ fn repartition_transitively_past_sort_with_projection() -> Result<()> {),);- let expected = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // Since this projection is trivial, increasing parallelism is not beneficial- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];-let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Since this projection is trivial, increasing parallelism is not beneficial+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -2334,28 +2490,30 @@ fn repartition_transitively_past_sort_with_filter() -> Result<()> {.into();let plan = sort_exec(sort_key, filter_exec(parquet_exec()));- // Test: run EnforceDistribution, then EnforceSort.- let expected = &[- "SortPreservingMergeExec: [a@0 ASC]",- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",- // Expect repartition on the input to the sort (as it can benefit from additional parallelism)- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [a@0 ASC]+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");++ // Expect repartition on the input to the sort (as it can benefit from additional parallelism)// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " FilterExec: c@2 = 0",- // Expect repartition on the input of the filter (as it can benefit from additional parallelism)- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Expect repartition on the input of the filter (as it can benefit from additional parallelism)Ok(())}@@ -2381,30 +2539,32 @@ fn repartition_transitively_past_sort_with_projection_and_filter() -> Result<()>),);- // Test: run EnforceDistribution, then EnforceSort.- let expected = &[- "SortPreservingMergeExec: [a@0 ASC]",- // Expect repartition on the input to the sort (as it can benefit from additional parallelism)- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",- " FilterExec: c@2 = 0",- // repartition is lowest down- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [a@0 ASC]+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");++ // Expect repartition on the input to the sort (as it can benefit from additional parallelism)+ // repartition is lowest down// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");Ok(())}@@ -2420,28 +2580,29 @@ fn parallelization_single_partition() -> Result<()> {.with_query_execution_partitions(2);// Test: with parquet- let expected_parquet = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- &expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2465,40 +2626,31 @@ fn parallelization_multiple_files() -> Result<()> {// The groups must have only contiguous ranges of rows from the same file// if any group has rows from multiple files, the data is no longer sorted destroyed// https://github.com/apache/datafusion/issues/8451- let expected_with_3_target_partitions = [- "SortRequiredExec: [a@0 ASC]",- " FilterExec: c@2 = 0",- " DataSourceExec: file_groups={3 groups: [[x:0..50], [y:0..100], [x:50..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];let test_config_concurrency_3 =test_config.clone().with_query_execution_partitions(3);- test_config_concurrency_3.run(- &expected_with_3_target_partitions,- plan.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config_concurrency_3.run(- &expected_with_3_target_partitions,- plan.clone(),- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_3_distrib =+ test_config_concurrency_3.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_3_distrib,+ @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ DataSourceExec: file_groups={3 groups: [[x:0..50], [y:0..100], [x:50..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");+ let plan_3_sort =+ test_config_concurrency_3.to_plan(plan.clone(), &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_3_distrib, plan_3_sort);- let expected_with_8_target_partitions = [- "SortRequiredExec: [a@0 ASC]",- " FilterExec: c@2 = 0",- " DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];let test_config_concurrency_8 = test_config.with_query_execution_partitions(8);- test_config_concurrency_8.run(- &expected_with_8_target_partitions,- plan.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config_concurrency_8.run(- &expected_with_8_target_partitions,- plan,- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_8_distrib =+ test_config_concurrency_8.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_8_distrib,+ @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");+ let plan_8_sort = test_config_concurrency_8.to_plan(plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_8_distrib, plan_8_sort);Ok(())}@@ -2570,30 +2722,30 @@ fn parallelization_two_partitions() -> Result<()> {.with_prefer_repartition_file_scans(10);// Test: with parquet- let expected_parquet = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- // Plan already has two partitions- " DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- &expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Plan already has two partitions+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- // Plan already has two partitions- " DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib, @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ // Plan already has two partitions+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2609,30 +2761,32 @@ fn parallelization_two_partitions_into_four() -> Result<()> {.with_prefer_repartition_file_scans(10);// Test: with parquet- let expected_parquet = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- // Multiple source files split across partitions- " DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- &expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ // Multiple source files split across partitions+ assert_plan!(plan_parquet_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Multiple source files split across partitions+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = [- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- // Multiple source files split across partitions- " DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ // Multiple source files split across partitions+ assert_plan!(plan_csv_distrib, @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ // Multiple source files split across partitions+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2651,32 +2805,32 @@ fn parallelization_sorted_limit() -> Result<()> {let test_config = TestConfig::default();// Test: with parquet- let expected_parquet = &[- "GlobalLimitExec: skip=0, fetch=100",- " LocalLimitExec: fetch=100",- // data is sorted so can't repartition here- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // Doesn't parallelize for SortExec without preserve_partitioning- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib, @r"+GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // data is sorted so can't repartition here+ // Doesn't parallelize for SortExec without preserve_partitioning+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "GlobalLimitExec: skip=0, fetch=100",- " LocalLimitExec: fetch=100",- // data is sorted so can't repartition here- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // Doesn't parallelize for SortExec without preserve_partitioning- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ // data is sorted so can't repartition here+ // Doesn't parallelize for SortExec without preserve_partitioning+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2696,40 +2850,41 @@ fn parallelization_limit_with_filter() -> Result<()> {let test_config = TestConfig::default();// Test: with parquet- let expected_parquet = &[- "GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // even though data is sorted, we can use repartition here. Since- // ordering is not used in subsequent stages anyway.- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // SortExec doesn't benefit from input partitioning- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ // even though data is sorted, we can use repartition here. Since+ // ordering is not used in subsequent stages anyway.+ // SortExec doesn't benefit from input partitioning+ assert_plan!(plan_parquet_distrib,+ @r"+GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // even though data is sorted, we can use repartition here. Since- // ordering is not used in subsequent stages anyway.- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // SortExec doesn't benefit from input partitioning- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ // even though data is sorted, we can use repartition here. Since+ // ordering is not used in subsequent stages anyway.+ // SortExec doesn't benefit from input partitioning+ assert_plan!(plan_csv_distrib,+ @r"+GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2747,48 +2902,49 @@ fn parallelization_ignores_limit() -> Result<()> {let test_config = TestConfig::default();// Test: with parquet- let expected_parquet = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // repartition should happen prior to the filter to maximize parallelism- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- // Limit doesn't benefit from input partitioning - no parallelism- " LocalLimitExec: fetch=100",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ ");+ // repartition should happen prior to the filter to maximize parallelism+ // Limit doesn't benefit from input partitioning - no parallelism+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- " CoalescePartitionsExec",- " LocalLimitExec: fetch=100",- " FilterExec: c@2 = 0",- // repartition should happen prior to the filter to maximize parallelism- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " GlobalLimitExec: skip=0, fetch=100",- // Limit doesn't benefit from input partitioning - no parallelism- " LocalLimitExec: fetch=100",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ CoalescePartitionsExec+ LocalLimitExec: fetch=100+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ GlobalLimitExec: skip=0, fetch=100+ LocalLimitExec: fetch=100+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ ");+ // repartition should happen prior to the filter to maximize parallelism+ // Limit doesn't benefit from input partitioning - no parallelism+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2801,34 +2957,35 @@ fn parallelization_union_inputs() -> Result<()> {let test_config = TestConfig::default();// Test: with parquet- let expected_parquet = &[- "UnionExec",- // Union doesn't benefit from input partitioning - no parallelism- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ // Union doesn't benefit from input partitioning - no parallelism+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "UnionExec",- // Union doesn't benefit from input partitioning - no parallelism- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false+");+ // Union doesn't benefit from input partitioning - no parallelism+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2855,22 +3012,21 @@ fn parallelization_prior_to_sort_preserving_merge() -> Result<()> {// parallelization is not beneficial for SortPreservingMerge// Test: with parquet- let expected_parquet = &[- "DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet"+ );+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false"+ );+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -2900,54 +3056,47 @@ fn parallelization_sort_preserving_merge_with_union() -> Result<()> {// should not sort (as the data was already sorted)// Test: with parquet- let expected_parquet = &[- "SortPreservingMergeExec: [c@2 ASC]",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- let expected_parquet_first_sort_enforcement = &[- // no SPM- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // has coalesce- " CoalescePartitionsExec",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(- expected_parquet_first_sort_enforcement,- plan_parquet,- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+ SortPreservingMergeExec: [c@2 ASC]+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ ");+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_sort,+ @r"+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ ");+ // no SPM+ // has coalesce// Test: with csv- let expected_csv = &[- "SortPreservingMergeExec: [c@2 ASC]",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- let expected_csv_first_sort_enforcement = &[- // no SPM- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- // has coalesce- " CoalescePartitionsExec",- " UnionExec",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- test_config.run(- expected_csv_first_sort_enforcement,- plan_csv.clone(),- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+ SortPreservingMergeExec: [c@2 ASC]+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+ ");+ let plan_csv_sort = test_config.to_plan(plan_csv.clone(), &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_sort,+ @r"+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ UnionExec+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+ ");+ // no SPM+ // has coalesceOk(())}@@ -2975,24 +3124,25 @@ fn parallelization_does_not_benefit() -> Result<()> {// no parallelization, because SortRequiredExec doesn't benefit from increased parallelism// Test: with parquet- let expected_parquet = &[- "SortRequiredExec: [c@2 ASC]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_parquet_distrib,+ @r"+ SortRequiredExec: [c@2 ASC]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ ");+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);// Test: with csv- let expected_csv = &[- "SortRequiredExec: [c@2 ASC]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_csv_distrib,+ @r"+ SortRequiredExec: [c@2 ASC]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+ ");+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_csv_distrib, plan_csv_sort);Ok(())}@@ -3023,26 +3173,26 @@ fn parallelization_ignores_transitively_with_projection_parquet() -> Result<()>.into();let plan_parquet =sort_preserving_merge_exec(sort_key_after_projection, proj_parquet);- let expected = &[- "SortPreservingMergeExec: [c2@1 ASC]",- " ProjectionExec: expr=[a@0 as a2, c@2 as c2]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- plans_matches_expected!(expected, &plan_parquet);+ assert_plan!(plan_parquet,+ @r"+ SortPreservingMergeExec: [c2@1 ASC]+ ProjectionExec: expr=[a@0 as a2, c@2 as c2]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+ ");++ let test_config = TestConfig::default();+ let plan_parquet_distrib =+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);// Expected Outcome:// data should not be repartitioned / resorted- let expected_parquet = &[- "ProjectionExec: expr=[a@0 as a2, c@2 as c2]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- let test_config = TestConfig::default();- test_config.run(- expected_parquet,- plan_parquet.clone(),- &DISTRIB_DISTRIB_SORT,- )?;- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;+ assert_plan!(plan_parquet_distrib,+ @r"+ProjectionExec: expr=[a@0 as a2, c@2 as c2]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);Ok(())}@@ -3071,22 +3221,24 @@ fn parallelization_ignores_transitively_with_projection_csv() -> Result<()> {}].into();let plan_csv = sort_preserving_merge_exec(sort_key_after_projection, proj_csv);- let expected = &[- "SortPreservingMergeExec: [c2@1 ASC]",- " ProjectionExec: expr=[a@0 as a2, c@2 as c2]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- plans_matches_expected!(expected, &plan_csv);+ assert_plan!(plan_csv,+ @r"+SortPreservingMergeExec: [c2@1 ASC]+ ProjectionExec: expr=[a@0 as a2, c@2 as c2]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+");+ let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+ProjectionExec: expr=[a@0 as a2, c@2 as c2]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false+");// Expected Outcome:// data should not be repartitioned / resorted- let expected_csv = &[- "ProjectionExec: expr=[a@0 as a2, c@2 as c2]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",- ];- let test_config = TestConfig::default();- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;+ let plan_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3096,24 +3248,25 @@ fn remove_redundant_roundrobins() -> Result<()> {let input = parquet_exec();let repartition = repartition_exec(repartition_exec(input));let physical_plan = repartition_exec(filter_exec(repartition));- let expected = &[- "RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- plans_matches_expected!(expected, &physical_plan);-- let expected = &[- "FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];+ assert_plan!(physical_plan,+ @r"+RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");let test_config = TestConfig::default();- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3133,18 +3286,19 @@ fn remove_unnecessary_spm_after_filter() -> Result<()> {// TestConfig: Prefer existing sort.let test_config = TestConfig::default().with_prefer_existing_sort();+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);// Expected Outcome:// Original plan expects its output to be ordered by c@2 ASC.// This is still satisfied since, after filter that column is constant.- let expected = &[- "CoalescePartitionsExec",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=c@2 ASC",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];-- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ assert_plan!(plan_distrib,+ @r"+CoalescePartitionsExec+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=c@2 ASC+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3164,14 +3318,16 @@ fn preserve_ordering_through_repartition() -> Result<()> {// TestConfig: Prefer existing sort.let test_config = TestConfig::default().with_prefer_existing_sort();- let expected = &[- "SortPreservingMergeExec: [d@3 ASC]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=d@3 ASC",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [d@3 ASC]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=d@3 ASC+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3189,29 +3345,27 @@ fn do_not_preserve_ordering_through_repartition() -> Result<()> {let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);// Test: run EnforceDistribution, then EnforceSort.- let expected = &[- "SortPreservingMergeExec: [a@0 ASC]",- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [a@0 ASC]+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];- test_config.run(- expected_first_sort_enforcement,- physical_plan,- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");Ok(())}@@ -3227,17 +3381,18 @@ fn no_need_for_sort_after_filter() -> Result<()> {let input = parquet_exec_multiple_sorted(vec![sort_key.clone()]);let physical_plan = sort_preserving_merge_exec(sort_key, filter_exec(input));- let expected = &[- // After CoalescePartitionsExec c is still constant. Hence c@2 ASC ordering is already satisfied.- "CoalescePartitionsExec",- // Since after this stage c is constant. c@2 ASC ordering is already satisfied.- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib, @r"+CoalescePartitionsExec+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);+ // After CoalescePartitionsExec c is still constant. Hence c@2 ASC ordering is already satisfied.+ // Since after this stage c is constant. c@2 ASC ordering is already satisfied.Ok(())}@@ -3261,30 +3416,28 @@ fn do_not_preserve_ordering_through_repartition2() -> Result<()> {let test_config = TestConfig::default();+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);// Test: run EnforceDistribution, then EnforceSort.- let expected = &[- "SortPreservingMergeExec: [a@0 ASC]",- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;+ assert_plan!(plan_distrib,+ @r"+SortPreservingMergeExec: [a@0 ASC]+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");// Test: result IS DIFFERENT, if EnforceSorting is run first:- let expected_first_sort_enforcement = &[- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(- expected_first_sort_enforcement,- physical_plan,- &SORT_DISTRIB_DISTRIB,- )?;+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_sort,+ @r"+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");Ok(())}@@ -3300,14 +3453,16 @@ fn do_not_preserve_ordering_through_repartition3() -> Result<()> {let input = parquet_exec_multiple_sorted(vec![sort_key]);let physical_plan = filter_exec(input);- let expected = &[- "FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3322,30 +3477,27 @@ fn do_not_put_sort_when_input_is_invalid() -> Result<()> {.into();let input = parquet_exec();let physical_plan = sort_required_exec_with_req(filter_exec(input), sort_key);- let expected = &[- // Ordering requirement of sort required exec is NOT satisfied- // by existing ordering at the source.- "SortRequiredExec: [a@0 ASC]",- " FilterExec: c@2 = 0",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- assert_plan_txt!(expected, physical_plan);-- let expected = &[- "SortRequiredExec: [a@0 ASC]",- // Since at the start of the rule ordering requirement is not satisfied- // EnforceDistribution rule doesn't satisfy this requirement either.- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];+ // Ordering requirement of sort required exec is NOT satisfied+ // by existing ordering at the source.+ assert_plan!(physical_plan, @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");let mut config = ConfigOptions::new();config.execution.target_partitions = 10;config.optimizer.enable_round_robin_repartition = true;config.optimizer.prefer_existing_sort = false;let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;- assert_plan_txt!(expected, dist_plan);+ // Since at the start of the rule ordering requirement is not satisfied+ // EnforceDistribution rule doesn't satisfy this requirement either.+ assert_plan!(dist_plan, @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");Ok(())}@@ -3361,29 +3513,26 @@ fn put_sort_when_input_is_valid() -> Result<()> {let input = parquet_exec_multiple_sorted(vec![sort_key.clone()]);let physical_plan = sort_required_exec_with_req(filter_exec(input), sort_key);- let expected = &[- // Ordering requirement of sort required exec is satisfied- // by existing ordering at the source.- "SortRequiredExec: [a@0 ASC]",- " FilterExec: c@2 = 0",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];- assert_plan_txt!(expected, physical_plan);-- let expected = &[- // Since at the start of the rule ordering requirement is satisfied- // EnforceDistribution rule satisfy this requirement also.- "SortRequiredExec: [a@0 ASC]",- " FilterExec: c@2 = 0",- " DataSourceExec: file_groups={10 groups: [[x:0..20], [y:0..20], [x:20..40], [y:20..40], [x:40..60], [y:40..60], [x:60..80], [y:60..80], [x:80..100], [y:80..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",- ];+ // Ordering requirement of sort required exec is satisfied+ // by existing ordering at the source.+ assert_plan!(physical_plan, @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");let mut config = ConfigOptions::new();config.execution.target_partitions = 10;config.optimizer.enable_round_robin_repartition = true;config.optimizer.prefer_existing_sort = false;let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;- assert_plan_txt!(expected, dist_plan);+ // Since at the start of the rule ordering requirement is satisfied+ // EnforceDistribution rule satisfy this requirement also.+ assert_plan!(dist_plan, @r"+SortRequiredExec: [a@0 ASC]+ FilterExec: c@2 = 0+ DataSourceExec: file_groups={10 groups: [[x:0..20], [y:0..20], [x:20..40], [y:20..40], [x:40..60], [y:40..60], [x:60..80], [y:60..80], [x:80..100], [y:80..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet+");Ok(())}@@ -3404,13 +3553,15 @@ fn do_not_add_unnecessary_hash() -> Result<()> {// Make sure target partition number is 1. In this case hash repartition is unnecessary.let test_config = TestConfig::default().with_query_execution_partitions(1);- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3432,19 +3583,21 @@ fn do_not_add_unnecessary_hash2() -> Result<()> {// Make sure target partition number is larger than 2 (e.g partition number at the source).let test_config = TestConfig::default().with_query_execution_partitions(4);- let expected = &[- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- // Since hash requirements of this operator is satisfied. There shouldn't be- // a hash repartition here- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",- " RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2",- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]+ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet+");+ // Since hash requirements of this operator is satisfied. There shouldn't be+ // a hash repartition here+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3452,19 +3605,21 @@ fn do_not_add_unnecessary_hash2() -> Result<()> {#[test]fn optimize_away_unnecessary_repartition() -> Result<()> {let physical_plan = coalesce_partitions_exec(repartition_exec(parquet_exec()));- let expected = &[- "CoalescePartitionsExec",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- plans_matches_expected!(expected, physical_plan.clone());-- let expected =- &["DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet"];+ assert_plan!(physical_plan,+ @r"+CoalescePartitionsExec+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");let test_config = TestConfig::default();- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3474,25 +3629,27 @@ fn optimize_away_unnecessary_repartition2() -> Result<()> {let physical_plan = filter_exec(repartition_exec(coalesce_partitions_exec(filter_exec(repartition_exec(parquet_exec())),)));- let expected = &[- "FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " CoalescePartitionsExec",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- plans_matches_expected!(expected, physical_plan.clone());+ assert_plan!(physical_plan,+ @r"+FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ CoalescePartitionsExec+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");- let expected = &[- "FilterExec: c@2 = 0",- " FilterExec: c@2 = 0",- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];let test_config = TestConfig::default();- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);+ assert_plan!(plan_distrib,+ @r"+FilterExec: c@2 = 0+ FilterExec: c@2 = 0+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);+ assert_plan!(plan_distrib, plan_sort);Ok(())}@@ -3512,27 +3669,31 @@ async fn test_distribute_sort_parquet() -> Result<()> {let physical_plan = sort_exec(sort_key, parquet_exec_with_stats(10000 * 8192));// prior to optimization, this is the starting plan- let starting = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- plans_matches_expected!(starting, physical_plan.clone());+ assert_plan!(physical_plan,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet+");// what the enforce distribution run does.- let expected = &[- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",- " CoalescePartitionsExec",- " DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected, physical_plan.clone(), &[Run::Distribution])?;+ let plan_distribution =+ test_config.to_plan(physical_plan.clone(), &[Run::Distribution]);+ assert_plan!(plan_distribution,+ @r"+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]+ CoalescePartitionsExec+ DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet+");// what the sort parallelization (in enforce sorting), does after the enforce distribution changes- let expected = &[- "SortPreservingMergeExec: [c@2 ASC]",- " SortExec: expr=[c@2 ASC], preserve_partitioning=[true]",- " DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet",- ];- test_config.run(expected, physical_plan, &[Run::Distribution, Run::Sorting])?;+ let plan_both =+ test_config.to_plan(physical_plan, &[Run::Distribution, Run::Sorting]);+ assert_plan!(plan_both,+ @r"+SortPreservingMergeExec: [c@2 ASC]+ SortExec: expr=[c@2 ASC], preserve_partitioning=[true]+ DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet+");Ok(())}@@ -3557,12 +3718,12 @@ async fn test_distribute_sort_memtable() -> Result<()> {let physical_plan = dataframe.create_physical_plan().await?;// this is the final, optimized plan- let expected = &[- "SortPreservingMergeExec: [id@0 ASC NULLS LAST]",- " SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]",- " DataSourceExec: partitions=3, partition_sizes=[34, 33, 33]",- ];- plans_matches_expected!(expected, physical_plan);+ assert_plan!(physical_plan,+ @r"+SortPreservingMergeExec: [id@0 ASC NULLS LAST]+ SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]+ DataSourceExec: partitions=3, partition_sizes=[34, 33, 33]+");Ok(())}