Skip to content

Apache DataFusion SQL Query Engine

Insta for enforce_distrubution (easy ones) (#18248)

Dmitrii Blaginine432d5561e5e
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index 63111f4380..db011c4be4 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -66,9 +66,52 @@ use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};
use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;
use datafusion_physical_plan::union::UnionExec;
use datafusion_physical_plan::{
- get_plan_string, DisplayAs, DisplayFormatType, ExecutionPlanProperties,
+ displayable, get_plan_string, DisplayAs, DisplayFormatType, ExecutionPlanProperties,
PlanProperties, Statistics,
};
+use insta::Settings;
+
+/// Helper function to replace only the first occurrence of a regex pattern in a plan
+/// Returns (captured_group_1, modified_string)
+fn hide_first(
+ plan: &dyn ExecutionPlan,
+ regex: &str,
+ replacement: &str,
+) -> (String, String) {
+ let plan_str = displayable(plan).indent(true).to_string();
+ let pattern = regex::Regex::new(regex).unwrap();
+
+ if let Some(captures) = pattern.captures(&plan_str) {
+ let full_match = captures.get(0).unwrap();
+ let captured_value = captures
+ .get(1)
+ .map(|m| m.as_str().to_string())
+ .unwrap_or_default();
+ let pos = full_match.start();
+ let end_pos = full_match.end();
+ let mut result = String::with_capacity(plan_str.len());
+ result.push_str(&plan_str[..pos]);
+ result.push_str(replacement);
+ result.push_str(&plan_str[end_pos..]);
+ (captured_value, result)
+ } else {
+ (String::new(), plan_str)
+ }
+}
+
+macro_rules! assert_plan {
+ ($plan: expr, @ $expected:literal) => {
+ insta::assert_snapshot!(
+ displayable($plan.as_ref()).indent(true).to_string(),
+ @ $expected
+ )
+ };
+ ($plan: expr, $another_plan: expr) => {
+ let plan1 = displayable($plan.as_ref()).indent(true).to_string();
+ let plan2 = displayable($another_plan.as_ref()).indent(true).to_string();
+ assert_eq!(plan1, plan2);
+ }
+}
/// Models operators like BoundedWindowExec that require an input
/// ordering but is easy to construct
@@ -352,22 +395,6 @@ fn ensure_distribution_helper(
ensure_distribution(distribution_context, &config).map(|item| item.data.plan)
}
-/// Test whether plan matches with expected plan
-macro_rules! plans_matches_expected {
- ($EXPECTED_LINES: expr, $PLAN: expr) => {
- let physical_plan = $PLAN;
- let actual = get_plan_string(&physical_plan);
-
- let expected_plan_lines: Vec<&str> = $EXPECTED_LINES
- .iter().map(|s| *s).collect();
-
- assert_eq!(
- expected_plan_lines, actual,
- "\n**Original Plan Mismatch\n\nexpected:\n\n{expected_plan_lines:#?}\nactual:\n\n{actual:#?}\n\n"
- );
- }
-}
-
fn test_suite_default_config_options() -> ConfigOptions {
let mut config = ConfigOptions::new();
@@ -442,6 +469,7 @@ impl TestConfig {
self
}
+ // This be deleted in https://github.com/apache/datafusion/pull/18185
/// Perform a series of runs using the current [`TestConfig`],
/// assert the expected plan result,
/// and return the result plan (for potential subsequent runs).
@@ -517,20 +545,79 @@ impl TestConfig {
Ok(optimized)
}
-}
-macro_rules! assert_plan_txt {
- ($EXPECTED_LINES: expr, $PLAN: expr) => {
- let expected_lines: Vec<&str> = $EXPECTED_LINES.iter().map(|s| *s).collect();
- // Now format correctly
- let actual_lines = get_plan_string(&$PLAN);
+ /// Perform a series of runs using the current [`TestConfig`],
+ /// assert the expected plan result,
+ /// and return the result plan (for potential subsequent runs).
+ fn try_to_plan(
+ &self,
+ plan: Arc<dyn ExecutionPlan>,
+ optimizers_to_run: &[Run],
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ // Add the ancillary output requirements operator at the start:
+ let optimizer = OutputRequirements::new_add_mode();
+ let mut optimized = optimizer.optimize(plan.clone(), &self.config)?;
- assert_eq!(
- &expected_lines, &actual_lines,
- "\n\nexpected:\n\n{:#?}\nactual:\n\n{:#?}\n\n",
- expected_lines, actual_lines
- );
- };
+ // This file has 2 rules that use tree node, apply these rules to original plan consecutively
+ // After these operations tree nodes should be in a consistent state.
+ // This code block makes sure that these rules doesn't violate tree node integrity.
+ {
+ let adjusted = if self.config.optimizer.top_down_join_key_reordering {
+ // Run adjust_input_keys_ordering rule
+ let plan_requirements =
+ PlanWithKeyRequirements::new_default(plan.clone());
+ let adjusted = plan_requirements
+ .transform_down(adjust_input_keys_ordering)
+ .data()
+ .and_then(check_integrity)?;
+ // TODO: End state payloads will be checked here.
+ adjusted.plan
+ } else {
+ // Run reorder_join_keys_to_inputs rule
+ plan.clone()
+ .transform_up(|plan| {
+ Ok(Transformed::yes(reorder_join_keys_to_inputs(plan)?))
+ })
+ .data()?
+ };
+
+ // Then run ensure_distribution rule
+ DistributionContext::new_default(adjusted)
+ .transform_up(|distribution_context| {
+ ensure_distribution(distribution_context, &self.config)
+ })
+ .data()
+ .and_then(check_integrity)?;
+ // TODO: End state payloads will be checked here.
+ }
+
+ for run in optimizers_to_run {
+ optimized = match run {
+ Run::Distribution => {
+ let optimizer = EnforceDistribution::new();
+ optimizer.optimize(optimized, &self.config)?
+ }
+ Run::Sorting => {
+ let optimizer = EnforceSorting::new();
+ optimizer.optimize(optimized, &self.config)?
+ }
+ };
+ }
+
+ // Remove the ancillary output requirements operator when done:
+ let optimizer = OutputRequirements::new_remove_mode();
+ let optimized = optimizer.optimize(optimized, &self.config)?;
+
+ Ok(optimized)
+ }
+
+ fn to_plan(
+ &self,
+ plan: Arc<dyn ExecutionPlan>,
+ optimizers_to_run: &[Run],
+ ) -> Arc<dyn ExecutionPlan> {
+ self.try_to_plan(plan, optimizers_to_run).unwrap()
+ }
}
#[test]
@@ -556,6 +643,8 @@ fn multi_hash_joins() -> Result<()> {
JoinType::RightAnti,
];
+ let settings = Settings::clone_current();
+
// Join on (a == b1)
let join_on = vec![(
Arc::new(Column::new_with_schema("a", &schema()).unwrap()) as _,
@@ -564,11 +653,17 @@ fn multi_hash_joins() -> Result<()> {
for join_type in join_types {
let join = hash_join_exec(left.clone(), right.clone(), &join_on, &join_type);
- let join_plan = |shift| -> String {
- format!("{}HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(a@0, b1@1)]", " ".repeat(shift))
- };
- let join_plan_indent2 = join_plan(2);
- let join_plan_indent4 = join_plan(4);
+
+ let mut settings = settings.clone();
+ settings.add_filter(
+ // join_type={} replace with join_type=... to avoid snapshot name issue
+ format!("join_type={join_type}").as_str(),
+ "join_type=...",
+ );
+
+ insta::allow_duplicates! {
+ settings.bind( || {
+
match join_type {
JoinType::Inner
@@ -589,50 +684,58 @@ fn multi_hash_joins() -> Result<()> {
&top_join_on,
&join_type,
);
- let top_join_plan =
- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(a@0, c@2)]");
- let expected = match join_type {
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);
+
+ match join_type {
// Should include 3 RepartitionExecs
- JoinType::Inner | JoinType::Left | JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => vec![
- top_join_plan.as_str(),
- &join_plan_indent2,
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ],
+ JoinType::Inner | JoinType::Left | JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {
+
+ assert_plan!(plan_distrib, @r"
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, c@2)]
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ },
// Should include 4 RepartitionExecs
- _ => vec![
- top_join_plan.as_str(),
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- &join_plan_indent4,
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ],
+ _ => {
+ assert_plan!(plan_distrib, @r"
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, c@2)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ },
};
- let test_config = TestConfig::default();
- test_config.run(&expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected, top_join, &SORT_DISTRIB_DISTRIB)?;
+
+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
}
JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {}
}
+
+
match join_type {
JoinType::Inner
| JoinType::Left
@@ -650,55 +753,64 @@ fn multi_hash_joins() -> Result<()> {
let top_join =
hash_join_exec(join, parquet_exec(), &top_join_on, &join_type);
- let top_join_plan = match join_type {
- JoinType::RightSemi | JoinType::RightAnti =>
- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(b1@1, c@2)]"),
- _ =>
- format!("HashJoinExec: mode=Partitioned, join_type={join_type}, on=[(b1@6, c@2)]"),
- };
- let expected = match join_type {
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);
+
+ match join_type {
// Should include 3 RepartitionExecs
- JoinType::Inner | JoinType::Right | JoinType::RightSemi | JoinType::RightAnti =>
- vec![
- top_join_plan.as_str(),
- &join_plan_indent2,
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ],
+ JoinType::Inner | JoinType::Right => {
+ assert_plan!(parquet_exec(), @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet");
+ },
+ // Should include 3 RepartitionExecs but have a different "on"
+ JoinType::RightSemi | JoinType::RightAnti => {
+ assert_plan!(plan_distrib, @r"
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(b1@1, c@2)]
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+
+ }
+
// Should include 4 RepartitionExecs
- _ =>
- vec![
- top_join_plan.as_str(),
- " RepartitionExec: partitioning=Hash([b1@6], 10), input_partitions=10",
- &join_plan_indent4,
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ],
+ _ => {
+ assert_plan!(plan_distrib, @r"
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(b1@6, c@2)]
+ RepartitionExec: partitioning=Hash([b1@6], 10), input_partitions=10
+ HashJoinExec: mode=Partitioned, join_type=..., on=[(a@0, b1@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1, d@3 as d1, e@4 as e1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+
+ },
};
- let test_config = TestConfig::default();
- test_config.run(&expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected, top_join, &SORT_DISTRIB_DISTRIB)?;
+
+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
}
JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => {}
}
+
+ });
+ }
}
Ok(())
@@ -737,23 +849,27 @@ fn multi_joins_after_alias() -> Result<()> {
);
// Output partition need to respect the Alias and should not introduce additional RepartitionExec
- let expected = &[
- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, c@2)]",
- " ProjectionExec: expr=[a@0 as a1, a@0 as a2]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, c@2)]
+ ProjectionExec: expr=[a@0 as a1, a@0 as a2]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
// Join on (a2 == c)
let top_join_on = vec![(
@@ -764,23 +880,27 @@ fn multi_joins_after_alias() -> Result<()> {
let top_join = hash_join_exec(projection, right, &top_join_on, &JoinType::Inner);
// Output partition need to respect the Alias and should not introduce additional RepartitionExec
- let expected = &[
- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a2@1, c@2)]",
- " ProjectionExec: expr=[a@0 as a1, a@0 as a2]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a2@1, c@2)]
+ ProjectionExec: expr=[a@0 as a1, a@0 as a2]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -816,26 +936,29 @@ fn multi_joins_after_multi_alias() -> Result<()> {
// The Column 'a' has different meaning now after the two Projections
// The original Output partition can not satisfy the Join requirements and need to add an additional RepartitionExec
- let expected = &[
- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, c@2)]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " ProjectionExec: expr=[c1@0 as a]",
- " ProjectionExec: expr=[c@2 as c1]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, top_join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(top_join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, c@2)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ ProjectionExec: expr=[c1@0 as a]
+ ProjectionExec: expr=[c@2 as c1]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, b@1)]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -861,22 +984,26 @@ fn join_after_agg_alias() -> Result<()> {
let join = hash_join_exec(left, right.clone(), &join_on, &JoinType::Inner);
// Only two RepartitionExecs added
- let expected = &[
- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, a2@0)]",
- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",
- " RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a2], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a1@0, a2@0)]
+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]
+ RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a2], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -914,23 +1041,27 @@ fn hash_join_key_ordering() -> Result<()> {
let join = hash_join_exec(left, right.clone(), &join_on, &JoinType::Inner);
// Only two RepartitionExecs added
- let expected = &[
- "HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b1@1, b@0), (a1@0, a@1)]",
- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",
- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b1@1, b@0), (a1@0, a@1)]
+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1034,30 +1165,35 @@ fn multi_hash_join_key_ordering() -> Result<()> {
Arc::new(FilterExec::try_new(predicate, top_join)?);
// The bottom joins' join key ordering is adjusted based on the top join. And the top join should not introduce additional RepartitionExec
- let expected = &[
- "FilterExec: c@6 > 1",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(B@2, b1@6), (C@3, c@2), (AA@1, a1@5)]",
- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]",
- " RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]",
- " RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, filter_top_join.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, filter_top_join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib =
+ test_config.to_plan(filter_top_join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(
+ plan_distrib,
+ @r"
+ FilterExec: c@6 > 1
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(B@2, b1@6), (C@3, c@2), (AA@1, a1@5)]
+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]
+ RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(b@1, b1@1), (c@2, c1@2), (a@0, a1@0)]
+ RepartitionExec: partitioning=Hash([b@1, c@2, a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b1@1, c1@2, a1@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ "
+ );
+ let plan_sort = test_config.to_plan(filter_top_join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1168,34 +1304,34 @@ fn reorder_join_keys_to_left_input() -> Result<()> {
&top_join_on,
&join_type,
);
- let top_join_plan =
- format!("HashJoinExec: mode=Partitioned, join_type={:?}, on=[(AA@1, a1@5), (B@2, b1@6), (C@3, c@2)]", &join_type);
- let reordered = reorder_join_keys_to_inputs(top_join)?;
+ let reordered = reorder_join_keys_to_inputs(top_join).unwrap();
// The top joins' join key ordering is adjusted based on the children inputs.
- let expected = &[
- top_join_plan.as_str(),
- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1), (c@2, c1@2)]",
- " RepartitionExec: partitioning=Hash([a@0, b@1, c@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([a1@0, b1@1, c1@2], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]",
- " RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
- assert_plan_txt!(expected, reordered);
+ let (captured_join_type, modified_plan) =
+ hide_first(reordered.as_ref(), r"join_type=(\w+)", "join_type=...");
+ assert_eq!(captured_join_type, join_type.to_string());
+
+ insta::allow_duplicates! {insta::assert_snapshot!(modified_plan, @r"
+HashJoinExec: mode=Partitioned, join_type=..., on=[(AA@1, a1@5), (B@2, b1@6), (C@3, c@2)]
+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1), (c@2, c1@2)]
+ RepartitionExec: partitioning=Hash([a@0, b@1, c@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([a1@0, b1@1, c1@2], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]
+ RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");}
}
Ok(())
@@ -1302,34 +1438,32 @@ fn reorder_join_keys_to_right_input() -> Result<()> {
&top_join_on,
&join_type,
);
- let top_join_plan =
- format!("HashJoinExec: mode=Partitioned, join_type={:?}, on=[(C@3, c@2), (B@2, b1@6), (AA@1, a1@5)]", &join_type);
- let reordered = reorder_join_keys_to_inputs(top_join)?;
+ let reordered = reorder_join_keys_to_inputs(top_join).unwrap();
// The top joins' join key ordering is adjusted based on the children inputs.
- let expected = &[
- top_join_plan.as_str(),
- " ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]",
- " RepartitionExec: partitioning=Hash([a@0, b@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([a1@0, b1@1], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]",
- " RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
- assert_plan_txt!(expected, reordered);
+ let (_, plan_str) =
+ hide_first(reordered.as_ref(), r"join_type=(\w+)", "join_type=...");
+ insta::allow_duplicates! {insta::assert_snapshot!(plan_str, @r"
+HashJoinExec: mode=Partitioned, join_type=..., on=[(C@3, c@2), (B@2, b1@6), (AA@1, a1@5)]
+ ProjectionExec: expr=[a@0 as A, a@0 as AA, b@1 as B, c@2 as C]
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(a@0, a1@0), (b@1, b1@1)]
+ RepartitionExec: partitioning=Hash([a@0, b@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([a1@0, b1@1], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c@2, c1@2), (b@1, b1@1), (a@0, a1@0)]
+ RepartitionExec: partitioning=Hash([c@2, b@1, a@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([c1@2, b1@1, a1@0], 10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ ProjectionExec: expr=[a@0 as a1, b@1 as b1, c@2 as c1]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");}
}
Ok(())
@@ -1670,52 +1804,52 @@ fn smj_join_key_ordering() -> Result<()> {
// Test: run EnforceDistribution, then EnforceSort.
// Only two RepartitionExecs added
- let expected = &[
- "SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]",
- " SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]",
- " ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]",
- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",
- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]",
- " ProjectionExec: expr=[a@1 as a2, b@0 as b2]",
- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected, join.clone(), &DISTRIB_DISTRIB_SORT)?;
+ let plan_distrib = test_config.to_plan(join.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib, @r"
+SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]
+ SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[true]
+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[true]
+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]",
- " RepartitionExec: partitioning=Hash([b3@1, a3@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b3@1 ASC, a3@0 ASC",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]",
- " ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]",
- " AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " RepartitionExec: partitioning=Hash([b2@1, a2@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b2@1 ASC, a2@0 ASC",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " ProjectionExec: expr=[a@1 as a2, b@0 as b2]",
- " AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, join, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(join, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort, @r"
+SortMergeJoin: join_type=Inner, on=[(b3@1, b2@1), (a3@0, a2@0)]
+ RepartitionExec: partitioning=Hash([b3@1, a3@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b3@1 ASC, a3@0 ASC
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ SortExec: expr=[b3@1 ASC, a3@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[a1@0 as a3, b1@1 as b3]
+ ProjectionExec: expr=[a1@1 as a1, b1@0 as b1]
+ AggregateExec: mode=FinalPartitioned, gby=[b1@0 as b1, a1@1 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([b1@0, a1@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b1, a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ RepartitionExec: partitioning=Hash([b2@1, a2@0], 10), input_partitions=10, preserve_order=true, sort_exprs=b2@1 ASC, a2@0 ASC
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ SortExec: expr=[b2@1 ASC, a2@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[a@1 as a2, b@0 as b2]
+ AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, a@1 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([b@0, a@1], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[b@1 as b, a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
Ok(())
}
@@ -1744,13 +1878,14 @@ fn merge_does_not_need_sort() -> Result<()> {
//
// The optimizer should not add an additional SortExec as the
// data is already sorted
- let expected = &[
- "SortPreservingMergeExec: [a@0 ASC]",
- " CoalesceBatchesExec: target_batch_size=4096",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, exec.clone(), &DISTRIB_DISTRIB_SORT)?;
+ let plan_distrib = test_config.to_plan(exec.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [a@0 ASC]
+ CoalesceBatchesExec: target_batch_size=4096
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
// Test: result IS DIFFERENT, if EnforceSorting is run first:
//
@@ -1758,13 +1893,14 @@ fn merge_does_not_need_sort() -> Result<()> {
// (according to flag: PREFER_EXISTING_SORT)
// hence in this case ordering lost during CoalescePartitionsExec and re-introduced with
// SortExec at the top.
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " CoalesceBatchesExec: target_batch_size=4096",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, exec, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(exec, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ CoalesceBatchesExec: target_batch_size=4096
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
Ok(())
}
@@ -1790,25 +1926,26 @@ fn union_to_interleave() -> Result<()> {
aggregate_exec_with_alias(plan, vec![("a1".to_string(), "a2".to_string())]);
// Only two RepartitionExecs added, no final RepartitionExec required
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",
- " AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]",
- " InterleaveExec",
- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]
+ AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]
+ InterleaveExec
+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1834,28 +1971,29 @@ fn union_not_to_interleave() -> Result<()> {
aggregate_exec_with_alias(plan, vec![("a1".to_string(), "a2".to_string())]);
// Only two RepartitionExecs added, no final RepartitionExec required
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]",
- " RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=20",
- " AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]",
- " UnionExec",
- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
// TestConfig: Prefer existing union.
let test_config = TestConfig::default().with_prefer_existing_union();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a2@0 as a2], aggr=[]
+ RepartitionExec: partitioning=Hash([a2@0], 10), input_partitions=20
+ AggregateExec: mode=Partial, gby=[a1@0 as a2], aggr=[]
+ UnionExec
+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ AggregateExec: mode=FinalPartitioned, gby=[a1@0 as a1], aggr=[]
+ RepartitionExec: partitioning=Hash([a1@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a1], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1865,17 +2003,18 @@ fn added_repartition_to_single_partition() -> Result<()> {
let alias = vec![("a".to_string(), "a".to_string())];
let plan = aggregate_exec_with_alias(parquet_exec(), alias);
- let expected = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(&expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1885,18 +2024,19 @@ fn repartition_deepest_node() -> Result<()> {
let alias = vec![("a".to_string(), "a".to_string())];
let plan = aggregate_exec_with_alias(filter_exec(parquet_exec()), alias);
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1905,19 +2045,20 @@ fn repartition_deepest_node() -> Result<()> {
fn repartition_unsorted_limit() -> Result<()> {
let plan = limit_exec(filter_exec(parquet_exec()));
- let expected = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // nothing sorts the data, so the local limit doesn't require sorted data either
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ // nothing sorts the data, so the local limit doesn't require sorted data either
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1932,17 +2073,18 @@ fn repartition_sorted_limit() -> Result<()> {
.into();
let plan = limit_exec(sort_exec(sort_key, parquet_exec()));
- let expected = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " LocalLimitExec: fetch=100",
- // data is sorted so can't repartition here
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // data is sorted so can't repartition here
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1960,19 +2102,20 @@ fn repartition_sorted_limit_with_filter() -> Result<()> {
sort_key,
);
- let expected = &[
- "SortRequiredExec: [c@2 ASC]",
- " FilterExec: c@2 = 0",
- // We can use repartition here, ordering requirement by SortRequiredExec
- // is still satisfied.
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortRequiredExec: [c@2 ASC]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // We can use repartition here, ordering requirement by SortRequiredExec
+ // is still satisfied.
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -1985,26 +2128,28 @@ fn repartition_ignores_limit() -> Result<()> {
alias,
);
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // repartition should happen prior to the filter to maximize parallelism
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- " LocalLimitExec: fetch=100",
- // Expect no repartition to happen for local limit
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // repartition should happen prior to the filter to maximize parallelism
+ // Expect no repartition to happen for local limit (DataSourceExec)
+
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2013,19 +2158,20 @@ fn repartition_ignores_limit() -> Result<()> {
fn repartition_ignores_union() -> Result<()> {
let plan = union_exec(vec![parquet_exec(); 5]);
- let expected = &[
- "UnionExec",
- // Expect no repartition of DataSourceExec
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Expect no repartition of DataSourceExec
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2041,15 +2187,15 @@ fn repartition_through_sort_preserving_merge() -> Result<()> {
.into();
let plan = sort_preserving_merge_exec(sort_key, parquet_exec());
- // need resort as the data was not sorted correctly
- let expected = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2068,24 +2214,25 @@ fn repartition_ignores_sort_preserving_merge() -> Result<()> {
parquet_exec_multiple_sorted(vec![sort_key]),
);
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
// Test: run EnforceDistribution, then EnforceSort
- //
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [c@2 ASC]
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
// should not sort (as the data was already sorted)
// should not repartition, since increased parallelism is not beneficial for SortPReservingMerge
- let expected = &[
- "SortPreservingMergeExec: [c@2 ASC]",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
Ok(())
}
@@ -2105,27 +2252,29 @@ fn repartition_ignores_sort_preserving_merge_with_union() -> Result<()> {
]);
let plan = sort_preserving_merge_exec(sort_key, input);
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
// Test: run EnforceDistribution, then EnforceSort.
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [c@2 ASC]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
//
// should not repartition / sort (as the data was already sorted)
- let expected = &[
- "SortPreservingMergeExec: [c@2 ASC]",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
// test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
Ok(())
}
@@ -2149,16 +2298,17 @@ fn repartition_does_not_destroy_sort() -> Result<()> {
// TestConfig: Prefer existing sort.
let test_config = TestConfig::default().with_prefer_existing_sort();
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortRequiredExec: [d@3 ASC]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet
+");
// during repartitioning ordering is preserved
- let expected = &[
- "SortRequiredExec: [d@3 ASC]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet",
- ];
-
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2187,22 +2337,25 @@ fn repartition_does_not_destroy_sort_more_complex() -> Result<()> {
let input2 = filter_exec(parquet_exec());
let plan = union_exec(vec![input1, input2]);
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+UnionExec
+ SortRequiredExec: [c@2 ASC]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // union input 1: no repartitioning
+ // union input 2: should repartition
+ //
// should not repartition below the SortRequired as that
// branch doesn't benefit from increased parallelism
- let expected = &[
- "UnionExec",
- // union input 1: no repartitioning
- " SortRequiredExec: [c@2 ASC]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- // union input 2: should repartition
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2227,28 +2380,28 @@ fn repartition_transitively_with_projection() -> Result<()> {
.into();
let plan = sort_preserving_merge_exec(sort_key, proj);
- // Test: run EnforceDistribution, then EnforceSort.
- let expected = &[
- "SortPreservingMergeExec: [sum@0 ASC]",
- " SortExec: expr=[sum@0 ASC], preserve_partitioning=[true]",
- // Since this projection is not trivial, increasing parallelism is beneficial
- " ProjectionExec: expr=[a@0 + b@1 as sum]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [sum@0 ASC]
+ SortExec: expr=[sum@0 ASC], preserve_partitioning=[true]
+ ProjectionExec: expr=[a@0 + b@1 as sum]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[sum@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- // Since this projection is not trivial, increasing parallelism is beneficial
- " ProjectionExec: expr=[a@0 + b@1 as sum]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[sum@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[a@0 + b@1 as sum]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Since this projection is not trivial, increasing parallelism is beneficial
Ok(())
}
@@ -2275,16 +2428,18 @@ fn repartition_ignores_transitively_with_projection() -> Result<()> {
sort_key,
);
- let expected = &[
- "SortRequiredExec: [c@2 ASC]",
- // Since this projection is trivial, increasing parallelism is not beneficial
- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortRequiredExec: [c@2 ASC]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ // Since this projection is trivial, increasing parallelism is not beneficial
+
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2310,16 +2465,17 @@ fn repartition_transitively_past_sort_with_projection() -> Result<()> {
),
);
- let expected = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // Since this projection is trivial, increasing parallelism is not beneficial
- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
-
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Since this projection is trivial, increasing parallelism is not beneficial
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -2334,28 +2490,30 @@ fn repartition_transitively_past_sort_with_filter() -> Result<()> {
.into();
let plan = sort_exec(sort_key, filter_exec(parquet_exec()));
- // Test: run EnforceDistribution, then EnforceSort.
- let expected = &[
- "SortPreservingMergeExec: [a@0 ASC]",
- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",
- // Expect repartition on the input to the sort (as it can benefit from additional parallelism)
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [a@0 ASC]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+
+ // Expect repartition on the input to the sort (as it can benefit from additional parallelism)
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " FilterExec: c@2 = 0",
- // Expect repartition on the input of the filter (as it can benefit from additional parallelism)
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Expect repartition on the input of the filter (as it can benefit from additional parallelism)
Ok(())
}
@@ -2381,30 +2539,32 @@ fn repartition_transitively_past_sort_with_projection_and_filter() -> Result<()>
),
);
- // Test: run EnforceDistribution, then EnforceSort.
- let expected = &[
- "SortPreservingMergeExec: [a@0 ASC]",
- // Expect repartition on the input to the sort (as it can benefit from additional parallelism)
- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",
- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",
- " FilterExec: c@2 = 0",
- // repartition is lowest down
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, plan.clone(), &DISTRIB_DISTRIB_SORT)?;
+ let plan_distrib = test_config.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [a@0 ASC]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+
+ // Expect repartition on the input to the sort (as it can benefit from additional parallelism)
+ // repartition is lowest down
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected_first_sort_enforcement, plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
Ok(())
}
@@ -2420,28 +2580,29 @@ fn parallelization_single_partition() -> Result<()> {
.with_query_execution_partitions(2);
// Test: with parquet
- let expected_parquet = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- &expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={2 groups: [[x:0..50], [x:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2465,40 +2626,31 @@ fn parallelization_multiple_files() -> Result<()> {
// The groups must have only contiguous ranges of rows from the same file
// if any group has rows from multiple files, the data is no longer sorted destroyed
// https://github.com/apache/datafusion/issues/8451
- let expected_with_3_target_partitions = [
- "SortRequiredExec: [a@0 ASC]",
- " FilterExec: c@2 = 0",
- " DataSourceExec: file_groups={3 groups: [[x:0..50], [y:0..100], [x:50..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
let test_config_concurrency_3 =
test_config.clone().with_query_execution_partitions(3);
- test_config_concurrency_3.run(
- &expected_with_3_target_partitions,
- plan.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config_concurrency_3.run(
- &expected_with_3_target_partitions,
- plan.clone(),
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_3_distrib =
+ test_config_concurrency_3.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_3_distrib,
+ @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ DataSourceExec: file_groups={3 groups: [[x:0..50], [y:0..100], [x:50..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
+ let plan_3_sort =
+ test_config_concurrency_3.to_plan(plan.clone(), &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_3_distrib, plan_3_sort);
- let expected_with_8_target_partitions = [
- "SortRequiredExec: [a@0 ASC]",
- " FilterExec: c@2 = 0",
- " DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
let test_config_concurrency_8 = test_config.with_query_execution_partitions(8);
- test_config_concurrency_8.run(
- &expected_with_8_target_partitions,
- plan.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config_concurrency_8.run(
- &expected_with_8_target_partitions,
- plan,
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_8_distrib =
+ test_config_concurrency_8.to_plan(plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_8_distrib,
+ @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ DataSourceExec: file_groups={8 groups: [[x:0..25], [y:0..25], [x:25..50], [y:25..50], [x:50..75], [y:50..75], [x:75..100], [y:75..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
+ let plan_8_sort = test_config_concurrency_8.to_plan(plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_8_distrib, plan_8_sort);
Ok(())
}
@@ -2570,30 +2722,30 @@ fn parallelization_two_partitions() -> Result<()> {
.with_prefer_repartition_file_scans(10);
// Test: with parquet
- let expected_parquet = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- // Plan already has two partitions
- " DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- &expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Plan already has two partitions
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- // Plan already has two partitions
- " DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib, @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 2), input_partitions=2
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={2 groups: [[x:0..100], [y:0..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ // Plan already has two partitions
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2609,30 +2761,32 @@ fn parallelization_two_partitions_into_four() -> Result<()> {
.with_prefer_repartition_file_scans(10);
// Test: with parquet
- let expected_parquet = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- // Multiple source files split across partitions
- " DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- &expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(&expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ // Multiple source files split across partitions
+ assert_plan!(plan_parquet_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Multiple source files split across partitions
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = [
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- // Multiple source files split across partitions
- " DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(&expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(&expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ // Multiple source files split across partitions
+ assert_plan!(plan_csv_distrib, @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={4 groups: [[x:0..50], [x:50..100], [y:0..50], [y:50..100]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ // Multiple source files split across partitions
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2651,32 +2805,32 @@ fn parallelization_sorted_limit() -> Result<()> {
let test_config = TestConfig::default();
// Test: with parquet
- let expected_parquet = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " LocalLimitExec: fetch=100",
- // data is sorted so can't repartition here
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // Doesn't parallelize for SortExec without preserve_partitioning
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib, @r"
+GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // data is sorted so can't repartition here
+ // Doesn't parallelize for SortExec without preserve_partitioning
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " LocalLimitExec: fetch=100",
- // data is sorted so can't repartition here
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // Doesn't parallelize for SortExec without preserve_partitioning
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ // data is sorted so can't repartition here
+ // Doesn't parallelize for SortExec without preserve_partitioning
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2696,40 +2850,41 @@ fn parallelization_limit_with_filter() -> Result<()> {
let test_config = TestConfig::default();
// Test: with parquet
- let expected_parquet = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // even though data is sorted, we can use repartition here. Since
- // ordering is not used in subsequent stages anyway.
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // SortExec doesn't benefit from input partitioning
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ // even though data is sorted, we can use repartition here. Since
+ // ordering is not used in subsequent stages anyway.
+ // SortExec doesn't benefit from input partitioning
+ assert_plan!(plan_parquet_distrib,
+ @r"
+GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // even though data is sorted, we can use repartition here. Since
- // ordering is not used in subsequent stages anyway.
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // SortExec doesn't benefit from input partitioning
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ // even though data is sorted, we can use repartition here. Since
+ // ordering is not used in subsequent stages anyway.
+ // SortExec doesn't benefit from input partitioning
+ assert_plan!(plan_csv_distrib,
+ @r"
+GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2747,48 +2902,49 @@ fn parallelization_ignores_limit() -> Result<()> {
let test_config = TestConfig::default();
// Test: with parquet
- let expected_parquet = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // repartition should happen prior to the filter to maximize parallelism
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- // Limit doesn't benefit from input partitioning - no parallelism
- " LocalLimitExec: fetch=100",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ ");
+ // repartition should happen prior to the filter to maximize parallelism
+ // Limit doesn't benefit from input partitioning - no parallelism
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- " CoalescePartitionsExec",
- " LocalLimitExec: fetch=100",
- " FilterExec: c@2 = 0",
- // repartition should happen prior to the filter to maximize parallelism
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " GlobalLimitExec: skip=0, fetch=100",
- // Limit doesn't benefit from input partitioning - no parallelism
- " LocalLimitExec: fetch=100",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 10), input_partitions=10
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ CoalescePartitionsExec
+ LocalLimitExec: fetch=100
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ GlobalLimitExec: skip=0, fetch=100
+ LocalLimitExec: fetch=100
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ ");
+ // repartition should happen prior to the filter to maximize parallelism
+ // Limit doesn't benefit from input partitioning - no parallelism
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2801,34 +2957,35 @@ fn parallelization_union_inputs() -> Result<()> {
let test_config = TestConfig::default();
// Test: with parquet
- let expected_parquet = &[
- "UnionExec",
- // Union doesn't benefit from input partitioning - no parallelism
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ // Union doesn't benefit from input partitioning - no parallelism
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "UnionExec",
- // Union doesn't benefit from input partitioning - no parallelism
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false
+");
+ // Union doesn't benefit from input partitioning - no parallelism
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2855,22 +3012,21 @@ fn parallelization_prior_to_sort_preserving_merge() -> Result<()> {
// parallelization is not beneficial for SortPreservingMerge
// Test: with parquet
- let expected_parquet = &[
- "DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet"
+ );
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @"DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false"
+ );
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -2900,54 +3056,47 @@ fn parallelization_sort_preserving_merge_with_union() -> Result<()> {
// should not sort (as the data was already sorted)
// Test: with parquet
- let expected_parquet = &[
- "SortPreservingMergeExec: [c@2 ASC]",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- let expected_parquet_first_sort_enforcement = &[
- // no SPM
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // has coalesce
- " CoalescePartitionsExec",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_parquet_first_sort_enforcement,
- plan_parquet,
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+ SortPreservingMergeExec: [c@2 ASC]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ ");
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_sort,
+ @r"
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ ");
+ // no SPM
+ // has coalesce
// Test: with csv
- let expected_csv = &[
- "SortPreservingMergeExec: [c@2 ASC]",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- let expected_csv_first_sort_enforcement = &[
- // no SPM
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- // has coalesce
- " CoalescePartitionsExec",
- " UnionExec",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- test_config.run(
- expected_csv_first_sort_enforcement,
- plan_csv.clone(),
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+ SortPreservingMergeExec: [c@2 ASC]
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+ ");
+ let plan_csv_sort = test_config.to_plan(plan_csv.clone(), &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_sort,
+ @r"
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ UnionExec
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+ ");
+ // no SPM
+ // has coalesce
Ok(())
}
@@ -2975,24 +3124,25 @@ fn parallelization_does_not_benefit() -> Result<()> {
// no parallelization, because SortRequiredExec doesn't benefit from increased parallelism
// Test: with parquet
- let expected_parquet = &[
- "SortRequiredExec: [c@2 ASC]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_parquet_distrib,
+ @r"
+ SortRequiredExec: [c@2 ASC]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ ");
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
// Test: with csv
- let expected_csv = &[
- "SortRequiredExec: [c@2 ASC]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_csv_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_csv_distrib,
+ @r"
+ SortRequiredExec: [c@2 ASC]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+ ");
+ let plan_csv_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_csv_distrib, plan_csv_sort);
Ok(())
}
@@ -3023,26 +3173,26 @@ fn parallelization_ignores_transitively_with_projection_parquet() -> Result<()>
.into();
let plan_parquet =
sort_preserving_merge_exec(sort_key_after_projection, proj_parquet);
- let expected = &[
- "SortPreservingMergeExec: [c2@1 ASC]",
- " ProjectionExec: expr=[a@0 as a2, c@2 as c2]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- plans_matches_expected!(expected, &plan_parquet);
+ assert_plan!(plan_parquet,
+ @r"
+ SortPreservingMergeExec: [c2@1 ASC]
+ ProjectionExec: expr=[a@0 as a2, c@2 as c2]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+ ");
+
+ let test_config = TestConfig::default();
+ let plan_parquet_distrib =
+ test_config.to_plan(plan_parquet.clone(), &DISTRIB_DISTRIB_SORT);
// Expected Outcome:
// data should not be repartitioned / resorted
- let expected_parquet = &[
- "ProjectionExec: expr=[a@0 as a2, c@2 as c2]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- let test_config = TestConfig::default();
- test_config.run(
- expected_parquet,
- plan_parquet.clone(),
- &DISTRIB_DISTRIB_SORT,
- )?;
- test_config.run(expected_parquet, plan_parquet, &SORT_DISTRIB_DISTRIB)?;
+ assert_plan!(plan_parquet_distrib,
+ @r"
+ProjectionExec: expr=[a@0 as a2, c@2 as c2]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ let plan_parquet_sort = test_config.to_plan(plan_parquet, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_parquet_distrib, plan_parquet_sort);
Ok(())
}
@@ -3071,22 +3221,24 @@ fn parallelization_ignores_transitively_with_projection_csv() -> Result<()> {
}]
.into();
let plan_csv = sort_preserving_merge_exec(sort_key_after_projection, proj_csv);
- let expected = &[
- "SortPreservingMergeExec: [c2@1 ASC]",
- " ProjectionExec: expr=[a@0 as a2, c@2 as c2]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- plans_matches_expected!(expected, &plan_csv);
+ assert_plan!(plan_csv,
+ @r"
+SortPreservingMergeExec: [c2@1 ASC]
+ ProjectionExec: expr=[a@0 as a2, c@2 as c2]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+");
+ let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(plan_csv.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+ProjectionExec: expr=[a@0 as a2, c@2 as c2]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false
+");
// Expected Outcome:
// data should not be repartitioned / resorted
- let expected_csv = &[
- "ProjectionExec: expr=[a@0 as a2, c@2 as c2]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=csv, has_header=false",
- ];
- let test_config = TestConfig::default();
- test_config.run(expected_csv, plan_csv.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected_csv, plan_csv, &SORT_DISTRIB_DISTRIB)?;
+ let plan_sort = test_config.to_plan(plan_csv, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3096,24 +3248,25 @@ fn remove_redundant_roundrobins() -> Result<()> {
let input = parquet_exec();
let repartition = repartition_exec(repartition_exec(input));
let physical_plan = repartition_exec(filter_exec(repartition));
- let expected = &[
- "RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- plans_matches_expected!(expected, &physical_plan);
-
- let expected = &[
- "FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
+ assert_plan!(physical_plan,
+ @r"
+RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=10
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
let test_config = TestConfig::default();
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3133,18 +3286,19 @@ fn remove_unnecessary_spm_after_filter() -> Result<()> {
// TestConfig: Prefer existing sort.
let test_config = TestConfig::default().with_prefer_existing_sort();
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
// Expected Outcome:
// Original plan expects its output to be ordered by c@2 ASC.
// This is still satisfied since, after filter that column is constant.
- let expected = &[
- "CoalescePartitionsExec",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=c@2 ASC",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
-
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ assert_plan!(plan_distrib,
+ @r"
+CoalescePartitionsExec
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=c@2 ASC
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3164,14 +3318,16 @@ fn preserve_ordering_through_repartition() -> Result<()> {
// TestConfig: Prefer existing sort.
let test_config = TestConfig::default().with_prefer_existing_sort();
- let expected = &[
- "SortPreservingMergeExec: [d@3 ASC]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=d@3 ASC",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [d@3 ASC]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2, preserve_order=true, sort_exprs=d@3 ASC
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[d@3 ASC], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3189,29 +3345,27 @@ fn do_not_preserve_ordering_through_repartition() -> Result<()> {
let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
// Test: run EnforceDistribution, then EnforceSort.
- let expected = &[
- "SortPreservingMergeExec: [a@0 ASC]",
- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [a@0 ASC]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_first_sort_enforcement,
- physical_plan,
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
Ok(())
}
@@ -3227,17 +3381,18 @@ fn no_need_for_sort_after_filter() -> Result<()> {
let input = parquet_exec_multiple_sorted(vec![sort_key.clone()]);
let physical_plan = sort_preserving_merge_exec(sort_key, filter_exec(input));
- let expected = &[
- // After CoalescePartitionsExec c is still constant. Hence c@2 ASC ordering is already satisfied.
- "CoalescePartitionsExec",
- // Since after this stage c is constant. c@2 ASC ordering is already satisfied.
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib, @r"
+CoalescePartitionsExec
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
+ // After CoalescePartitionsExec c is still constant. Hence c@2 ASC ordering is already satisfied.
+ // Since after this stage c is constant. c@2 ASC ordering is already satisfied.
Ok(())
}
@@ -3261,30 +3416,28 @@ fn do_not_preserve_ordering_through_repartition2() -> Result<()> {
let test_config = TestConfig::default();
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
// Test: run EnforceDistribution, then EnforceSort.
- let expected = &[
- "SortPreservingMergeExec: [a@0 ASC]",
- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
+ assert_plan!(plan_distrib,
+ @r"
+SortPreservingMergeExec: [a@0 ASC]
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
// Test: result IS DIFFERENT, if EnforceSorting is run first:
- let expected_first_sort_enforcement = &[
- "SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " SortExec: expr=[a@0 ASC], preserve_partitioning=[true]",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(
- expected_first_sort_enforcement,
- physical_plan,
- &SORT_DISTRIB_DISTRIB,
- )?;
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_sort,
+ @r"
+SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ SortExec: expr=[a@0 ASC], preserve_partitioning=[true]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
Ok(())
}
@@ -3300,14 +3453,16 @@ fn do_not_preserve_ordering_through_repartition3() -> Result<()> {
let input = parquet_exec_multiple_sorted(vec![sort_key]);
let physical_plan = filter_exec(input);
- let expected = &[
- "FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3322,30 +3477,27 @@ fn do_not_put_sort_when_input_is_invalid() -> Result<()> {
.into();
let input = parquet_exec();
let physical_plan = sort_required_exec_with_req(filter_exec(input), sort_key);
- let expected = &[
- // Ordering requirement of sort required exec is NOT satisfied
- // by existing ordering at the source.
- "SortRequiredExec: [a@0 ASC]",
- " FilterExec: c@2 = 0",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- assert_plan_txt!(expected, physical_plan);
-
- let expected = &[
- "SortRequiredExec: [a@0 ASC]",
- // Since at the start of the rule ordering requirement is not satisfied
- // EnforceDistribution rule doesn't satisfy this requirement either.
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
+ // Ordering requirement of sort required exec is NOT satisfied
+ // by existing ordering at the source.
+ assert_plan!(physical_plan, @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
let mut config = ConfigOptions::new();
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_existing_sort = false;
let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;
- assert_plan_txt!(expected, dist_plan);
+ // Since at the start of the rule ordering requirement is not satisfied
+ // EnforceDistribution rule doesn't satisfy this requirement either.
+ assert_plan!(dist_plan, @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
Ok(())
}
@@ -3361,29 +3513,26 @@ fn put_sort_when_input_is_valid() -> Result<()> {
let input = parquet_exec_multiple_sorted(vec![sort_key.clone()]);
let physical_plan = sort_required_exec_with_req(filter_exec(input), sort_key);
- let expected = &[
- // Ordering requirement of sort required exec is satisfied
- // by existing ordering at the source.
- "SortRequiredExec: [a@0 ASC]",
- " FilterExec: c@2 = 0",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
- assert_plan_txt!(expected, physical_plan);
-
- let expected = &[
- // Since at the start of the rule ordering requirement is satisfied
- // EnforceDistribution rule satisfy this requirement also.
- "SortRequiredExec: [a@0 ASC]",
- " FilterExec: c@2 = 0",
- " DataSourceExec: file_groups={10 groups: [[x:0..20], [y:0..20], [x:20..40], [y:20..40], [x:40..60], [y:40..60], [x:60..80], [y:60..80], [x:80..100], [y:80..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet",
- ];
+ // Ordering requirement of sort required exec is satisfied
+ // by existing ordering at the source.
+ assert_plan!(physical_plan, @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
let mut config = ConfigOptions::new();
config.execution.target_partitions = 10;
config.optimizer.enable_round_robin_repartition = true;
config.optimizer.prefer_existing_sort = false;
let dist_plan = EnforceDistribution::new().optimize(physical_plan, &config)?;
- assert_plan_txt!(expected, dist_plan);
+ // Since at the start of the rule ordering requirement is satisfied
+ // EnforceDistribution rule satisfy this requirement also.
+ assert_plan!(dist_plan, @r"
+SortRequiredExec: [a@0 ASC]
+ FilterExec: c@2 = 0
+ DataSourceExec: file_groups={10 groups: [[x:0..20], [y:0..20], [x:20..40], [y:20..40], [x:40..60], [y:40..60], [x:60..80], [y:60..80], [x:80..100], [y:80..100]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
+");
Ok(())
}
@@ -3404,13 +3553,15 @@ fn do_not_add_unnecessary_hash() -> Result<()> {
// Make sure target partition number is 1. In this case hash repartition is unnecessary.
let test_config = TestConfig::default().with_query_execution_partitions(1);
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3432,19 +3583,21 @@ fn do_not_add_unnecessary_hash2() -> Result<()> {
// Make sure target partition number is larger than 2 (e.g partition number at the source).
let test_config = TestConfig::default().with_query_execution_partitions(4);
- let expected = &[
- "AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- // Since hash requirements of this operator is satisfied. There shouldn't be
- // a hash repartition here
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4",
- " AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]",
- " RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2",
- " DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=4
+ AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[]
+ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2
+ DataSourceExec: file_groups={2 groups: [[x], [y]]}, projection=[a, b, c, d, e], output_ordering=[c@2 ASC], file_type=parquet
+");
+ // Since hash requirements of this operator is satisfied. There shouldn't be
+ // a hash repartition here
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3452,19 +3605,21 @@ fn do_not_add_unnecessary_hash2() -> Result<()> {
#[test]
fn optimize_away_unnecessary_repartition() -> Result<()> {
let physical_plan = coalesce_partitions_exec(repartition_exec(parquet_exec()));
- let expected = &[
- "CoalescePartitionsExec",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- plans_matches_expected!(expected, physical_plan.clone());
-
- let expected =
- &["DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet"];
+ assert_plan!(physical_plan,
+ @r"
+CoalescePartitionsExec
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
let test_config = TestConfig::default();
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3474,25 +3629,27 @@ fn optimize_away_unnecessary_repartition2() -> Result<()> {
let physical_plan = filter_exec(repartition_exec(coalesce_partitions_exec(
filter_exec(repartition_exec(parquet_exec())),
)));
- let expected = &[
- "FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " CoalescePartitionsExec",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- plans_matches_expected!(expected, physical_plan.clone());
+ assert_plan!(physical_plan,
+ @r"
+FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ CoalescePartitionsExec
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
- let expected = &[
- "FilterExec: c@2 = 0",
- " FilterExec: c@2 = 0",
- " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
let test_config = TestConfig::default();
- test_config.run(expected, physical_plan.clone(), &DISTRIB_DISTRIB_SORT)?;
- test_config.run(expected, physical_plan, &SORT_DISTRIB_DISTRIB)?;
+ let plan_distrib = test_config.to_plan(physical_plan.clone(), &DISTRIB_DISTRIB_SORT);
+ assert_plan!(plan_distrib,
+ @r"
+FilterExec: c@2 = 0
+ FilterExec: c@2 = 0
+ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
+ let plan_sort = test_config.to_plan(physical_plan, &SORT_DISTRIB_DISTRIB);
+ assert_plan!(plan_distrib, plan_sort);
Ok(())
}
@@ -3512,27 +3669,31 @@ async fn test_distribute_sort_parquet() -> Result<()> {
let physical_plan = sort_exec(sort_key, parquet_exec_with_stats(10000 * 8192));
// prior to optimization, this is the starting plan
- let starting = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- plans_matches_expected!(starting, physical_plan.clone());
+ assert_plan!(physical_plan,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=parquet
+");
// what the enforce distribution run does.
- let expected = &[
- "SortExec: expr=[c@2 ASC], preserve_partitioning=[false]",
- " CoalescePartitionsExec",
- " DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected, physical_plan.clone(), &[Run::Distribution])?;
+ let plan_distribution =
+ test_config.to_plan(physical_plan.clone(), &[Run::Distribution]);
+ assert_plan!(plan_distribution,
+ @r"
+SortExec: expr=[c@2 ASC], preserve_partitioning=[false]
+ CoalescePartitionsExec
+ DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet
+");
// what the sort parallelization (in enforce sorting), does after the enforce distribution changes
- let expected = &[
- "SortPreservingMergeExec: [c@2 ASC]",
- " SortExec: expr=[c@2 ASC], preserve_partitioning=[true]",
- " DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet",
- ];
- test_config.run(expected, physical_plan, &[Run::Distribution, Run::Sorting])?;
+ let plan_both =
+ test_config.to_plan(physical_plan, &[Run::Distribution, Run::Sorting]);
+ assert_plan!(plan_both,
+ @r"
+SortPreservingMergeExec: [c@2 ASC]
+ SortExec: expr=[c@2 ASC], preserve_partitioning=[true]
+ DataSourceExec: file_groups={10 groups: [[x:0..8192000], [x:8192000..16384000], [x:16384000..24576000], [x:24576000..32768000], [x:32768000..40960000], [x:40960000..49152000], [x:49152000..57344000], [x:57344000..65536000], [x:65536000..73728000], [x:73728000..81920000]]}, projection=[a, b, c, d, e], file_type=parquet
+");
Ok(())
}
@@ -3557,12 +3718,12 @@ async fn test_distribute_sort_memtable() -> Result<()> {
let physical_plan = dataframe.create_physical_plan().await?;
// this is the final, optimized plan
- let expected = &[
- "SortPreservingMergeExec: [id@0 ASC NULLS LAST]",
- " SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]",
- " DataSourceExec: partitions=3, partition_sizes=[34, 33, 33]",
- ];
- plans_matches_expected!(expected, physical_plan);
+ assert_plan!(physical_plan,
+ @r"
+SortPreservingMergeExec: [id@0 ASC NULLS LAST]
+ SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]
+ DataSourceExec: partitions=3, partition_sizes=[34, 33, 33]
+");
Ok(())
}