Apache DataFusion SQL Query Engine
Eliminate redundant `ProjectionExec`s (#21333)
Daniël Heresc17c87c575be
diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rsindex 80bbde1f6b..1c661744e0 100644--- a/datafusion/core/tests/dataframe/mod.rs+++ b/datafusion/core/tests/dataframe/mod.rs@@ -3429,31 +3429,30 @@ async fn test_count_wildcard_on_where_scalar_subquery() -> Result<()> {assert_snapshot!(pretty_format_batches(&sql_results).unwrap(),@r"- +---------------+----------------------------------------------------------------------------------------------------------------------------+- | plan_type | plan |- +---------------+----------------------------------------------------------------------------------------------------------------------------+- | logical_plan | Projection: t1.a, t1.b |- | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |- | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |- | | Left Join: t1.a = __scalar_sq_1.a |- | | TableScan: t1 projection=[a, b] |- | | SubqueryAlias: __scalar_sq_1 |- | | Projection: count(Int64(1)) AS count(*), t2.a, Boolean(true) AS __always_true |- | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1))]] |- | | TableScan: t2 projection=[a] |- | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |- | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |- | | ProjectionExec: expr=[a@2 as a, b@3 as b, count(*)@0 as count(*), __always_true@1 as __always_true] |- | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[count(*)@0, __always_true@2, a@3, b@4] |- | | CoalescePartitionsExec |- | | ProjectionExec: expr=[count(Int64(1))@1 as count(*), a@0 as a, true as __always_true] |- | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(Int64(1))] |- | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |- | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(Int64(1))] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | |- +---------------+----------------------------------------------------------------------------------------------------------------------------++ +---------------+--------------------------------------------------------------------------------------------------------------------------++ | plan_type | plan |+ +---------------+--------------------------------------------------------------------------------------------------------------------------++ | logical_plan | Projection: t1.a, t1.b |+ | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |+ | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |+ | | Left Join: t1.a = __scalar_sq_1.a |+ | | TableScan: t1 projection=[a, b] |+ | | SubqueryAlias: __scalar_sq_1 |+ | | Projection: count(Int64(1)) AS count(*), t2.a, Boolean(true) AS __always_true |+ | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1))]] |+ | | TableScan: t2 projection=[a] |+ | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |+ | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |+ | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[a@3, b@4, count(*)@0, __always_true@2] |+ | | CoalescePartitionsExec |+ | | ProjectionExec: expr=[count(Int64(1))@1 as count(*), a@0 as a, true as __always_true] |+ | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(Int64(1))] |+ | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |+ | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(Int64(1))] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | |+ +---------------+--------------------------------------------------------------------------------------------------------------------------+");@@ -3485,31 +3484,30 @@ async fn test_count_wildcard_on_where_scalar_subquery() -> Result<()> {assert_snapshot!(pretty_format_batches(&df_results).unwrap(),@r"- +---------------+----------------------------------------------------------------------------------------------------------------------------+- | plan_type | plan |- +---------------+----------------------------------------------------------------------------------------------------------------------------+- | logical_plan | Projection: t1.a, t1.b |- | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |- | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |- | | Left Join: t1.a = __scalar_sq_1.a |- | | TableScan: t1 projection=[a, b] |- | | SubqueryAlias: __scalar_sq_1 |- | | Projection: count(*), t2.a, Boolean(true) AS __always_true |- | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1)) AS count(*)]] |- | | TableScan: t2 projection=[a] |- | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |- | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |- | | ProjectionExec: expr=[a@2 as a, b@3 as b, count(*)@0 as count(*), __always_true@1 as __always_true] |- | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[count(*)@0, __always_true@2, a@3, b@4] |- | | CoalescePartitionsExec |- | | ProjectionExec: expr=[count(*)@1 as count(*), a@0 as a, true as __always_true] |- | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(*)] |- | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |- | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(*)] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | DataSourceExec: partitions=1, partition_sizes=[1] |- | | |- +---------------+----------------------------------------------------------------------------------------------------------------------------++ +---------------+--------------------------------------------------------------------------------------------------------------------------++ | plan_type | plan |+ +---------------+--------------------------------------------------------------------------------------------------------------------------++ | logical_plan | Projection: t1.a, t1.b |+ | | Filter: CASE WHEN __scalar_sq_1.__always_true IS NULL THEN Int64(0) ELSE __scalar_sq_1.count(*) END > Int64(0) |+ | | Projection: t1.a, t1.b, __scalar_sq_1.count(*), __scalar_sq_1.__always_true |+ | | Left Join: t1.a = __scalar_sq_1.a |+ | | TableScan: t1 projection=[a, b] |+ | | SubqueryAlias: __scalar_sq_1 |+ | | Projection: count(*), t2.a, Boolean(true) AS __always_true |+ | | Aggregate: groupBy=[[t2.a]], aggr=[[count(Int64(1)) AS count(*)]] |+ | | TableScan: t2 projection=[a] |+ | physical_plan | FilterExec: CASE WHEN __always_true@3 IS NULL THEN 0 ELSE count(*)@2 END > 0, projection=[a@0, b@1] |+ | | RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 |+ | | HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@1, a@0)], projection=[a@3, b@4, count(*)@0, __always_true@2] |+ | | CoalescePartitionsExec |+ | | ProjectionExec: expr=[count(*)@1 as count(*), a@0 as a, true as __always_true] |+ | | AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[count(*)] |+ | | RepartitionExec: partitioning=Hash([a@0], 4), input_partitions=1 |+ | | AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[count(*)] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | DataSourceExec: partitions=1, partition_sizes=[1] |+ | | |+ +---------------+--------------------------------------------------------------------------------------------------------------------------+");diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rsindex a85ce8d080..95cd34b6e4 100644--- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs@@ -1306,8 +1306,8 @@ fn test_hash_join_after_projection() -> Result<()> {assert_snapshot!(actual,@r"- ProjectionExec: expr=[c@2 as c_from_left, b@1 as b_from_left, a@0 as a_from_left, c@3 as c_from_right]- HashJoinExec: mode=Auto, join_type=Inner, on=[(b@1, c@2)], filter=b_left_inter@0 - 1 + a_right_inter@1 <= a_right_inter@1 + c_left_inter@2, projection=[a@0, b@1, c@2, c@7]+ ProjectionExec: expr=[c@0 as c_from_left, b@1 as b_from_left, a@2 as a_from_left, c@3 as c_from_right]+ HashJoinExec: mode=Auto, join_type=Inner, on=[(b@1, c@2)], filter=b_left_inter@0 - 1 + a_right_inter@1 <= a_right_inter@1 + c_left_inter@2, projection=[c@2, b@1, a@0, c@7]DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=falseDataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], file_type=csv, has_header=false"diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex 9cc75a68fe..9cb4382e21 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -50,7 +50,6 @@ use datafusion_execution::TaskContext;use datafusion_expr::ExpressionPlacement;use datafusion_physical_expr::equivalence::ProjectionMapping;use datafusion_physical_expr::projection::Projector;-use datafusion_physical_expr::utils::collect_columns;use datafusion_physical_expr_common::physical_expr::{PhysicalExprRef, fmt_sql};use datafusion_physical_expr_common::sort_expr::{LexOrdering, LexRequirement, PhysicalSortExpr,@@ -602,13 +601,7 @@ pub fn try_embed_projection<Exec: EmbeddedProjection + 'static>(return Ok(None);};- // If the projection indices is the same as the input columns, we don't need to embed the projection to hash join.- // Check the projection_index is 0..n-1 and the length of projection_index is the same as the length of execution_plan schema fields.- if projection_index.len() == projection_index.last().unwrap() + 1- && projection_index.len() == execution_plan.schema().fields().len()- {- return Ok(None);- }+ let columns_reduced = projection_index.len() < execution_plan.schema().fields().len();let new_execution_plan =Arc::new(execution_plan.with_projection(Some(projection_index.to_vec()))?);@@ -643,9 +636,16 @@ pub fn try_embed_projection<Exec: EmbeddedProjection + 'static>(Arc::clone(&new_execution_plan) as _,)?);if is_projection_removable(&new_projection) {+ // Residual is identity — embedding fully absorbed the projection.Ok(Some(new_execution_plan))- } else {+ } else if columns_reduced {+ // Embedding reduced columns even though a residual is still needed+ // for renames or expressions — worth keeping.Ok(Some(new_projection))+ } else {+ // No columns eliminated and residual still needed — embedding just+ // adds an unnecessary column reorder inside the operator.+ Ok(None)}}@@ -1074,15 +1074,37 @@ fn try_unifying_projections(/// Collect all column indices from the given projection expressions.fn collect_column_indices(exprs: &[ProjectionExpr]) -> Vec<usize> {- // Collect indices and remove duplicates.- let mut indices = exprs- .iter()- .flat_map(|proj_expr| collect_columns(&proj_expr.expr))- .map(|x| x.index())- .collect::<std::collections::HashSet<_>>()- .into_iter()- .collect::<Vec<_>>();- indices.sort();+ // Collect column indices in a deterministic order that preserves the+ // projection's column ordering. For simple Column expressions, we use+ // the column index directly. For complex expressions, we walk the+ // expression tree to collect column references in traversal order.+ // This allows the embedded projection to match the desired output+ // column order, avoiding a residual ProjectionExec.+ let mut seen = std::collections::HashSet::new();+ let mut indices = Vec::new();+ for proj_expr in exprs {+ if let Some(col) = proj_expr.expr.as_any().downcast_ref::<Column>() {+ // Simple column reference: preserve projection order.+ if seen.insert(col.index()) {+ indices.push(col.index());+ }+ } else {+ // Complex expression: collect all referenced columns in+ // expression tree traversal order (deterministic) to preserve+ // the natural ordering of column references.+ proj_expr+ .expr+ .apply(|expr| {+ if let Some(col) = expr.as_any().downcast_ref::<Column>()+ && seen.insert(col.index())+ {+ indices.push(col.index());+ }+ Ok(TreeNodeRecursion::Continue)+ })+ .expect("closure always returns OK");+ }+ }indices}@@ -1196,7 +1218,8 @@ mod tests {expr,alias: "b-(1+a)".to_string(),}]);- assert_eq!(column_indices, vec![1, 7]);+ // Tree traversal order: b@7 is visited before a@1+ assert_eq!(column_indices, vec![7, 1]);Ok(())}diff --git a/datafusion/sqllogictest/test_files/clickbench.slt b/datafusion/sqllogictest/test_files/clickbench.sltindex 4e9849e365..314c3f9736 100644--- a/datafusion/sqllogictest/test_files/clickbench.slt+++ b/datafusion/sqllogictest/test_files/clickbench.slt@@ -673,10 +673,9 @@ physical_plan01)ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase]02)--SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST], fetch=1003)----SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST], preserve_partitioning=[true]-04)------ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime]-05)--------FilterExec: SearchPhrase@1 !=-06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]+04)------FilterExec: SearchPhrase@1 != , projection=[SearchPhrase@1, EventTime@0]+05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]query TSELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10;@@ -717,10 +716,9 @@ physical_plan01)ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase]02)--SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=1003)----SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true]-04)------ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime]-05)--------FilterExec: SearchPhrase@1 !=-06)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-07)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]+04)------FilterExec: SearchPhrase@1 != , projection=[SearchPhrase@1, EventTime@0]+05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()]query TSELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10;@@ -1110,8 +1108,8 @@ physical_plan05)--------AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))]06)----------RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 4), input_partitions=407)------------AggregateExec: mode=Partial, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))]-08)--------------ProjectionExec: expr=[URLHash@1 as URLHash, CAST(CAST(EventDate@0 AS Int32) AS Date32) as EventDate]-09)----------------FilterExec: CounterID@1 = 62 AND CAST(CAST(EventDate@0 AS Int32) AS Date32) >= 2013-07-01 AND CAST(CAST(EventDate@0 AS Int32) AS Date32) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5]+08)--------------ProjectionExec: expr=[URLHash@0 as URLHash, CAST(CAST(EventDate@1 AS Int32) AS Date32) as EventDate]+09)----------------FilterExec: CounterID@1 = 62 AND CAST(CAST(EventDate@0 AS Int32) AS Date32) >= 2013-07-01 AND CAST(CAST(EventDate@0 AS Int32) AS Date32) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[URLHash@5, EventDate@0]10)------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=111)--------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/clickbench_hits_10.parquet]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(CAST(EventDate@5 AS Int32) AS Date32) >= 2013-07-01 AND CAST(CAST(EventDate@5 AS Int32) AS Date32) <= 2013-07-31 AND IsRefresh@15 = 0 AND (TraficSourceID@37 = -1 OR TraficSourceID@37 = 6) AND RefererHash@102 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND EventDate_null_count@5 != row_count@3 AND CAST(CAST(EventDate_max@4 AS Int32) AS Date32) >= 2013-07-01 AND EventDate_null_count@5 != row_count@3 AND CAST(CAST(EventDate_min@6 AS Int32) AS Date32) <= 2013-07-31 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND (TraficSourceID_null_count@12 != row_count@3 AND TraficSourceID_min@10 <= -1 AND -1 <= TraficSourceID_max@11 OR TraficSourceID_null_count@12 != row_count@3 AND TraficSourceID_min@10 <= 6 AND 6 <= TraficSourceID_max@11) AND RefererHash_null_count@15 != row_count@3 AND RefererHash_min@13 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@14, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)]diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.sltindex 3d0b50b6a6..0a30118493 100644--- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt+++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt@@ -155,10 +155,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]# Disable Join dynamic filter pushdownstatement ok@@ -178,10 +177,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet# Re-enable for next testsstatement ok@@ -203,10 +201,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet+01)HashJoinExec: mode=CollectLeft, join_type=Right, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet# LEFT JOIN correctness: all left rows appear, unmatched right rows produce NULLsquery ITT@@ -236,10 +233,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]# RIGHT JOIN correctness: all right rows appear, unmatched left rows produce NULLsquery ITT@@ -468,10 +464,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]# Enable TopK, disable Joinstatement ok@@ -494,10 +489,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet# Test 4: Aggregate dynamic filter pushdown@@ -643,10 +637,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet# Re-enablestatement ok@@ -691,10 +684,9 @@ logical_plan05)----SubqueryAlias: r06)------TableScan: right_parquet projection=[id, info]physical_plan-01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3]-03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet-04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[id@2, data@3, info@1]+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet+03)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ]# Test 6: Regression test for issue #20213 - dynamic filter applied to wrong table# when subquery join has same column names on both sides.diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.sltindex 630fdc4b14..46d01f39a9 100644--- a/datafusion/sqllogictest/test_files/explain_tree.slt+++ b/datafusion/sqllogictest/test_files/explain_tree.slt@@ -284,38 +284,30 @@ ON----physical_plan01)┌───────────────────────────┐-02)│ ProjectionExec │+02)│ HashJoinExec │03)│ -------------------- │-04)│ date_col: date_col │-05)│ │-06)│ string_col: │-07)│ string_col │-08)└─────────────┬─────────────┘-09)┌─────────────┴─────────────┐-10)│ HashJoinExec │-11)│ -------------------- │-12)│ filter: │-13)│ CAST(int_col + int_col AS │-14)│ Int64) % 2 = 0 ├──────────────┐-15)│ │ │-16)│ on: │ │-17)│ (int_col = int_col) │ │-18)└─────────────┬─────────────┘ │-19)┌─────────────┴─────────────┐┌─────────────┴─────────────┐-20)│ DataSourceExec ││ RepartitionExec │-21)│ -------------------- ││ -------------------- │-22)│ files: 1 ││ partition_count(in->out): │-23)│ format: parquet ││ 1 -> 4 │-24)│ ││ │-25)│ ││ partitioning_scheme: │-26)│ ││ RoundRobinBatch(4) │-27)└───────────────────────────┘└─────────────┬─────────────┘-28)-----------------------------┌─────────────┴─────────────┐-29)-----------------------------│ DataSourceExec │-30)-----------------------------│ -------------------- │-31)-----------------------------│ files: 1 │-32)-----------------------------│ format: csv │-33)-----------------------------└───────────────────────────┘+04)│ filter: │+05)│ CAST(int_col + int_col AS │+06)│ Int64) % 2 = 0 ├──────────────┐+07)│ │ │+08)│ on: │ │+09)│ (int_col = int_col) │ │+10)└─────────────┬─────────────┘ │+11)┌─────────────┴─────────────┐┌─────────────┴─────────────┐+12)│ DataSourceExec ││ RepartitionExec │+13)│ -------------------- ││ -------------------- │+14)│ files: 1 ││ partition_count(in->out): │+15)│ format: parquet ││ 1 -> 4 │+16)│ ││ │+17)│ ││ partitioning_scheme: │+18)│ ││ RoundRobinBatch(4) │+19)└───────────────────────────┘└─────────────┬─────────────┘+20)-----------------------------┌─────────────┴─────────────┐+21)-----------------------------│ DataSourceExec │+22)-----------------------------│ -------------------- │+23)-----------------------------│ files: 1 │+24)-----------------------------│ format: csv │+25)-----------------------------└───────────────────────────┘# 3 Joinsquery TT@@ -329,49 +321,33 @@ FROM----physical_plan01)┌───────────────────────────┐-02)│ ProjectionExec │+02)│ HashJoinExec │03)│ -------------------- │-04)│ date_col: date_col │-05)│ │-06)│ string_col: │-07)│ string_col │-08)└─────────────┬─────────────┘-09)┌─────────────┴─────────────┐-10)│ HashJoinExec │-11)│ -------------------- │-12)│ on: ├──────────────┐-13)│ (int_col = int_col) │ │-14)└─────────────┬─────────────┘ │-15)┌─────────────┴─────────────┐┌─────────────┴─────────────┐-16)│ DataSourceExec ││ ProjectionExec │-17)│ -------------------- ││ -------------------- │-18)│ bytes: 512 ││ date_col: date_col │-19)│ format: memory ││ int_col: int_col │-20)│ rows: 1 ││ │-21)│ ││ string_col: │-22)│ ││ string_col │-23)└───────────────────────────┘└─────────────┬─────────────┘-24)-----------------------------┌─────────────┴─────────────┐-25)-----------------------------│ HashJoinExec │-26)-----------------------------│ -------------------- │-27)-----------------------------│ on: ├──────────────┐-28)-----------------------------│ (int_col = int_col) │ │-29)-----------------------------└─────────────┬─────────────┘ │-30)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐-31)-----------------------------│ DataSourceExec ││ RepartitionExec │-32)-----------------------------│ -------------------- ││ -------------------- │-33)-----------------------------│ files: 1 ││ partition_count(in->out): │-34)-----------------------------│ format: parquet ││ 1 -> 4 │-35)-----------------------------│ ││ │-36)-----------------------------│ predicate: ││ partitioning_scheme: │-37)-----------------------------│ DynamicFilter [ empty ] ││ RoundRobinBatch(4) │-38)-----------------------------└───────────────────────────┘└─────────────┬─────────────┘-39)----------------------------------------------------------┌─────────────┴─────────────┐-40)----------------------------------------------------------│ DataSourceExec │-41)----------------------------------------------------------│ -------------------- │-42)----------------------------------------------------------│ files: 1 │-43)----------------------------------------------------------│ format: csv │-44)----------------------------------------------------------└───────────────────────────┘+04)│ on: ├──────────────┐+05)│ (int_col = int_col) │ │+06)└─────────────┬─────────────┘ │+07)┌─────────────┴─────────────┐┌─────────────┴─────────────┐+08)│ DataSourceExec ││ HashJoinExec │+09)│ -------------------- ││ -------------------- │+10)│ bytes: 512 ││ on: ├──────────────┐+11)│ format: memory ││ (int_col = int_col) │ │+12)│ rows: 1 ││ │ │+13)└───────────────────────────┘└─────────────┬─────────────┘ │+14)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐+15)-----------------------------│ DataSourceExec ││ RepartitionExec │+16)-----------------------------│ -------------------- ││ -------------------- │+17)-----------------------------│ files: 1 ││ partition_count(in->out): │+18)-----------------------------│ format: parquet ││ 1 -> 4 │+19)-----------------------------│ ││ │+20)-----------------------------│ predicate: ││ partitioning_scheme: │+21)-----------------------------│ DynamicFilter [ empty ] ││ RoundRobinBatch(4) │+22)-----------------------------└───────────────────────────┘└─────────────┬─────────────┘+23)----------------------------------------------------------┌─────────────┴─────────────┐+24)----------------------------------------------------------│ DataSourceExec │+25)----------------------------------------------------------│ -------------------- │+26)----------------------------------------------------------│ files: 1 │+27)----------------------------------------------------------│ format: csv │+28)----------------------------------------------------------└───────────────────────────┘# Long Filter (demonstrate what happens with wrapping)query TT@@ -1075,40 +1051,28 @@ explain select * from table1 inner join table2 on table1.int_col = table2.int_co----physical_plan01)┌───────────────────────────┐-02)│ ProjectionExec │+02)│ HashJoinExec │03)│ -------------------- │-04)│ bigint_col: │-05)│ bigint_col │-06)│ │-07)│ date_col: date_col │-08)│ int_col: int_col │-09)│ │-10)│ string_col: │-11)│ string_col │-12)└─────────────┬─────────────┘-13)┌─────────────┴─────────────┐-14)│ HashJoinExec │-15)│ -------------------- │-16)│ on: │-17)│ (int_col = int_col), ├──────────────┐-18)│ (string_col = │ │-19)│ string_col) │ │-20)└─────────────┬─────────────┘ │-21)┌─────────────┴─────────────┐┌─────────────┴─────────────┐-22)│ DataSourceExec ││ RepartitionExec │-23)│ -------------------- ││ -------------------- │-24)│ files: 1 ││ partition_count(in->out): │-25)│ format: parquet ││ 1 -> 4 │-26)│ ││ │-27)│ ││ partitioning_scheme: │-28)│ ││ RoundRobinBatch(4) │-29)└───────────────────────────┘└─────────────┬─────────────┘-30)-----------------------------┌─────────────┴─────────────┐-31)-----------------------------│ DataSourceExec │-32)-----------------------------│ -------------------- │-33)-----------------------------│ files: 1 │-34)-----------------------------│ format: csv │-35)-----------------------------└───────────────────────────┘+04)│ on: │+05)│ (int_col = int_col), ├──────────────┐+06)│ (string_col = │ │+07)│ string_col) │ │+08)└─────────────┬─────────────┘ │+09)┌─────────────┴─────────────┐┌─────────────┴─────────────┐+10)│ DataSourceExec ││ RepartitionExec │+11)│ -------------------- ││ -------------------- │+12)│ files: 1 ││ partition_count(in->out): │+13)│ format: parquet ││ 1 -> 4 │+14)│ ││ │+15)│ ││ partitioning_scheme: │+16)│ ││ RoundRobinBatch(4) │+17)└───────────────────────────┘└─────────────┬─────────────┘+18)-----------------------------┌─────────────┴─────────────┐+19)-----------------------------│ DataSourceExec │+20)-----------------------------│ -------------------- │+21)-----------------------------│ files: 1 │+22)-----------------------------│ format: csv │+23)-----------------------------└───────────────────────────┘# Query with outer hash join.query TT@@ -1116,42 +1080,30 @@ explain select * from table1 left outer join table2 on table1.int_col = table2.i----physical_plan01)┌───────────────────────────┐-02)│ ProjectionExec │+02)│ HashJoinExec │03)│ -------------------- │-04)│ bigint_col: │-05)│ bigint_col │-06)│ │-07)│ date_col: date_col │-08)│ int_col: int_col │-09)│ │-10)│ string_col: │-11)│ string_col │-12)└─────────────┬─────────────┘-13)┌─────────────┴─────────────┐-14)│ HashJoinExec │-15)│ -------------------- │-16)│ join_type: Right │-17)│ │-18)│ on: ├──────────────┐-19)│ (int_col = int_col), │ │-20)│ (string_col = │ │-21)│ string_col) │ │-22)└─────────────┬─────────────┘ │-23)┌─────────────┴─────────────┐┌─────────────┴─────────────┐-24)│ DataSourceExec ││ RepartitionExec │-25)│ -------------------- ││ -------------------- │-26)│ files: 1 ││ partition_count(in->out): │-27)│ format: parquet ││ 1 -> 4 │-28)│ ││ │-29)│ ││ partitioning_scheme: │-30)│ ││ RoundRobinBatch(4) │-31)└───────────────────────────┘└─────────────┬─────────────┘-32)-----------------------------┌─────────────┴─────────────┐-33)-----------------------------│ DataSourceExec │-34)-----------------------------│ -------------------- │-35)-----------------------------│ files: 1 │-36)-----------------------------│ format: csv │-37)-----------------------------└───────────────────────────┘+04)│ join_type: Right │+05)│ │+06)│ on: ├──────────────┐+07)│ (int_col = int_col), │ │+08)│ (string_col = │ │+09)│ string_col) │ │+10)└─────────────┬─────────────┘ │+11)┌─────────────┴─────────────┐┌─────────────┴─────────────┐+12)│ DataSourceExec ││ RepartitionExec │+13)│ -------------------- ││ -------------------- │+14)│ files: 1 ││ partition_count(in->out): │+15)│ format: parquet ││ 1 -> 4 │+16)│ ││ │+17)│ ││ partitioning_scheme: │+18)│ ││ RoundRobinBatch(4) │+19)└───────────────────────────┘└─────────────┬─────────────┘+20)-----------------------------┌─────────────┴─────────────┐+21)-----------------------------│ DataSourceExec │+22)-----------------------------│ -------------------- │+23)-----------------------------│ files: 1 │+24)-----------------------------│ format: csv │+25)-----------------------------└───────────────────────────┘# Query with nested loop join.query TTdiff --git a/datafusion/sqllogictest/test_files/group_by.slt b/datafusion/sqllogictest/test_files/group_by.sltindex 366326479d..59db63ba42 100644--- a/datafusion/sqllogictest/test_files/group_by.slt+++ b/datafusion/sqllogictest/test_files/group_by.slt@@ -2025,10 +2025,9 @@ physical_plan05)--------RepartitionExec: partitioning=Hash([col0@0, col1@1, col2@2], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[col0@0 as col0, col1@1 as col1, col2@2 as col2], aggr=[last_value(r.col1) ORDER BY [r.col0 ASC NULLS LAST]]07)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-08)--------------ProjectionExec: expr=[col0@2 as col0, col1@3 as col1, col2@4 as col2, col0@0 as col0, col1@1 as col1]-09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(col0@0, col0@0)]-10)------------------DataSourceExec: partitions=1, partition_sizes=[3]-11)------------------DataSourceExec: partitions=1, partition_sizes=[3]+08)--------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(col0@0, col0@0)], projection=[col0@2, col1@3, col2@4, col0@0, col1@1]+09)----------------DataSourceExec: partitions=1, partition_sizes=[3]+10)----------------DataSourceExec: partitions=1, partition_sizes=[3]# Columns in the table are a,b,c,d. Source is DataSourceExec which is ordered by# a,b,c column. Column a has cardinality 2, column b has cardinality 4.@@ -2946,10 +2945,9 @@ physical_plan01)SortExec: expr=[sn@2 ASC NULLS LAST], preserve_partitioning=[false]02)--ProjectionExec: expr=[zip_code@1 as zip_code, country@2 as country, sn@0 as sn, ts@3 as ts, currency@4 as currency, last_value(e.amount) ORDER BY [e.sn ASC NULLS LAST]@5 as last_rate]03)----AggregateExec: mode=Single, gby=[sn@2 as sn, zip_code@0 as zip_code, country@1 as country, ts@3 as ts, currency@4 as currency], aggr=[last_value(e.amount) ORDER BY [e.sn ASC NULLS LAST]]-04)------ProjectionExec: expr=[zip_code@2 as zip_code, country@3 as country, sn@4 as sn, ts@5 as ts, currency@6 as currency, sn@0 as sn, amount@1 as amount]-05)--------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(currency@2, currency@4)], filter=ts@0 >= ts@1, projection=[sn@0, amount@3, zip_code@4, country@5, sn@6, ts@7, currency@8]-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----------DataSourceExec: partitions=1, partition_sizes=[1]+04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(currency@2, currency@4)], filter=ts@0 >= ts@1, projection=[zip_code@4, country@5, sn@6, ts@7, currency@8, sn@0, amount@3]+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--------DataSourceExec: partitions=1, partition_sizes=[1]query ITIPTR rowsortSELECT s.zip_code, s.country, s.sn, s.ts, s.currency, LAST_VALUE(e.amount ORDER BY e.sn) AS last_rate@@ -4081,14 +4079,13 @@ logical_plan09)--------Aggregate: groupBy=[[multiple_ordered_table_with_pk.c, multiple_ordered_table_with_pk.b]], aggr=[[sum(CAST(multiple_ordered_table_with_pk.d AS Int64))]]10)----------TableScan: multiple_ordered_table_with_pk projection=[b, c, d]physical_plan-01)ProjectionExec: expr=[c@0 as c, c@2 as c, sum1@1 as sum1, sum1@3 as sum1]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(b@1, b@1)], projection=[c@0, sum1@2, c@3, sum1@5]-03)----ProjectionExec: expr=[c@0 as c, b@1 as b, sum(multiple_ordered_table_with_pk.d)@2 as sum1]-04)------AggregateExec: mode=Single, gby=[c@1 as c, b@0 as b], aggr=[sum(multiple_ordered_table_with_pk.d)], ordering_mode=PartiallySorted([0])-05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c, d], output_ordering=[c@1 ASC NULLS LAST], constraints=[PrimaryKey([3])], file_type=csv, has_header=true-06)----ProjectionExec: expr=[c@0 as c, b@1 as b, sum(multiple_ordered_table_with_pk.d)@2 as sum1]-07)------AggregateExec: mode=Single, gby=[c@1 as c, b@0 as b], aggr=[sum(multiple_ordered_table_with_pk.d)], ordering_mode=PartiallySorted([0])-08)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c, d], output_ordering=[c@1 ASC NULLS LAST], constraints=[PrimaryKey([3])], file_type=csv, has_header=true+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(b@1, b@1)], projection=[c@0, c@3, sum1@2, sum1@5]+02)--ProjectionExec: expr=[c@0 as c, b@1 as b, sum(multiple_ordered_table_with_pk.d)@2 as sum1]+03)----AggregateExec: mode=Single, gby=[c@1 as c, b@0 as b], aggr=[sum(multiple_ordered_table_with_pk.d)], ordering_mode=PartiallySorted([0])+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c, d], output_ordering=[c@1 ASC NULLS LAST], constraints=[PrimaryKey([3])], file_type=csv, has_header=true+05)--ProjectionExec: expr=[c@0 as c, b@1 as b, sum(multiple_ordered_table_with_pk.d)@2 as sum1]+06)----AggregateExec: mode=Single, gby=[c@1 as c, b@0 as b], aggr=[sum(multiple_ordered_table_with_pk.d)], ordering_mode=PartiallySorted([0])+07)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[b, c, d], output_ordering=[c@1 ASC NULLS LAST], constraints=[PrimaryKey([3])], file_type=csv, has_header=truequery TTEXPLAIN SELECT lhs.c, rhs.c, lhs.sum1, rhs.sum1diff --git a/datafusion/sqllogictest/test_files/join.slt.part b/datafusion/sqllogictest/test_files/join.slt.partindex c0a838c97d..b9d163d877 100644--- a/datafusion/sqllogictest/test_files/join.slt.part+++ b/datafusion/sqllogictest/test_files/join.slt.part@@ -1367,18 +1367,16 @@ logical_plan06)----TableScan: f projection=[a]07)--TableScan: s projection=[b]physical_plan-01)ProjectionExec: expr=[col0@1 as col0, col1@2 as col1, a@3 as a, b@0 as b]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(s.b AS Int64)@1, col1@1)], projection=[b@0, col0@2, col1@3, a@4]-03)----ProjectionExec: expr=[b@0 as b, CAST(b@0 AS Int64) as CAST(s.b AS Int64)]-04)------DataSourceExec: partitions=1, partition_sizes=[1]-05)----ProjectionExec: expr=[col0@1 as col0, col1@2 as col1, a@0 as a]-06)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(f.a AS Int64)@1, col0@0)], projection=[a@0, col0@2, col1@3]-07)--------ProjectionExec: expr=[a@0 as a, CAST(a@0 AS Int64) as CAST(f.a AS Int64)]-08)----------DataSourceExec: partitions=1, partition_sizes=[1]-09)--------ProjectionExec: expr=[CAST(x@0 AS Int64) + 1 as col0, CAST(y@1 AS Int64) + 1 as col1]-10)----------RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1-11)------------FilterExec: y@1 = x@0-12)--------------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(s.b AS Int64)@1, col1@1)], projection=[col0@2, col1@3, a@4, b@0]+02)--ProjectionExec: expr=[b@0 as b, CAST(b@0 AS Int64) as CAST(s.b AS Int64)]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(f.a AS Int64)@1, col0@0)], projection=[col0@2, col1@3, a@0]+05)----ProjectionExec: expr=[a@0 as a, CAST(a@0 AS Int64) as CAST(f.a AS Int64)]+06)------DataSourceExec: partitions=1, partition_sizes=[1]+07)----ProjectionExec: expr=[CAST(x@0 AS Int64) + 1 as col0, CAST(y@1 AS Int64) + 1 as col1]+08)------RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1+09)--------FilterExec: y@1 = x@0+10)----------DataSourceExec: partitions=1, partition_sizes=[1]statement okdrop table pairs;@@ -1421,15 +1419,14 @@ logical_plan05)------Aggregate: groupBy=[[t1.v0]], aggr=[[sum(t1.v1)]]06)--------TableScan: t1 projection=[v0, v1]physical_plan-01)ProjectionExec: expr=[v0@1 as v0, v1@2 as v1, sum(t1.v1)@0 as sum(t1.v1)]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(v0@1, v0@0)], projection=[sum(t1.v1)@0, v0@2, v1@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[sum(t1.v1)@1 as sum(t1.v1), v0@0 as v0]-05)--------AggregateExec: mode=FinalPartitioned, gby=[v0@0 as v0], aggr=[sum(t1.v1)]-06)----------RepartitionExec: partitioning=Hash([v0@0], 4), input_partitions=1-07)------------AggregateExec: mode=Partial, gby=[v0@0 as v0], aggr=[sum(t1.v1)]-08)--------------DataSourceExec: partitions=1, partition_sizes=[1]-09)----DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Right, on=[(v0@1, v0@0)], projection=[v0@2, v1@3, sum(t1.v1)@0]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[sum(t1.v1)@1 as sum(t1.v1), v0@0 as v0]+04)------AggregateExec: mode=FinalPartitioned, gby=[v0@0 as v0], aggr=[sum(t1.v1)]+05)--------RepartitionExec: partitioning=Hash([v0@0], 4), input_partitions=1+06)----------AggregateExec: mode=Partial, gby=[v0@0 as v0], aggr=[sum(t1.v1)]+07)------------DataSourceExec: partitions=1, partition_sizes=[1]+08)--DataSourceExec: partitions=1, partition_sizes=[1]query IIISELECT *diff --git a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.sltindex 2bab89c99e..1b6f2e4c86 100644--- a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt+++ b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt@@ -140,8 +140,8 @@ logical_plan04)------TableScan: t1 projection=[id, val]05)----TableScan: t2 projection=[id, val]physical_plan-01)ProjectionExec: expr=[id@0 as t1_id, id@2 as t2_id, val@1 as val, val@3 as val]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1.val + Int64(1)@2, t2.val + Int64(1)@2)], projection=[id@0, val@1, id@3, val@4], NullsEqual: true+01)ProjectionExec: expr=[id@0 as t1_id, id@1 as t2_id, val@2 as val, val@3 as val]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1.val + Int64(1)@2, t2.val + Int64(1)@2)], projection=[id@0, id@3, val@1, val@4], NullsEqual: true03)----CoalescePartitionsExec04)------ProjectionExec: expr=[id@0 as id, val@1 as val, CAST(val@1 AS Int64) + 1 as t1.val + Int64(1)]05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1@@ -169,8 +169,8 @@ logical_plan03)----TableScan: t1 projection=[id, val]04)----TableScan: t2 projection=[id, val]physical_plan-01)ProjectionExec: expr=[id@0 as t1_id, id@2 as t2_id, val@1 as val, val@3 as val]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1.val + Int64(1)@2, t2.val + Int64(1)@2)], filter=CAST(val@0 AS Int64) % 3 IS DISTINCT FROM CAST(val@1 AS Int64) % 3, projection=[id@0, val@1, id@3, val@4], NullsEqual: true+01)ProjectionExec: expr=[id@0 as t1_id, id@1 as t2_id, val@2 as val, val@3 as val]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t1.val + Int64(1)@2, t2.val + Int64(1)@2)], filter=CAST(val@0 AS Int64) % 3 IS DISTINCT FROM CAST(val@1 AS Int64) % 3, projection=[id@0, id@3, val@1, val@4], NullsEqual: true03)----ProjectionExec: expr=[id@0 as id, val@1 as val, CAST(val@1 AS Int64) + 1 as t1.val + Int64(1)]04)------DataSourceExec: partitions=1, partition_sizes=[1]05)----ProjectionExec: expr=[id@0 as id, val@1 as val, CAST(val@1 AS Int64) + 1 as t2.val + Int64(1)]@@ -229,8 +229,8 @@ logical_plan05)------TableScan: t2 projection=[id, val]06)----TableScan: t0 projection=[val]physical_plan-01)ProjectionExec: expr=[id@0 as t1_id, id@2 as t2_id, val@1 as val, val@3 as val]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(val@0, val@1)], projection=[id@1, val@2, id@3, val@4], NullsEqual: true+01)ProjectionExec: expr=[id@0 as t1_id, id@1 as t2_id, val@2 as val, val@3 as val]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(val@0, val@1)], projection=[id@1, id@3, val@2, val@4], NullsEqual: true03)----DataSourceExec: partitions=1, partition_sizes=[1]04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=105)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(val@1, val@1)], NullsEqual: truediff --git a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt b/datafusion/sqllogictest/test_files/join_limit_pushdown.sltindex 24d48e5661..ea9c971065 100644--- a/datafusion/sqllogictest/test_files/join_limit_pushdown.slt+++ b/datafusion/sqllogictest/test_files/join_limit_pushdown.slt@@ -74,10 +74,9 @@ logical_plan04)----Limit: skip=0, fetch=305)------TableScan: t2 projection=[x], fetch=3physical_plan-01)ProjectionExec: expr=[a@1 as a, x@0 as x]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(x@0, a@0)], fetch=3-03)----DataSourceExec: partitions=1, partition_sizes=[1], fetch=3-04)----DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(x@0, a@0)], projection=[a@1, x@0], fetch=3+02)--DataSourceExec: partitions=1, partition_sizes=[1], fetch=3+03)--DataSourceExec: partitions=1, partition_sizes=[1]query IISELECT t1.a, t2.x FROM t1 RIGHT JOIN t2 ON t1.a = t2.x LIMIT 3;@@ -221,12 +220,11 @@ logical_plan05)------TableScan: t2 projection=[x]06)----TableScan: t3 projection=[p]physical_plan-01)ProjectionExec: expr=[a@1 as a, x@2 as x, p@0 as p]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p@0, x@1)], fetch=2-03)----DataSourceExec: partitions=1, partition_sizes=[1]-04)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)]-05)------DataSourceExec: partitions=1, partition_sizes=[1]-06)------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p@0, x@1)], projection=[a@1, x@2, p@0], fetch=2+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(a@0, x@0)]+04)----DataSourceExec: partitions=1, partition_sizes=[1]+05)----DataSourceExec: partitions=1, partition_sizes=[1]query IIISELECT t1.a, t2.x, t3.pdiff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.sltindex 0c14958f1a..1ed3fc89b2 100644--- a/datafusion/sqllogictest/test_files/joins.slt+++ b/datafusion/sqllogictest/test_files/joins.slt@@ -1536,15 +1536,14 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id + UInt32(1)@1, join_t1.t1_id + UInt32(12)@2)], projection=[t2_id@0, t1_id@2, t1_name@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 + 1 as join_t2.t2_id + UInt32(1)]-05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 12 as join_t1.t1_id + UInt32(12)]-08)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-09)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id + UInt32(1)@1, join_t1.t1_id + UInt32(12)@2)], projection=[t1_id@2, t2_id@0, t1_name@3]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 + 1 as join_t2.t2_id + UInt32(1)]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 12 as join_t1.t1_id + UInt32(12)]+07)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+08)------DataSourceExec: partitions=1, partition_sizes=[1]statement okset datafusion.optimizer.repartition_joins = true;@@ -1561,15 +1560,14 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id + UInt32(1)@1, join_t1.t1_id + UInt32(12)@2)], projection=[t2_id@0, t1_id@2, t1_name@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 + 1 as join_t2.t2_id + UInt32(1)]-05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 12 as join_t1.t1_id + UInt32(12)]-08)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-09)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id + UInt32(1)@1, join_t1.t1_id + UInt32(12)@2)], projection=[t1_id@2, t2_id@0, t1_name@3]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 + 1 as join_t2.t2_id + UInt32(1)]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 12 as join_t1.t1_id + UInt32(12)]+07)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+08)------DataSourceExec: partitions=1, partition_sizes=[1]# Left side expr key inner join@@ -1589,12 +1587,11 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t2_id@0, join_t1.t1_id + UInt32(11)@2)], projection=[t2_id@0, t1_id@1, t1_name@2]-03)----DataSourceExec: partitions=1, partition_sizes=[1]-04)----ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t2_id@0, join_t1.t1_id + UInt32(11)@2)], projection=[t1_id@1, t2_id@0, t1_name@2]+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]+04)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)------DataSourceExec: partitions=1, partition_sizes=[1]# don't reorder join based on stats, and use the order in the query@@ -1612,14 +1609,13 @@ INNER JOIN join_t2ON join_t1.t1_id + cast(11 as INT UNSIGNED) = join_t2.t2_id----physical_plan-01)ProjectionExec: expr=[t1_id@0 as t1_id, t2_id@2 as t2_id, t1_name@1 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t1.t1_id + UInt32(11)@2, t2_id@0)], projection=[t1_id@0, t1_name@1, t2_id@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]-05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-08)------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t1.t1_id + UInt32(11)@2, t2_id@0)], projection=[t1_id@0, t2_id@3, t1_name@1]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+07)----DataSourceExec: partitions=1, partition_sizes=[1]statement okset datafusion.optimizer.join_reordering = true;@@ -1643,12 +1639,11 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t2_id@0, join_t1.t1_id + UInt32(11)@2)], projection=[t2_id@0, t1_id@1, t1_name@2]-03)----DataSourceExec: partitions=1, partition_sizes=[1]-04)----ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]-05)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(t2_id@0, join_t1.t1_id + UInt32(11)@2)], projection=[t1_id@1, t2_id@0, t1_name@2]+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--ProjectionExec: expr=[t1_id@0 as t1_id, t1_name@1 as t1_name, t1_id@0 + 11 as join_t1.t1_id + UInt32(11)]+04)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)------DataSourceExec: partitions=1, partition_sizes=[1]# Right side expr key inner join@@ -1668,14 +1663,13 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id - UInt32(11)@1, t1_id@0)], projection=[t2_id@0, t1_id@2, t1_name@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 - 11 as join_t2.t2_id - UInt32(11)]-05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-08)------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id - UInt32(11)@1, t1_id@0)], projection=[t1_id@2, t2_id@0, t1_name@3]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 - 11 as join_t2.t2_id - UInt32(11)]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+07)----DataSourceExec: partitions=1, partition_sizes=[1]statement okset datafusion.optimizer.repartition_joins = true;@@ -1693,14 +1687,13 @@ logical_plan03)----TableScan: join_t1 projection=[t1_id, t1_name]04)----TableScan: join_t2 projection=[t2_id]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id, t1_name@2 as t1_name]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id - UInt32(11)@1, t1_id@0)], projection=[t2_id@0, t1_id@2, t1_name@3]-03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 - 11 as join_t2.t2_id - UInt32(11)]-05)--------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1-08)------DataSourceExec: partitions=1, partition_sizes=[1]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(join_t2.t2_id - UInt32(11)@1, t1_id@0)], projection=[t1_id@2, t2_id@0, t1_name@3]+02)--CoalescePartitionsExec+03)----ProjectionExec: expr=[t2_id@0 as t2_id, t2_id@0 - 11 as join_t2.t2_id - UInt32(11)]+04)------RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1+07)----DataSourceExec: partitions=1, partition_sizes=[1]# Select wildcard with expr key inner join@@ -2063,15 +2056,14 @@ logical_plan05)----Filter: join_t2.t2_int > UInt32(1)06)------TableScan: join_t2 projection=[t2_id, t2_int]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, t2_id@0 as t2_id]-02)--NestedLoopJoinExec: join_type=Inner, filter=t1_id@0 > t2_id@1-03)----CoalescePartitionsExec-04)------FilterExec: t2_int@1 > 1, projection=[t2_id@0]-05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-06)----------DataSourceExec: partitions=1, partition_sizes=[1]-07)----FilterExec: t1_id@0 > 10-08)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-09)--------DataSourceExec: partitions=1, partition_sizes=[1]+01)NestedLoopJoinExec: join_type=Inner, filter=t1_id@0 > t2_id@1, projection=[t1_id@1, t2_id@0]+02)--CoalescePartitionsExec+03)----FilterExec: t2_int@1 > 1, projection=[t2_id@0]+04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)--------DataSourceExec: partitions=1, partition_sizes=[1]+06)--FilterExec: t1_id@0 > 10+07)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+08)------DataSourceExec: partitions=1, partition_sizes=[1]query IISELECT join_t1.t1_id, join_t2.t2_id@@ -3634,15 +3626,14 @@ logical_plan09)----Projection: Int64(1) AS e, Int64(3) AS f10)------EmptyRelation: rows=1physical_plan-01)ProjectionExec: expr=[c@2 as c, d@3 as d, e@0 as e, f@1 as f]-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(e@0, c@0)]-03)----ProjectionExec: expr=[1 as e, 3 as f]-04)------PlaceholderRowExec-05)----UnionExec-06)------ProjectionExec: expr=[1 as c, 2 as d]-07)--------PlaceholderRowExec-08)------ProjectionExec: expr=[1 as c, 3 as d]-09)--------PlaceholderRowExec+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(e@0, c@0)], projection=[c@2, d@3, e@0, f@1]+02)--ProjectionExec: expr=[1 as e, 3 as f]+03)----PlaceholderRowExec+04)--UnionExec+05)----ProjectionExec: expr=[1 as c, 2 as d]+06)------PlaceholderRowExec+07)----ProjectionExec: expr=[1 as c, 3 as d]+08)------PlaceholderRowExecquery IIII rowsortSELECT * FROM (@@ -3676,15 +3667,14 @@ logical_plan09)----Projection: Int64(1) AS e, Int64(3) AS f10)------EmptyRelation: rows=1physical_plan-01)ProjectionExec: expr=[c@2 as c, d@3 as d, e@0 as e, f@1 as f]-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(e@0, c@0)]-03)----ProjectionExec: expr=[1 as e, 3 as f]-04)------PlaceholderRowExec-05)----UnionExec-06)------ProjectionExec: expr=[1 as c, 2 as d]-07)--------PlaceholderRowExec-08)------ProjectionExec: expr=[1 as c, 3 as d]-09)--------PlaceholderRowExec+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(e@0, c@0)], projection=[c@2, d@3, e@0, f@1]+02)--ProjectionExec: expr=[1 as e, 3 as f]+03)----PlaceholderRowExec+04)--UnionExec+05)----ProjectionExec: expr=[1 as c, 2 as d]+06)------PlaceholderRowExec+07)----ProjectionExec: expr=[1 as c, 3 as d]+08)------PlaceholderRowExecquery IIII rowsortSELECT * FROM (@@ -3888,11 +3878,10 @@ logical_plan05)----Sort: right_table_no_nulls.b ASC NULLS LAST, fetch=1006)------TableScan: right_table_no_nulls projection=[a, b]physical_plan-01)ProjectionExec: expr=[a@2 as a, b@3 as b, a@0 as a, b@1 as b]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)]-03)----SortExec: TopK(fetch=10), expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]-04)------DataSourceExec: partitions=1, partition_sizes=[2]-05)----DataSourceExec: partitions=1, partition_sizes=[2]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)], projection=[a@2, b@3, a@0, b@1]+02)--SortExec: TopK(fetch=10), expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[2]+04)--DataSourceExec: partitions=1, partition_sizes=[2]@@ -3945,10 +3934,9 @@ logical_plan04)--SubqueryAlias: rhs05)----TableScan: right_table_no_nulls projection=[a, b]physical_plan-01)ProjectionExec: expr=[a@2 as a, b@3 as b, a@0 as a, b@1 as b]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)]-03)----DataSourceExec: partitions=1, partition_sizes=[2]-04)----DataSourceExec: partitions=1, partition_sizes=[2]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)], projection=[a@2, b@3, a@0, b@1]+02)--DataSourceExec: partitions=1, partition_sizes=[2]+03)--DataSourceExec: partitions=1, partition_sizes=[2]# Null build indices:@@ -4004,11 +3992,10 @@ logical_plan05)----Sort: right_table_no_nulls.b ASC NULLS LAST, fetch=1006)------TableScan: right_table_no_nulls projection=[a, b]physical_plan-01)ProjectionExec: expr=[a@2 as a, b@3 as b, a@0 as a, b@1 as b]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)]-03)----SortExec: TopK(fetch=10), expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]-04)------DataSourceExec: partitions=1, partition_sizes=[2]-05)----DataSourceExec: partitions=1, partition_sizes=[2]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(b@1, b@1)], projection=[a@2, b@3, a@0, b@1]+02)--SortExec: TopK(fetch=10), expr=[b@1 ASC NULLS LAST], preserve_partitioning=[false]+03)----DataSourceExec: partitions=1, partition_sizes=[2]+04)--DataSourceExec: partitions=1, partition_sizes=[2]# Test CROSS JOIN LATERAL syntax (planning)@@ -4908,10 +4895,9 @@ FROM t1JOIN t2 ON k1 = k2----physical_plan-01)ProjectionExec: expr=[k1@2 as k1, v1@3 as v1, k2@0 as k2, v2@1 as v2]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k2@0, k1@0)]-03)----DataSourceExec: partitions=1, partition_sizes=[0]-04)----DataSourceExec: partitions=1, partition_sizes=[10000]+01)HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k2@0, k1@0)], projection=[k1@2, v1@3, k2@0, v2@1]+02)--DataSourceExec: partitions=1, partition_sizes=[0]+03)--DataSourceExec: partitions=1, partition_sizes=[10000]query IIIISELECT sum(k1), sum(v1), sum(k2), sum(v2)@@ -4928,10 +4914,9 @@ FROM t1LEFT JOIN t2 ON k1 = k2----physical_plan-01)ProjectionExec: expr=[k1@2 as k1, v1@3 as v1, k2@0 as k2, v2@1 as v2]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(k2@0, k1@0)]-03)----DataSourceExec: partitions=1, partition_sizes=[0]-04)----DataSourceExec: partitions=1, partition_sizes=[10000]+01)HashJoinExec: mode=CollectLeft, join_type=Right, on=[(k2@0, k1@0)], projection=[k1@2, v1@3, k2@0, v2@1]+02)--DataSourceExec: partitions=1, partition_sizes=[0]+03)--DataSourceExec: partitions=1, partition_sizes=[10000]query IIIISELECT sum(k1), sum(v1), sum(k2), sum(v2)@@ -4948,10 +4933,9 @@ FROM t1RIGHT JOIN t2 ON k1 = k2----physical_plan-01)ProjectionExec: expr=[k1@2 as k1, v1@3 as v1, k2@0 as k2, v2@1 as v2]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(k2@0, k1@0)]-03)----DataSourceExec: partitions=1, partition_sizes=[0]-04)----DataSourceExec: partitions=1, partition_sizes=[10000]+01)HashJoinExec: mode=CollectLeft, join_type=Left, on=[(k2@0, k1@0)], projection=[k1@2, v1@3, k2@0, v2@1]+02)--DataSourceExec: partitions=1, partition_sizes=[0]+03)--DataSourceExec: partitions=1, partition_sizes=[10000]query IIIISELECT sum(k1), sum(v1), sum(k2), sum(v2)@@ -5044,10 +5028,9 @@ FROM t1FULL JOIN t2 ON k1 = k2----physical_plan-01)ProjectionExec: expr=[k1@2 as k1, v1@3 as v1, k2@0 as k2, v2@1 as v2]-02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(k2@0, k1@0)]-03)----DataSourceExec: partitions=1, partition_sizes=[0]-04)----DataSourceExec: partitions=1, partition_sizes=[10000]+01)HashJoinExec: mode=CollectLeft, join_type=Full, on=[(k2@0, k1@0)], projection=[k1@2, v1@3, k2@0, v2@1]+02)--DataSourceExec: partitions=1, partition_sizes=[0]+03)--DataSourceExec: partitions=1, partition_sizes=[10000]query IIIISELECT sum(k1), sum(v1), sum(k2), sum(v2)diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.sltindex ddc2fe5860..ec34d62a43 100644--- a/datafusion/sqllogictest/test_files/lateral_join.slt+++ b/datafusion/sqllogictest/test_files/lateral_join.slt@@ -527,8 +527,8 @@ logical_planphysical_plan01)SortPreservingMergeExec: [id@0 ASC NULLS LAST]02)--SortExec: expr=[id@0 ASC NULLS LAST], preserve_partitioning=[true]-03)----ProjectionExec: expr=[id@0 as id, CASE WHEN __always_true@2 IS NULL THEN 0 ELSE cnt@1 END as cnt]-04)------HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@0, t1_id@1)], projection=[id@0, cnt@1, __always_true@3]+03)----ProjectionExec: expr=[id@0 as id, CASE WHEN __always_true@1 IS NULL THEN 0 ELSE cnt@2 END as cnt]+04)------HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@0, t1_id@1)], projection=[id@0, __always_true@3, cnt@1]05)--------DataSourceExec: partitions=1, partition_sizes=[1]06)--------ProjectionExec: expr=[count(Int64(1))@1 as cnt, t1_id@0 as t1_id, true as __always_true]07)----------AggregateExec: mode=FinalPartitioned, gby=[t1_id@0 as t1_id], aggr=[count(Int64(1))]diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.sltindex 8bc2ca465e..dd1f69b286 100644--- a/datafusion/sqllogictest/test_files/predicates.slt+++ b/datafusion/sqllogictest/test_files/predicates.slt@@ -755,16 +755,15 @@ logical_plan10)------TableScan: partsupp projection=[ps_partkey, ps_suppkey]physical_plan01)AggregateExec: mode=SinglePartitioned, gby=[p_partkey@2 as p_partkey], aggr=[sum(lineitem.l_extendedprice), avg(lineitem.l_discount), count(DISTINCT partsupp.ps_suppkey)]-02)--ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, p_partkey@3 as p_partkey, ps_suppkey@0 as ps_suppkey]-03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, p_partkey@2)], projection=[ps_suppkey@1, l_extendedprice@2, l_discount@3, p_partkey@4]-04)------DataSourceExec: partitions=1, partition_sizes=[1]-05)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_extendedprice@1, l_discount@2, p_partkey@3]-06)--------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=1-07)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/lineitem.csv]]}, projection=[l_partkey, l_extendedprice, l_discount], file_type=csv, has_header=true-08)--------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4-09)----------FilterExec: p_brand@1 = Brand#12 OR p_brand@1 = Brand#23, projection=[p_partkey@0]-10)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-11)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/part.csv]]}, projection=[p_partkey, p_brand], file_type=csv, has_header=true+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, p_partkey@2)], projection=[l_extendedprice@2, l_discount@3, p_partkey@4, ps_suppkey@1]+03)----DataSourceExec: partitions=1, partition_sizes=[1]+04)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], projection=[l_extendedprice@1, l_discount@2, p_partkey@3]+05)------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=1+06)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/lineitem.csv]]}, projection=[l_partkey, l_extendedprice, l_discount], file_type=csv, has_header=true+07)------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4+08)--------FilterExec: p_brand@1 = Brand#12 OR p_brand@1 = Brand#23, projection=[p_partkey@0]+09)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+10)------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/tpch-csv/part.csv]]}, projection=[p_partkey, p_brand], file_type=csv, has_header=true# Simplification of a binary operator with a NULL valuediff --git a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt b/datafusion/sqllogictest/test_files/preserve_file_partitioning.sltindex 7c24a6bd2a..48d99625ea 100644--- a/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt+++ b/datafusion/sqllogictest/test_files/preserve_file_partitioning.slt@@ -362,13 +362,12 @@ physical_plan03)----AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey], aggr=[max(d.env), max(d.service), count(Int64(1)), sum(f.value)], ordering_mode=Sorted04)------RepartitionExec: partitioning=Hash([f_dkey@0], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST05)--------AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey], aggr=[max(d.env), max(d.service), count(Int64(1)), sum(f.value)], ordering_mode=Sorted-06)----------ProjectionExec: expr=[value@2 as value, f_dkey@3 as f_dkey, env@0 as env, service@1 as service]-07)------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@0, f_dkey@1)], projection=[env@1, service@2, value@3, f_dkey@4]-08)--------------CoalescePartitionsExec-09)----------------FilterExec: service@2 = log-10)------------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1-11)--------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-12)--------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+06)----------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@0, f_dkey@1)], projection=[value@3, f_dkey@4, env@1, service@2]+07)------------CoalescePartitionsExec+08)--------------FilterExec: service@2 = log+09)----------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1+10)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+11)------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]# Verify results without optimizationquery TTTIR rowsort@@ -414,13 +413,12 @@ physical_plan01)SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST]02)--ProjectionExec: expr=[f_dkey@0 as f_dkey, max(d.env)@1 as max(d.env), max(d.service)@2 as max(d.service), count(Int64(1))@3 as count(*), sum(f.value)@4 as sum(f.value)]03)----AggregateExec: mode=SinglePartitioned, gby=[f_dkey@1 as f_dkey], aggr=[max(d.env), max(d.service), count(Int64(1)), sum(f.value)], ordering_mode=Sorted-04)------ProjectionExec: expr=[value@2 as value, f_dkey@3 as f_dkey, env@0 as env, service@1 as service]-05)--------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@0, f_dkey@1)], projection=[env@1, service@2, value@3, f_dkey@4]-06)----------CoalescePartitionsExec-07)------------FilterExec: service@2 = log-08)--------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1-09)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-10)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+04)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@0, f_dkey@1)], projection=[value@3, f_dkey@4, env@1, service@2]+05)--------CoalescePartitionsExec+06)----------FilterExec: service@2 = log+07)------------RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1+08)--------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension/data.parquet]]}, projection=[d_dkey, env, service], file_type=parquet, predicate=service@2 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+09)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], output_ordering=[f_dkey@1 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]query TTTIR rowsortSELECT f.f_dkey, MAX(d.env), MAX(d.service), count(*), sum(f.value)@@ -641,12 +639,11 @@ physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)]02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=303)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)]-04)------ProjectionExec: expr=[value@1 as value, f_dkey@2 as f_dkey, env@0 as env]-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[env@0, value@2, f_dkey@3]-06)----------RepartitionExec: partitioning=Hash([d_dkey@1], 3), input_partitions=3-07)------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], file_type=parquet-08)----------RepartitionExec: partitioning=Hash([f_dkey@1], 3), input_partitions=3-09)------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], file_type=parquet, predicate=DynamicFilter [ empty ]+04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0]+05)--------RepartitionExec: partitioning=Hash([d_dkey@1], 3), input_partitions=3+06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], file_type=parquet+07)--------RepartitionExec: partitioning=Hash([f_dkey@1], 3), input_partitions=3+08)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], file_type=parquet, predicate=DynamicFilter [ empty ]query TTR rowsortSELECT f.f_dkey, d.env, sum(f.value)@@ -687,10 +684,9 @@ physical_plan01)AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, env@1 as env], aggr=[sum(f.value)]02)--RepartitionExec: partitioning=Hash([f_dkey@0, env@1], 3), input_partitions=303)----AggregateExec: mode=Partial, gby=[f_dkey@1 as f_dkey, env@2 as env], aggr=[sum(f.value)]-04)------ProjectionExec: expr=[value@1 as value, f_dkey@2 as f_dkey, env@0 as env]-05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[env@0, value@2, f_dkey@3]-06)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], file_type=parquet-07)----------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], file_type=parquet+04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@1)], projection=[value@2, f_dkey@3, env@0]+05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/dimension_partitioned/d_dkey=C/data.parquet]]}, projection=[env, d_dkey], file_type=parquet+06)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/preserve_file_partitioning/fact/f_dkey=C/data.parquet]]}, projection=[value, f_dkey], file_type=parquetquery TTR rowsortSELECT f.f_dkey, d.env, sum(f.value)diff --git a/datafusion/sqllogictest/test_files/projection_pushdown.slt b/datafusion/sqllogictest/test_files/projection_pushdown.sltindex 777f1e00ed..a40c1b8c7e 100644--- a/datafusion/sqllogictest/test_files/projection_pushdown.slt+++ b/datafusion/sqllogictest/test_files/projection_pushdown.slt@@ -1533,8 +1533,8 @@ logical_plan05)----Projection: get_field(join_right.s, Utf8("role")) AS __datafusion_extracted_2, join_right.id06)------TableScan: join_right projection=[id, s]physical_plan-01)ProjectionExec: expr=[id@1 as id, __datafusion_extracted_1@0 as simple_struct.s[label], __datafusion_extracted_2@2 as join_right.s[role]]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@1, id@1)], projection=[__datafusion_extracted_1@0, id@1, __datafusion_extracted_2@2]+01)ProjectionExec: expr=[id@0 as id, __datafusion_extracted_1@1 as simple_struct.s[label], __datafusion_extracted_2@2 as join_right.s[role]]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@1, id@1)], projection=[id@1, __datafusion_extracted_1@0, __datafusion_extracted_2@2]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, label) as __datafusion_extracted_1, id], file_type=parquet04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/join_right.parquet]]}, projection=[get_field(s@1, role) as __datafusion_extracted_2, id], file_type=parquet, predicate=DynamicFilter [ empty ]@@ -1603,8 +1603,8 @@ logical_plan07)--------Projection: get_field(join_right.s, Utf8("level")) AS __datafusion_extracted_1, join_right.id, get_field(join_right.s, Utf8("level")) AS __datafusion_extracted_308)----------TableScan: join_right projection=[id, s], partial_filters=[get_field(join_right.s, Utf8("level")) > Int64(5)]physical_plan-01)ProjectionExec: expr=[id@1 as id, __datafusion_extracted_2@0 as simple_struct.s[value], __datafusion_extracted_3@2 as join_right.s[level]]-02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@1, id@0)], projection=[__datafusion_extracted_2@0, id@1, __datafusion_extracted_3@3]+01)ProjectionExec: expr=[id@0 as id, __datafusion_extracted_2@1 as simple_struct.s[value], __datafusion_extracted_3@2 as join_right.s[level]]+02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(id@1, id@0)], projection=[id@1, __datafusion_extracted_2@0, __datafusion_extracted_3@3]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_2, id], file_type=parquet04)----FilterExec: __datafusion_extracted_1@0 > 5, projection=[id@1, __datafusion_extracted_3@2]05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/join_right.parquet]]}, projection=[get_field(s@1, level) as __datafusion_extracted_1, id, get_field(s@1, level) as __datafusion_extracted_3], file_type=parquet, predicate=get_field(s@1, level) > 5 AND DynamicFilter [ empty ]@@ -1896,8 +1896,8 @@ logical_plan07)------Projection: get_field(simple_struct.s, Utf8("value")) AS __datafusion_extracted_1, get_field(simple_struct.s, Utf8("value")) AS __datafusion_extracted_3, simple_struct.id08)--------TableScan: simple_struct projection=[id, s]physical_plan-01)ProjectionExec: expr=[__datafusion_extracted_3@1 as s.s[value], __datafusion_extracted_4@0 as j.s[role]]-02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@2, id@2)], filter=__datafusion_extracted_1@1 > __datafusion_extracted_2@0, projection=[__datafusion_extracted_4@1, __datafusion_extracted_3@4]+01)ProjectionExec: expr=[__datafusion_extracted_3@0 as s.s[value], __datafusion_extracted_4@1 as j.s[role]]+02)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@2, id@2)], filter=__datafusion_extracted_1@1 > __datafusion_extracted_2@0, projection=[__datafusion_extracted_3@4, __datafusion_extracted_4@1]03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/join_right.parquet]]}, projection=[get_field(s@1, level) as __datafusion_extracted_2, get_field(s@1, role) as __datafusion_extracted_4, id], file_type=parquet04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection_pushdown/simple.parquet]]}, projection=[get_field(s@1, value) as __datafusion_extracted_1, get_field(s@1, value) as __datafusion_extracted_3, id], file_type=parquet, predicate=DynamicFilter [ empty ]diff --git a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.sltindex fd49fd9004..dbf31dec5e 100644--- a/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt+++ b/datafusion/sqllogictest/test_files/repartition_subset_satisfaction.slt@@ -376,12 +376,11 @@ physical_plan08)--------------AggregateExec: mode=FinalPartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1 as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp), env@2 as env], aggr=[max(j.value)], ordering_mode=PartiallySorted([0, 1])09)----------------RepartitionExec: partitioning=Hash([f_dkey@0, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1, env@2], 3), input_partitions=3, preserve_order=true, sort_exprs=f_dkey@0 ASC NULLS LAST, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1 ASC NULLS LAST10)------------------AggregateExec: mode=Partial, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@2) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp), env@1 as env], aggr=[max(j.value)], ordering_mode=PartiallySorted([0, 1])-11)--------------------ProjectionExec: expr=[f_dkey@3 as f_dkey, env@0 as env, timestamp@1 as timestamp, value@2 as value]-12)----------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[env@0, timestamp@2, value@3, f_dkey@4]-13)------------------------CoalescePartitionsExec-14)--------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-15)----------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-16)------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+11)--------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]+12)----------------------CoalescePartitionsExec+13)------------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]+14)--------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+15)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]# Verify results without subset satisfactionquery TPR rowsort@@ -472,12 +471,11 @@ physical_plan06)----------AggregateExec: mode=Partial, gby=[env@1 as env, time_bin@0 as time_bin], aggr=[avg(a.max_bin_value)]07)------------ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1 as time_bin, env@2 as env, max(j.value)@3 as max_bin_value]08)--------------AggregateExec: mode=SinglePartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@2) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp), env@1 as env], aggr=[max(j.value)], ordering_mode=PartiallySorted([0, 1])-09)----------------ProjectionExec: expr=[f_dkey@3 as f_dkey, env@0 as env, timestamp@1 as timestamp, value@2 as value]-10)------------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[env@0, timestamp@2, value@3, f_dkey@4]-11)--------------------CoalescePartitionsExec-12)----------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]-13)------------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]-14)--------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]+09)----------------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[f_dkey@4, env@0, timestamp@2, value@3]+10)------------------CoalescePartitionsExec+11)--------------------FilterExec: service@1 = log, projection=[env@0, d_dkey@2]+12)----------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=A/data.parquet, WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=D/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/dimension/d_dkey=C/data.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)]+13)------------------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=A/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=B/data.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition_subset_satisfaction/fact/f_dkey=C/data.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ]# Verify results match with subset satisfactionquery TPR rowsortdiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex 7f88199b3c..5d54de6dbc 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -200,8 +200,8 @@ logical_plan06)--------Aggregate: groupBy=[[t2.t2_id]], aggr=[[sum(CAST(t2.t2_int AS Int64))]]07)----------TableScan: t2 projection=[t2_id, t2_int]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, sum(t2.t2_int)@0 as t2_sum]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[sum(t2.t2_int)@0, t1_id@2]+01)ProjectionExec: expr=[t1_id@0 as t1_id, sum(t2.t2_int)@1 as t2_sum]+02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[t1_id@2, sum(t2.t2_int)@0]03)----CoalescePartitionsExec04)------ProjectionExec: expr=[sum(t2.t2_int)@1 as sum(t2.t2_int), t2_id@0 as t2_id]05)--------AggregateExec: mode=FinalPartitioned, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]@@ -233,8 +233,8 @@ logical_plan06)--------Aggregate: groupBy=[[t2.t2_id]], aggr=[[sum(CAST(t2.t2_int AS Float64)) AS sum(t2.t2_int * Float64(1))]]07)----------TableScan: t2 projection=[t2_id, t2_int]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, sum(t2.t2_int * Float64(1)) + Int64(1)@0 as t2_sum]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[sum(t2.t2_int * Float64(1)) + Int64(1)@0, t1_id@2]+01)ProjectionExec: expr=[t1_id@0 as t1_id, sum(t2.t2_int * Float64(1)) + Int64(1)@1 as t2_sum]+02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[t1_id@2, sum(t2.t2_int * Float64(1)) + Int64(1)@0]03)----CoalescePartitionsExec04)------ProjectionExec: expr=[sum(t2.t2_int * Float64(1))@1 + 1 as sum(t2.t2_int * Float64(1)) + Int64(1), t2_id@0 as t2_id]05)--------AggregateExec: mode=FinalPartitioned, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int * Float64(1))]@@ -266,8 +266,8 @@ logical_plan06)--------Aggregate: groupBy=[[t2.t2_id]], aggr=[[sum(CAST(t2.t2_int AS Int64))]]07)----------TableScan: t2 projection=[t2_id, t2_int]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, sum(t2.t2_int)@0 as t2_sum]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[sum(t2.t2_int)@0, t1_id@2]+01)ProjectionExec: expr=[t1_id@0 as t1_id, sum(t2.t2_int)@1 as t2_sum]+02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[t1_id@2, sum(t2.t2_int)@0]03)----CoalescePartitionsExec04)------ProjectionExec: expr=[sum(t2.t2_int)@1 as sum(t2.t2_int), t2_id@0 as t2_id]05)--------AggregateExec: mode=FinalPartitioned, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]@@ -300,18 +300,17 @@ logical_plan07)----------Aggregate: groupBy=[[t2.t2_id]], aggr=[[sum(CAST(t2.t2_int AS Int64))]]08)------------TableScan: t2 projection=[t2_id, t2_int]physical_plan-01)ProjectionExec: expr=[t1_id@1 as t1_id, sum(t2.t2_int)@0 as t2_sum]-02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[sum(t2.t2_int)@0, t1_id@2]+01)ProjectionExec: expr=[t1_id@0 as t1_id, sum(t2.t2_int)@1 as t2_sum]+02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(t2_id@1, t1_id@0)], projection=[t1_id@2, sum(t2.t2_int)@0]03)----CoalescePartitionsExec-04)------ProjectionExec: expr=[sum(t2.t2_int)@1 as sum(t2.t2_int), t2_id@0 as t2_id]-05)--------FilterExec: sum(t2.t2_int)@1 < 3-06)----------AggregateExec: mode=FinalPartitioned, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]-07)------------RepartitionExec: partitioning=Hash([t2_id@0], 4), input_partitions=4-08)--------------AggregateExec: mode=Partial, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]-09)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-10)------------------DataSourceExec: partitions=1, partition_sizes=[2]-11)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-12)------DataSourceExec: partitions=1, partition_sizes=[2]+04)------FilterExec: sum(t2.t2_int)@1 < 3, projection=[sum(t2.t2_int)@1, t2_id@0]+05)--------AggregateExec: mode=FinalPartitioned, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]+06)----------RepartitionExec: partitioning=Hash([t2_id@0], 4), input_partitions=4+07)------------AggregateExec: mode=Partial, gby=[t2_id@0 as t2_id], aggr=[sum(t2.t2_int)]+08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+09)----------------DataSourceExec: partitions=1, partition_sizes=[2]+10)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+11)------DataSourceExec: partitions=1, partition_sizes=[2]query II rowsortSELECT t1_id, (SELECT sum(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id having sum(t2_int) < 3) as t2_sum from t1diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.partindex 84237e4393..7e3617b1d5 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q1.slt.part@@ -53,6 +53,6 @@ physical_plan04)------AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]05)--------RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))]-07)------------ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus]-08)--------------FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5]+07)------------ProjectionExec: expr=[l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as __common_expr_1, l_quantity@2 as l_quantity, l_extendedprice@0 as l_extendedprice, l_discount@1 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus]+08)--------------FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_extendedprice@1, l_discount@2, l_quantity@0, l_tax@3, l_returnflag@4, l_linestatus@5]09)----------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.partindex d5ff672440..b1a1538827 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q2.slt.part@@ -101,48 +101,46 @@ logical_planphysical_plan01)SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], fetch=1002)--SortExec: TopK(fetch=10), expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true]-03)----ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment]-04)------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8]-05)--------RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 4), input_partitions=4-06)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]-07)------------RepartitionExec: partitioning=Hash([n_regionkey@9], 4), input_partitions=4-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@4, n_nationkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@10, n_regionkey@11]-09)----------------RepartitionExec: partitioning=Hash([s_nationkey@4], 4), input_partitions=4-10)------------------ProjectionExec: expr=[p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_name@3 as s_name, s_address@4 as s_address, s_nationkey@5 as s_nationkey, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@2 as ps_supplycost]-11)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@2, s_suppkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_supplycost@3, s_name@5, s_address@6, s_nationkey@7, s_phone@8, s_acctbal@9, s_comment@10]-12)----------------------RepartitionExec: partitioning=Hash([ps_suppkey@2], 4), input_partitions=4-13)------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4]-14)--------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4-15)----------------------------FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1]-16)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-17)--------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl]]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=csv, has_header=false-18)--------------------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4-19)----------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=csv, has_header=false-20)----------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-21)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=csv, has_header=false-22)----------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=1-23)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=csv, has_header=false-24)------------RepartitionExec: partitioning=Hash([r_regionkey@0], 4), input_partitions=4-25)--------------FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]-26)----------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-27)------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/region.tbl]]}, projection=[r_regionkey, r_name], file_type=csv, has_header=false-28)--------RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 4), input_partitions=4-29)----------ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey]-30)------------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]-31)--------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4-32)----------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]-33)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]-34)--------------------RepartitionExec: partitioning=Hash([n_regionkey@2], 4), input_partitions=4-35)----------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_partkey@0, ps_supplycost@1, n_regionkey@4]-36)------------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4-37)--------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_supplycost@2, s_nationkey@4]-38)----------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4-39)------------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=csv, has_header=false-40)----------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1-41)------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false-42)------------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=1-43)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_regionkey], file_type=csv, has_header=false-44)--------------------RepartitionExec: partitioning=Hash([r_regionkey@0], 4), input_partitions=4-45)----------------------FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]-46)------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1-47)--------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/region.tbl]]}, projection=[r_regionkey, r_name], file_type=csv, has_header=false+03)----HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[s_acctbal@5, s_name@2, n_name@8, p_partkey@0, p_mfgr@1, s_address@3, s_phone@4, s_comment@6]+04)------RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 4), input_partitions=4+05)--------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@9, r_regionkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, ps_supplycost@7, n_name@8]+06)----------RepartitionExec: partitioning=Hash([n_regionkey@9], 4), input_partitions=4+07)------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@4, n_nationkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@10, n_regionkey@11]+08)--------------RepartitionExec: partitioning=Hash([s_nationkey@4], 4), input_partitions=4+09)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@2, s_suppkey@0)], projection=[p_partkey@0, p_mfgr@1, s_name@5, s_address@6, s_nationkey@7, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@3]+10)------------------RepartitionExec: partitioning=Hash([ps_suppkey@2], 4), input_partitions=4+11)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4]+12)----------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4+13)------------------------FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1]+14)--------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+15)----------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl]]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=csv, has_header=false+16)----------------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4+17)------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=csv, has_header=false+18)------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+19)--------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=csv, has_header=false+20)--------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=1+21)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_name, n_regionkey], file_type=csv, has_header=false+22)----------RepartitionExec: partitioning=Hash([r_regionkey@0], 4), input_partitions=4+23)------------FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]+24)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+25)----------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/region.tbl]]}, projection=[r_regionkey, r_name], file_type=csv, has_header=false+26)------RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 4), input_partitions=4+27)--------ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey]+28)----------AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]+29)------------RepartitionExec: partitioning=Hash([ps_partkey@0], 4), input_partitions=4+30)--------------AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)]+31)----------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@2, r_regionkey@0)], projection=[ps_partkey@0, ps_supplycost@1]+32)------------------RepartitionExec: partitioning=Hash([n_regionkey@2], 4), input_partitions=4+33)--------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[ps_partkey@0, ps_supplycost@1, n_regionkey@4]+34)----------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4+35)------------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ps_suppkey@1, s_suppkey@0)], projection=[ps_partkey@0, ps_supplycost@2, s_nationkey@4]+36)--------------------------RepartitionExec: partitioning=Hash([ps_suppkey@1], 4), input_partitions=4+37)----------------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:0..2932049], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:2932049..5864098], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:5864098..8796147], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/partsupp.tbl:8796147..11728193]]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=csv, has_header=false+38)--------------------------RepartitionExec: partitioning=Hash([s_suppkey@0], 4), input_partitions=1+39)----------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/supplier.tbl]]}, projection=[s_suppkey, s_nationkey], file_type=csv, has_header=false+40)----------------------RepartitionExec: partitioning=Hash([n_nationkey@0], 4), input_partitions=1+41)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/nation.tbl]]}, projection=[n_nationkey, n_regionkey], file_type=csv, has_header=false+42)------------------RepartitionExec: partitioning=Hash([r_regionkey@0], 4), input_partitions=4+43)--------------------FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0]+44)----------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+45)------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/region.tbl]]}, projection=[r_regionkey, r_name], file_type=csv, has_header=falsediff --git a/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.partindex b4e7099339..590a737703 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q7.slt.part@@ -90,8 +90,8 @@ physical_plan04)------AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]05)--------RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)]-07)------------ProjectionExec: expr=[n_name@3 as supp_nation, n_name@4 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume]-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[l_extendedprice@0, l_discount@1, l_shipdate@2, n_name@4, n_name@6]+07)------------ProjectionExec: expr=[n_name@0 as supp_nation, n_name@1 as cust_nation, date_part(YEAR, l_shipdate@2) as l_year, l_extendedprice@3 * (Some(1),20,0 - l_discount@4) as volume]+08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_nationkey@3, n_nationkey@0)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@4, n_name@6, l_shipdate@2, l_extendedprice@0, l_discount@1]09)----------------RepartitionExec: partitioning=Hash([c_nationkey@3], 4), input_partitions=410)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@0, n_nationkey@0)], projection=[l_extendedprice@1, l_discount@2, l_shipdate@3, c_nationkey@4, n_name@6]11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@0], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.partindex 12f19d43d4..7f160ce072 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q8.slt.part@@ -96,8 +96,8 @@ physical_plan04)------AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]05)--------RepartitionExec: partitioning=Hash([o_year@0], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)]-07)------------ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation]-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[l_extendedprice@0, l_discount@1, o_orderdate@2, n_name@4]+07)------------ProjectionExec: expr=[date_part(YEAR, o_orderdate@0) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume, n_name@3 as nation]+08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(n_regionkey@3, r_regionkey@0)], projection=[o_orderdate@2, l_extendedprice@0, l_discount@1, n_name@4]09)----------------RepartitionExec: partitioning=Hash([n_regionkey@3], 4), input_partitions=410)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@2, n_nationkey@0)], projection=[l_extendedprice@0, l_discount@1, o_orderdate@3, n_regionkey@4, n_name@6]11)--------------------RepartitionExec: partitioning=Hash([s_nationkey@2], 4), input_partitions=4diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.partindex 4ec434c903..7a973490be 100644--- a/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part+++ b/datafusion/sqllogictest/test_files/tpch/plans/q9.slt.part@@ -81,8 +81,8 @@ physical_plan04)------AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]05)--------RepartitionExec: partitioning=Hash([nation@0, o_year@1], 4), input_partitions=406)----------AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)]-07)------------ProjectionExec: expr=[n_name@5 as nation, date_part(YEAR, o_orderdate@4) as o_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) - ps_supplycost@3 * l_quantity@0 as amount]-08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[l_quantity@0, l_extendedprice@1, l_discount@2, ps_supplycost@4, o_orderdate@5, n_name@7]+07)------------ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@1) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@5 as amount]+08)--------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(s_nationkey@3, n_nationkey@0)], projection=[n_name@7, o_orderdate@5, l_extendedprice@1, l_discount@2, ps_supplycost@4, l_quantity@0]09)----------------RepartitionExec: partitioning=Hash([s_nationkey@3], 4), input_partitions=410)------------------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_quantity@1, l_extendedprice@2, l_discount@3, s_nationkey@4, ps_supplycost@5, o_orderdate@7]11)--------------------RepartitionExec: partitioning=Hash([l_orderkey@0], 4), input_partitions=4