Apache DataFusion SQL Query Engine
fix: ProjectionPushdown internal error on NestedLoopJoin mark joins (#22902)
linfeng283648001bbc
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex aa624e050c..1994f9d74c 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1520,20 +1520,23 @@ impl ExecutionPlan for HashJoinExec {return Ok(None);}+ // TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.let schema = self.schema();- if let Some(JoinData {- projected_left_child,- projected_right_child,- join_filter,- join_on,- }) = try_pushdown_through_join(- projection,- self.left(),- self.right(),- self.on(),- &schema,- self.filter(),- )? {+ if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)+ && let Some(JoinData {+ projected_left_child,+ projected_right_child,+ join_filter,+ join_on,+ }) = try_pushdown_through_join(+ projection,+ self.left(),+ self.right(),+ self.on(),+ &schema,+ self.filter(),+ )?+ {self.builder().with_new_children(vec![Arc::new(projected_left_child),diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex a4cea2c0cc..d13e172352 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -734,20 +734,23 @@ impl ExecutionPlan for NestedLoopJoinExec {return Ok(None);}+ // TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.let schema = self.schema();- if let Some(JoinData {- projected_left_child,- projected_right_child,- join_filter,- ..- }) = try_pushdown_through_join(- projection,- self.left(),- self.right(),- &[],- &schema,- self.filter(),- )? {+ if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)+ && let Some(JoinData {+ projected_left_child,+ projected_right_child,+ join_filter,+ ..+ }) = try_pushdown_through_join(+ projection,+ self.left(),+ self.right(),+ &[],+ &schema,+ self.filter(),+ )?+ {Ok(Some(Arc::new(NestedLoopJoinExec::try_new(Arc::new(projected_left_child),Arc::new(projected_right_child),diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex d305109a48..908ee6bb3b 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -1317,6 +1317,59 @@ physical_plan04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=105)------DataSourceExec: partitions=1, partition_sizes=[2]+query TT+explain select t1_id from t1+where t1_id > 40 or exists (select 1 from t2 where t2.t2_int = t1.t1_int)+----+logical_plan+01)Projection: t1.t1_id+02)--Filter: t1.t1_id > Int32(40) OR __correlated_sq_1.mark+03)----Projection: t1.t1_id, __correlated_sq_1.mark+04)------LeftMark Join: t1.t1_int = __correlated_sq_1.t2_int+05)--------TableScan: t1 projection=[t1_id, t1_int]+06)--------SubqueryAlias: __correlated_sq_1+07)----------TableScan: t2 projection=[t2_int]+physical_plan+01)FilterExec: t1_id@0 > 40 OR mark@1, projection=[t1_id@0]+02)--HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(t2_int@0, t1_int@1)], projection=[t1_id@0, mark@2]+03)----DataSourceExec: partitions=1, partition_sizes=[2]+04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)------DataSourceExec: partitions=1, partition_sizes=[2]++query I rowsort+select t1_id from t1+where t1_id > 40 or exists (select 1 from t2 where t2.t2_int = t1.t1_int)+----+11+33+44++query TT+explain select t1_id from t1+where t1_id > 40 or not exists (select 1 from t2 where t2.t2_int > t1.t1_int)+----+logical_plan+01)Projection: t1.t1_id+02)--Filter: t1.t1_id > Int32(40) OR NOT __correlated_sq_1.mark+03)----Projection: t1.t1_id, __correlated_sq_1.mark+04)------LeftMark Join: Filter: __correlated_sq_1.t2_int > t1.t1_int+05)--------TableScan: t1 projection=[t1_id, t1_int]+06)--------SubqueryAlias: __correlated_sq_1+07)----------TableScan: t2 projection=[t2_int]+physical_plan+01)FilterExec: t1_id@0 > 40 OR NOT mark@1, projection=[t1_id@0]+02)--NestedLoopJoinExec: join_type=RightMark, filter=t2_int@1 > t1_int@0, projection=[t1_id@0, mark@2]+03)----DataSourceExec: partitions=1, partition_sizes=[2]+04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+05)------DataSourceExec: partitions=1, partition_sizes=[2]++query I rowsort+select t1_id from t1+where t1_id > 40 or not exists (select 1 from t2 where t2.t2_int > t1.t1_int)+----+33+44+statement okset datafusion.explain.logical_plan_only = true;