Skip to content

Apache DataFusion SQL Query Engine

fix: ProjectionPushdown internal error on NestedLoopJoin mark joins (#22902)

linfeng283648001bbc
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index aa624e050c..1994f9d74c 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -1520,20 +1520,23 @@ impl ExecutionPlan for HashJoinExec {
return Ok(None);
}
+ // TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.
let schema = self.schema();
- if let Some(JoinData {
- projected_left_child,
- projected_right_child,
- join_filter,
- join_on,
- }) = try_pushdown_through_join(
- projection,
- self.left(),
- self.right(),
- self.on(),
- &schema,
- self.filter(),
- )? {
+ if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)
+ && let Some(JoinData {
+ projected_left_child,
+ projected_right_child,
+ join_filter,
+ join_on,
+ }) = try_pushdown_through_join(
+ projection,
+ self.left(),
+ self.right(),
+ self.on(),
+ &schema,
+ self.filter(),
+ )?
+ {
self.builder()
.with_new_children(vec![
Arc::new(projected_left_child),
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index a4cea2c0cc..d13e172352 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -734,20 +734,23 @@ impl ExecutionPlan for NestedLoopJoinExec {
return Ok(None);
}
+ // TODO: split by `col`/`JoinSide` instead so mark joins can also push down to children.
let schema = self.schema();
- if let Some(JoinData {
- projected_left_child,
- projected_right_child,
- join_filter,
- ..
- }) = try_pushdown_through_join(
- projection,
- self.left(),
- self.right(),
- &[],
- &schema,
- self.filter(),
- )? {
+ if !matches!(self.join_type(), JoinType::LeftMark | JoinType::RightMark)
+ && let Some(JoinData {
+ projected_left_child,
+ projected_right_child,
+ join_filter,
+ ..
+ }) = try_pushdown_through_join(
+ projection,
+ self.left(),
+ self.right(),
+ &[],
+ &schema,
+ self.filter(),
+ )?
+ {
Ok(Some(Arc::new(NestedLoopJoinExec::try_new(
Arc::new(projected_left_child),
Arc::new(projected_right_child),
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index d305109a48..908ee6bb3b 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -1317,6 +1317,59 @@ physical_plan
04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
05)------DataSourceExec: partitions=1, partition_sizes=[2]
+query TT
+explain select t1_id from t1
+where t1_id > 40 or exists (select 1 from t2 where t2.t2_int = t1.t1_int)
+----
+logical_plan
+01)Projection: t1.t1_id
+02)--Filter: t1.t1_id > Int32(40) OR __correlated_sq_1.mark
+03)----Projection: t1.t1_id, __correlated_sq_1.mark
+04)------LeftMark Join: t1.t1_int = __correlated_sq_1.t2_int
+05)--------TableScan: t1 projection=[t1_id, t1_int]
+06)--------SubqueryAlias: __correlated_sq_1
+07)----------TableScan: t2 projection=[t2_int]
+physical_plan
+01)FilterExec: t1_id@0 > 40 OR mark@1, projection=[t1_id@0]
+02)--HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(t2_int@0, t1_int@1)], projection=[t1_id@0, mark@2]
+03)----DataSourceExec: partitions=1, partition_sizes=[2]
+04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+05)------DataSourceExec: partitions=1, partition_sizes=[2]
+
+query I rowsort
+select t1_id from t1
+where t1_id > 40 or exists (select 1 from t2 where t2.t2_int = t1.t1_int)
+----
+11
+33
+44
+
+query TT
+explain select t1_id from t1
+where t1_id > 40 or not exists (select 1 from t2 where t2.t2_int > t1.t1_int)
+----
+logical_plan
+01)Projection: t1.t1_id
+02)--Filter: t1.t1_id > Int32(40) OR NOT __correlated_sq_1.mark
+03)----Projection: t1.t1_id, __correlated_sq_1.mark
+04)------LeftMark Join: Filter: __correlated_sq_1.t2_int > t1.t1_int
+05)--------TableScan: t1 projection=[t1_id, t1_int]
+06)--------SubqueryAlias: __correlated_sq_1
+07)----------TableScan: t2 projection=[t2_int]
+physical_plan
+01)FilterExec: t1_id@0 > 40 OR NOT mark@1, projection=[t1_id@0]
+02)--NestedLoopJoinExec: join_type=RightMark, filter=t2_int@1 > t1_int@0, projection=[t1_id@0, mark@2]
+03)----DataSourceExec: partitions=1, partition_sizes=[2]
+04)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+05)------DataSourceExec: partitions=1, partition_sizes=[2]
+
+query I rowsort
+select t1_id from t1
+where t1_id > 40 or not exists (select 1 from t2 where t2.t2_int > t1.t1_int)
+----
+33
+44
+
statement ok
set datafusion.explain.logical_plan_only = true;