Skip to content

Apache DataFusion SQL Query Engine

fix: keep a correlated filter below the nullable side of an outer join (#25764)

Kita Renji871058cd7302
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index a288ded88b..4f89532e50 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -20,6 +20,7 @@
use std::collections::BTreeSet;
use std::sync::Arc;
+use crate::push_down_filter::lr_is_preserved;
use crate::simplify_expressions::ExprSimplifier;
use datafusion_common::tree_node::{
@@ -29,6 +30,7 @@ use datafusion_common::{
Column, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err, plan_err,
};
use datafusion_expr::expr::{Alias, GroupingSet};
+use datafusion_expr::logical_plan::Join;
use datafusion_expr::simplify::SimplifyContext;
use datafusion_expr::utils::{
collect_subquery_cols, conjunction, find_join_exprs, split_conjunction,
@@ -142,6 +144,17 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
LogicalPlan::Subquery(_) => {
Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump))
}
+ // A correlated filter can only move above a join from a side whose
+ // rows the join preserves. Below the side an outer join fills with
+ // NULLs, the filter decides which rows are unmatched, so pulling it
+ // above the join changes the result. The side a semi, anti or mark
+ // join does not output cannot give its columns to a pulled up
+ // filter either.
+ LogicalPlan::Join(ref join) if !correlated_inputs_are_preserved(join) => {
+ // the unsupported case
+ self.can_pull_up = false;
+ Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump))
+ }
LogicalPlan::Union(_) | LogicalPlan::Sort(_) | LogicalPlan::Extension(_) => {
let plan_hold_outer = !plan.all_out_ref_exprs().is_empty();
if plan_hold_outer {
@@ -535,6 +548,36 @@ impl PullUpCorrelatedExpr {
}
}
+/// Whether every input of `join` that holds outer references is a side whose
+/// rows the join preserves, so a correlated filter below it can be pulled above
+/// the join without changing the result.
+fn correlated_inputs_are_preserved(join: &Join) -> bool {
+ let (left_preserved, right_preserved) = lr_is_preserved(join.join_type);
+ (left_preserved || !holds_outer_reference(&join.left))
+ && (right_preserved || !holds_outer_reference(&join.right))
+}
+
+/// Whether `plan` or any of its inputs holds an outer reference of the scope
+/// being decorrelated. Like [`PullUpCorrelatedExpr`], this does not descend into
+/// a [`LogicalPlan::Subquery`], whose outer references belong to a nested
+/// scope, such as the right side of a `LATERAL` join that is not decorrelated
+/// yet.
+fn holds_outer_reference(plan: &LogicalPlan) -> bool {
+ let mut found = false;
+ plan.apply(|node| {
+ Ok(match node {
+ LogicalPlan::Subquery(_) => TreeNodeRecursion::Jump,
+ _ if node.contains_outer_reference() => {
+ found = true;
+ TreeNodeRecursion::Stop
+ }
+ _ => TreeNodeRecursion::Continue,
+ })
+ })
+ .expect("apply closure is infallible");
+ found
+}
+
fn can_pullup_over_aggregation(expr: &Expr) -> bool {
if let Expr::BinaryExpr(BinaryExpr {
left,
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index be0d4c57bb..d059b2ae7d 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -173,10 +173,14 @@ pub(crate) fn lr_is_preserved(join_type: JoinType) -> (bool, bool) {
JoinType::Right => (false, true),
JoinType::Full => (false, false),
// No columns from the right side of the join can be referenced in output
- // predicates for semi/anti joins, so whether we specify t/f doesn't matter.
+ // predicates for semi/anti joins. The right side must stay `false`:
+ // `PullUpCorrelatedExpr` uses it to refuse to pull a correlated filter
+ // out of a side that the join does not output.
JoinType::LeftSemi | JoinType::LeftAnti | JoinType::LeftMark => (true, false),
// No columns from the left side of the join can be referenced in output
- // predicates for semi/anti joins, so whether we specify t/f doesn't matter.
+ // predicates for semi/anti joins. The left side must stay `false`:
+ // `PullUpCorrelatedExpr` uses it to refuse to pull a correlated filter
+ // out of a side that the join does not output.
JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => (false, true),
}
}
diff --git a/datafusion/sqllogictest/test_files/lateral_join.slt b/datafusion/sqllogictest/test_files/lateral_join.slt
index cae3e67153..02d4d9f0ea 100644
--- a/datafusion/sqllogictest/test_files/lateral_join.slt
+++ b/datafusion/sqllogictest/test_files/lateral_join.slt
@@ -739,6 +739,20 @@ FROM t1, LATERAL (
) AS sub
ORDER BY t1.id;
+# --- Correlated filter on the nullable side of an outer join ---
+# https://github.com/apache/datafusion/issues/25507
+# Pulling `t2.t1_id = t1.id` above the LEFT JOIN would drop the t3 rows that
+# have no match instead of extending them with NULLs.
+# Expected result once supported: every t3 row for every t1 row (9 rows),
+# with y set only where t2.t1_id = t1.id.
+statement error OuterReferenceColumn
+SELECT t1.id, sub.x, sub.y
+FROM t1, LATERAL (
+ SELECT t3.id AS x, t2.t1_id AS y
+ FROM t3 LEFT JOIN (SELECT * FROM t2 WHERE t2.t1_id = t1.id) AS t2 ON t3.t2_id = t2.id
+) AS sub
+ORDER BY t1.id, sub.x;
+
# --- Implicit LATERAL (comma without LATERAL keyword) ---
# DuckDB auto-detects lateral correlation; DataFusion requires
# the explicit LATERAL keyword.
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index f383f66176..d27c244e0d 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2848,3 +2848,177 @@ DROP TABLE gs_outer;
statement ok
DROP TABLE gs_inner;
+
+# Regression test for #25507: a correlated filter below the side of an outer
+# join that the join fills with NULLs must not be pulled above the join. The
+# filter decides which rows of the other side are unmatched, so once it is
+# above the join the rows that get NULLs are different.
+statement ok
+CREATE TABLE oj_outer(k INT) AS VALUES (1), (5);
+
+statement ok
+CREATE TABLE oj_a(id INT) AS VALUES (1), (2);
+
+statement ok
+CREATE TABLE oj_b(id INT, y INT) AS VALUES (1, 1), (2, 2);
+
+statement ok
+set datafusion.explain.logical_plan_only = true;
+
+# The correlated filter is on the right side of a LEFT JOIN, so the subquery
+# stays correlated.
+query TT
+EXPLAIN SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+----
+logical_plan
+01)Projection: oj_outer.k, EXISTS (<subquery>) AS e
+02)--Subquery:
+03)----Projection: Int64(1)
+04)------Filter: b.y IS NULL
+05)--------Left Join: oj_a.id = b.id
+06)----------TableScan: oj_a
+07)----------SubqueryAlias: b
+08)------------Projection: oj_b.id, oj_b.y
+09)--------------Filter: oj_b.y = outer_ref(oj_outer.k)
+10)----------------TableScan: oj_b
+11)--TableScan: oj_outer projection=[k]
+
+# It errors rather than returning wrong results. The right answer is
+# `(1, true), (5, true)`.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+
+# The same for IN. The right answer is `(1, true), (5, NULL)`.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery
+SELECT oj_outer.k, oj_outer.k IN (SELECT b.y FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id) AS m FROM oj_outer;
+
+# And for EXISTS and NOT EXISTS in WHERE. `oj_a` is never empty, so the right
+# answers are every row and no row.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k FROM oj_outer WHERE EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id);
+
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k FROM oj_outer WHERE NOT EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id);
+
+# And for a scalar subquery. The right answer is `(1, 1), (5, 2)`.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression ScalarSubquery
+SELECT oj_outer.k, (SELECT count(*) FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y IS NULL) AS c FROM oj_outer;
+
+# The left side of a RIGHT JOIN, and either side of a FULL JOIN.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b RIGHT JOIN oj_a ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b FULL JOIN oj_a ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a FULL JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+
+# The filter can sit deeper below the nullable side, here under an inner join.
+statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT oj_b.id, oj_b.y FROM oj_b JOIN oj_a AS a2 ON oj_b.id = a2.id WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y IS NULL) AS e FROM oj_outer;
+
+# A correlated filter on the preserved side of a LEFT JOIN is still pulled up.
+query TT
+EXPLAIN SELECT oj_outer.k, EXISTS (SELECT 1 FROM (SELECT * FROM oj_a WHERE oj_a.id = oj_outer.k) AS a LEFT JOIN oj_b ON a.id = oj_b.id WHERE oj_b.y IS NULL) AS e FROM oj_outer;
+----
+logical_plan
+01)Projection: oj_outer.k, __correlated_sq_1.mark AS e
+02)--LeftMark Join: oj_outer.k = __correlated_sq_1.id
+03)----TableScan: oj_outer projection=[k]
+04)----SubqueryAlias: __correlated_sq_1
+05)------Projection: a.id
+06)--------Filter: oj_b.y IS NULL
+07)----------Projection: a.id, oj_b.y
+08)------------Left Join: a.id = oj_b.id
+09)--------------SubqueryAlias: a
+10)----------------TableScan: oj_a projection=[id]
+11)--------------TableScan: oj_b projection=[id, y]
+
+query IB
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM (SELECT * FROM oj_a WHERE oj_a.id = oj_outer.k) AS a LEFT JOIN oj_b ON a.id = oj_b.id) AS e FROM oj_outer ORDER BY oj_outer.k;
+----
+1 true
+5 false
+
+# So is one on the preserved side of a RIGHT JOIN.
+query IB
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_b RIGHT JOIN (SELECT * FROM oj_a WHERE oj_a.id = oj_outer.k) AS a ON a.id = oj_b.id) AS e FROM oj_outer ORDER BY oj_outer.k;
+----
+1 true
+5 false
+
+# And one on either side of an inner join.
+query TT
+EXPLAIN SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id) AS e FROM oj_outer;
+----
+logical_plan
+01)Projection: oj_outer.k, __correlated_sq_1.mark AS e
+02)--LeftMark Join: oj_outer.k = __correlated_sq_1.y
+03)----TableScan: oj_outer projection=[k]
+04)----SubqueryAlias: __correlated_sq_1
+05)------Projection: b.y
+06)--------RightSemi Join: oj_a.id = b.id
+07)----------TableScan: oj_a projection=[id]
+08)----------SubqueryAlias: b
+09)------------TableScan: oj_b projection=[id, y]
+
+query IB
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id) AS e FROM oj_outer ORDER BY oj_outer.k;
+----
+1 true
+5 false
+
+# A correlated filter above the outer join is pulled up as before, even when it
+# reads a column of the nullable side.
+query TT
+EXPLAIN SELECT oj_outer.k FROM oj_outer WHERE EXISTS (SELECT 1 FROM oj_a LEFT JOIN oj_b ON oj_a.id = oj_b.id WHERE oj_b.y = oj_outer.k);
+----
+logical_plan
+01)LeftSemi Join: oj_outer.k = __correlated_sq_1.y
+02)--TableScan: oj_outer projection=[k]
+03)--SubqueryAlias: __correlated_sq_1
+04)----Projection: oj_b.y
+05)------Left Join: oj_a.id = oj_b.id
+06)--------TableScan: oj_a projection=[id]
+07)--------TableScan: oj_b projection=[id, y]
+
+query I
+SELECT oj_outer.k FROM oj_outer WHERE EXISTS (SELECT 1 FROM oj_a LEFT JOIN oj_b ON oj_a.id = oj_b.id WHERE oj_b.y = oj_outer.k) ORDER BY oj_outer.k;
+----
+1
+
+# A null-rejecting filter above the outer join makes it an inner join, so the
+# subquery is still decorrelated (in a later optimizer pass).
+query TT
+EXPLAIN SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y > 0) AS e FROM oj_outer;
+----
+logical_plan
+01)Projection: oj_outer.k, __correlated_sq_2.mark AS e
+02)--LeftMark Join: oj_outer.k = __correlated_sq_2.y
+03)----TableScan: oj_outer projection=[k]
+04)----SubqueryAlias: __correlated_sq_2
+05)------Projection: b.y
+06)--------RightSemi Join: oj_a.id = b.id
+07)----------TableScan: oj_a projection=[id]
+08)----------SubqueryAlias: b
+09)------------Filter: oj_b.y > Int32(0)
+10)--------------TableScan: oj_b projection=[id, y]
+
+query IB
+SELECT oj_outer.k, EXISTS (SELECT 1 FROM oj_a LEFT JOIN (SELECT * FROM oj_b WHERE oj_b.y = oj_outer.k) AS b ON oj_a.id = b.id WHERE b.y > 0) AS e FROM oj_outer ORDER BY oj_outer.k;
+----
+1 true
+5 false
+
+statement ok
+RESET datafusion.explain.logical_plan_only;
+
+statement ok
+DROP TABLE oj_outer;
+
+statement ok
+DROP TABLE oj_a;
+
+statement ok
+DROP TABLE oj_b;