Skip to content

Apache DataFusion SQL Query Engine

fix(optimizer): preserve correlated columns when removing filter above SubqueryAlias (#26551)

Prince Jain836b4dda4fb2
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index af5de0b29a..246639b389 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -494,7 +494,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
LogicalPlanBuilder::from((*plan_filter.input).clone())
.build()?;
self.correlated_subquery_cols_map
- .insert(new_plan.clone(), correlated_subquery_cols);
+ .entry(new_plan.clone())
+ .or_default()
+ .extend(correlated_subquery_cols);
Ok(Transformed::yes(new_plan))
}
(None, _) => {
@@ -506,7 +508,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
}
let new_plan = plan.build()?;
self.correlated_subquery_cols_map
- .insert(new_plan.clone(), correlated_subquery_cols);
+ .entry(new_plan.clone())
+ .or_default()
+ .extend(correlated_subquery_cols);
Ok(Transformed::yes(new_plan))
}
}
diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
index febbd3a6c8..732683da5d 100644
--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs
@@ -3507,4 +3507,38 @@ mod tests {
"
)
}
+
+ #[test]
+ fn exists_subquery_filter_above_correlated_derived_table() -> Result<()> {
+ let sq_scan = test_table_scan_with_name("sq")?;
+ let derived = LogicalPlanBuilder::from(sq_scan)
+ .filter(col("sq.c").eq(out_ref_col(DataType::UInt32, "test.a")))?
+ .project(vec![col("sq.c")])?
+ .alias("t")?
+ .build()?;
+ let subquery = LogicalPlanBuilder::from(derived)
+ .filter(out_ref_col(DataType::UInt32, "test.b").gt(lit(0u32)))?
+ .project(vec![lit(1)])?
+ .build()?;
+
+ let outer_scan = test_table_scan()?;
+ let plan = LogicalPlanBuilder::from(outer_scan)
+ .filter(exists(Arc::new(subquery)))?
+ .project(vec![col("test.a")])?
+ .build()?;
+
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: test.a [a:UInt32]
+ LeftSemi Join: Filter: __correlated_sq_1.c = test.a AND test.b > UInt32(0) [a:UInt32, b:UInt32, c:UInt32]
+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]
+ SubqueryAlias: __correlated_sq_1 [Int32(1):Int32, c:UInt32]
+ Projection: Int32(1), t.c [Int32(1):Int32, c:UInt32]
+ SubqueryAlias: t [c:UInt32]
+ Projection: sq.c [c:UInt32]
+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]
+ "
+ )
+ }
}
diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rs
index 0a1ea917ce..934424f20f 100644
--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs
+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs
@@ -1276,4 +1276,46 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn correlated_scalar_subquery_filter_above_correlated_derived_table() -> Result<()> {
+ let derived = LogicalPlanBuilder::from(scan_tpch_table("orders"))
+ .filter(
+ col("orders.o_custkey")
+ .eq(out_ref_col(DataType::Int64, "customer.c_custkey")),
+ )?
+ .aggregate(
+ Vec::<Expr>::new(),
+ vec![max(col("orders.o_custkey")).alias("n")],
+ )?
+ .project(vec![col("n")])?
+ .alias("t")?
+ .build()?;
+
+ let sq = Arc::new(
+ LogicalPlanBuilder::from(derived)
+ .filter(out_ref_col(DataType::Int64, "customer.c_custkey").gt(lit(0i64)))?
+ .project(vec![col("t.n")])?
+ .build()?,
+ );
+
+ let plan = LogicalPlanBuilder::from(scan_tpch_table("customer"))
+ .project(vec![col("customer.c_custkey"), scalar_subquery(sq)])?
+ .build()?;
+
+ assert_optimized_plan_equal!(
+ plan,
+ @r"
+ Projection: customer.c_custkey, __scalar_sq_1.n AS n [c_custkey:Int64, n:Int64;N]
+ Left Join: Filter: __scalar_sq_1.o_custkey = customer.c_custkey AND customer.c_custkey > Int64(0) [c_custkey:Int64, c_name:Utf8, n:Int64;N, o_custkey:Int64;N, __always_true:Boolean;N]
+ TableScan: customer [c_custkey:Int64, c_name:Utf8]
+ SubqueryAlias: __scalar_sq_1 [n:Int64;N, o_custkey:Int64, __always_true:Boolean]
+ Projection: t.n, t.o_custkey, t.__always_true [n:Int64;N, o_custkey:Int64, __always_true:Boolean]
+ SubqueryAlias: t [n:Int64;N, o_custkey:Int64, __always_true:Boolean]
+ Projection: n, orders.o_custkey, __always_true [n:Int64;N, o_custkey:Int64, __always_true:Boolean]
+ Aggregate: groupBy=[[orders.o_custkey, Boolean(true) AS __always_true]], aggr=[[max(orders.o_custkey) AS n]] [o_custkey:Int64, __always_true:Boolean, n:Int64;N]
+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]
+ "
+ )
+ }
}
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index d4a42cca0c..fba2de5ac9 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -4195,3 +4195,23 @@ DROP TABLE emp;
statement ok
RESET datafusion.explain.logical_plan_only;
+
+# Regression test for #26504: correlated subquery filtering an aliased derived
+# table on an outer column must preserve the derived table's correlated columns.
+query I
+SELECT * FROM (VALUES (1)) AS x(d)
+WHERE EXISTS (SELECT 1 FROM (SELECT * FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0);
+----
+1
+
+query I
+SELECT (SELECT n FROM (SELECT sum(y.c) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0) AS v
+FROM (VALUES (1)) AS x(d);
+----
+1
+
+query I
+SELECT * FROM (VALUES (1)) AS x(d)
+WHERE x.d IN (SELECT t.c FROM (SELECT * FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0);
+----
+1