Apache DataFusion SQL Query Engine
fix(optimizer): preserve correlated columns when removing filter above SubqueryAlias (#26551)
Prince Jain836b4dda4fb2
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex af5de0b29a..246639b389 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -494,7 +494,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {LogicalPlanBuilder::from((*plan_filter.input).clone()).build()?;self.correlated_subquery_cols_map- .insert(new_plan.clone(), correlated_subquery_cols);+ .entry(new_plan.clone())+ .or_default()+ .extend(correlated_subquery_cols);Ok(Transformed::yes(new_plan))}(None, _) => {@@ -506,7 +508,9 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {}let new_plan = plan.build()?;self.correlated_subquery_cols_map- .insert(new_plan.clone(), correlated_subquery_cols);+ .entry(new_plan.clone())+ .or_default()+ .extend(correlated_subquery_cols);Ok(Transformed::yes(new_plan))}}diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rsindex febbd3a6c8..732683da5d 100644--- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs+++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs@@ -3507,4 +3507,38 @@ mod tests {")}++ #[test]+ fn exists_subquery_filter_above_correlated_derived_table() -> Result<()> {+ let sq_scan = test_table_scan_with_name("sq")?;+ let derived = LogicalPlanBuilder::from(sq_scan)+ .filter(col("sq.c").eq(out_ref_col(DataType::UInt32, "test.a")))?+ .project(vec![col("sq.c")])?+ .alias("t")?+ .build()?;+ let subquery = LogicalPlanBuilder::from(derived)+ .filter(out_ref_col(DataType::UInt32, "test.b").gt(lit(0u32)))?+ .project(vec![lit(1)])?+ .build()?;++ let outer_scan = test_table_scan()?;+ let plan = LogicalPlanBuilder::from(outer_scan)+ .filter(exists(Arc::new(subquery)))?+ .project(vec![col("test.a")])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: test.a [a:UInt32]+ LeftSemi Join: Filter: __correlated_sq_1.c = test.a AND test.b > UInt32(0) [a:UInt32, b:UInt32, c:UInt32]+ TableScan: test [a:UInt32, b:UInt32, c:UInt32]+ SubqueryAlias: __correlated_sq_1 [Int32(1):Int32, c:UInt32]+ Projection: Int32(1), t.c [Int32(1):Int32, c:UInt32]+ SubqueryAlias: t [c:UInt32]+ Projection: sq.c [c:UInt32]+ TableScan: sq [a:UInt32, b:UInt32, c:UInt32]+ "+ )+ }}diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rsindex 0a1ea917ce..934424f20f 100644--- a/datafusion/optimizer/src/scalar_subquery_to_join.rs+++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs@@ -1276,4 +1276,46 @@ mod tests {Ok(())}++ #[test]+ fn correlated_scalar_subquery_filter_above_correlated_derived_table() -> Result<()> {+ let derived = LogicalPlanBuilder::from(scan_tpch_table("orders"))+ .filter(+ col("orders.o_custkey")+ .eq(out_ref_col(DataType::Int64, "customer.c_custkey")),+ )?+ .aggregate(+ Vec::<Expr>::new(),+ vec![max(col("orders.o_custkey")).alias("n")],+ )?+ .project(vec![col("n")])?+ .alias("t")?+ .build()?;++ let sq = Arc::new(+ LogicalPlanBuilder::from(derived)+ .filter(out_ref_col(DataType::Int64, "customer.c_custkey").gt(lit(0i64)))?+ .project(vec![col("t.n")])?+ .build()?,+ );++ let plan = LogicalPlanBuilder::from(scan_tpch_table("customer"))+ .project(vec![col("customer.c_custkey"), scalar_subquery(sq)])?+ .build()?;++ assert_optimized_plan_equal!(+ plan,+ @r"+ Projection: customer.c_custkey, __scalar_sq_1.n AS n [c_custkey:Int64, n:Int64;N]+ Left Join: Filter: __scalar_sq_1.o_custkey = customer.c_custkey AND customer.c_custkey > Int64(0) [c_custkey:Int64, c_name:Utf8, n:Int64;N, o_custkey:Int64;N, __always_true:Boolean;N]+ TableScan: customer [c_custkey:Int64, c_name:Utf8]+ SubqueryAlias: __scalar_sq_1 [n:Int64;N, o_custkey:Int64, __always_true:Boolean]+ Projection: t.n, t.o_custkey, t.__always_true [n:Int64;N, o_custkey:Int64, __always_true:Boolean]+ SubqueryAlias: t [n:Int64;N, o_custkey:Int64, __always_true:Boolean]+ Projection: n, orders.o_custkey, __always_true [n:Int64;N, o_custkey:Int64, __always_true:Boolean]+ Aggregate: groupBy=[[orders.o_custkey, Boolean(true) AS __always_true]], aggr=[[max(orders.o_custkey) AS n]] [o_custkey:Int64, __always_true:Boolean, n:Int64;N]+ TableScan: orders [o_orderkey:Int64, o_custkey:Int64, o_orderstatus:Utf8, o_totalprice:Float64;N]+ "+ )+ }}diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex d4a42cca0c..fba2de5ac9 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -4195,3 +4195,23 @@ DROP TABLE emp;statement okRESET datafusion.explain.logical_plan_only;++# Regression test for #26504: correlated subquery filtering an aliased derived+# table on an outer column must preserve the derived table's correlated columns.+query I+SELECT * FROM (VALUES (1)) AS x(d)+WHERE EXISTS (SELECT 1 FROM (SELECT * FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0);+----+1++query I+SELECT (SELECT n FROM (SELECT sum(y.c) AS n FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0) AS v+FROM (VALUES (1)) AS x(d);+----+1++query I+SELECT * FROM (VALUES (1)) AS x(d)+WHERE x.d IN (SELECT t.c FROM (SELECT * FROM (VALUES (1)) AS y(c) WHERE y.c = x.d) AS t WHERE x.d > 0);+----+1