Apache DataFusion SQL Query Engine
fix: ScalarSubqueryToJoin fails when CTE inside correlated scalar subquery (#24363)
Ruchir Tripathibb038a673909
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 9490af0e59..aab58c5519 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -350,15 +350,31 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {new_correlated_cols.insert(Column::new(Some(alias.alias.clone()), col.name.clone()));}++ let new_plan = if alias.input.schema().fields().len()+ != alias.schema.fields().len()+ {+ LogicalPlanBuilder::from((*alias.input).clone())+ .alias(alias.alias.clone())?+ .build()?+ } else {+ plan.clone()+ };+self.correlated_subquery_cols_map- .insert(plan.clone(), new_correlated_cols);+ .insert(new_plan.clone(), new_correlated_cols);if let Some(input_map) =self.collected_count_expr_map.get(alias.input.deref()){self.collected_count_expr_map- .insert(plan.clone(), input_map.clone());+ .insert(new_plan.clone(), input_map.clone());+ }++ if new_plan != plan {+ Ok(Transformed::yes(new_plan))+ } else {+ Ok(Transformed::no(plan))}- Ok(Transformed::no(plan))}LogicalPlan::Limit(limit) => {let input_expr_map = selfdiff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.sltindex dcca13c416..594964778b 100644--- a/datafusion/sqllogictest/test_files/subquery.slt+++ b/datafusion/sqllogictest/test_files/subquery.slt@@ -2657,3 +2657,30 @@ DROP TABLE nia_right_no_null;statement okreset datafusion.optimizer.prefer_hash_join;++# Regression test for #24139: CTE inside a correlated scalar subquery causing a schema error+statement ok+CREATE TABLE metrics (host VARCHAR, bytes_sent DOUBLE, ts BIGINT) AS VALUES+ ('a', 100.0, 1),+ ('b', 200.0, 2),+ ('a', 300.0, 3),+ ('b', 400.0, 4);++query TR+SELECT m.host, m.bytes_sent+FROM metrics m+WHERE m.bytes_sent > (+ WITH avg_per_host AS (+ SELECT avg(bytes_sent) AS avg_bytes+ FROM metrics+ WHERE host = m.host+ )+ SELECT avg_bytes FROM avg_per_host+)+ORDER BY m.host;+----+a 300+b 400++statement ok+DROP TABLE metrics;