Skip to content

Apache DataFusion SQL Query Engine

fix: ScalarSubqueryToJoin fails when CTE inside correlated scalar subquery (#24363)

Ruchir Tripathibb038a673909
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index 9490af0e59..aab58c5519 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -350,15 +350,31 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
new_correlated_cols
.insert(Column::new(Some(alias.alias.clone()), col.name.clone()));
}
+
+ let new_plan = if alias.input.schema().fields().len()
+ != alias.schema.fields().len()
+ {
+ LogicalPlanBuilder::from((*alias.input).clone())
+ .alias(alias.alias.clone())?
+ .build()?
+ } else {
+ plan.clone()
+ };
+
self.correlated_subquery_cols_map
- .insert(plan.clone(), new_correlated_cols);
+ .insert(new_plan.clone(), new_correlated_cols);
if let Some(input_map) =
self.collected_count_expr_map.get(alias.input.deref())
{
self.collected_count_expr_map
- .insert(plan.clone(), input_map.clone());
+ .insert(new_plan.clone(), input_map.clone());
+ }
+
+ if new_plan != plan {
+ Ok(Transformed::yes(new_plan))
+ } else {
+ Ok(Transformed::no(plan))
}
- Ok(Transformed::no(plan))
}
LogicalPlan::Limit(limit) => {
let input_expr_map = self
diff --git a/datafusion/sqllogictest/test_files/subquery.slt b/datafusion/sqllogictest/test_files/subquery.slt
index dcca13c416..594964778b 100644
--- a/datafusion/sqllogictest/test_files/subquery.slt
+++ b/datafusion/sqllogictest/test_files/subquery.slt
@@ -2657,3 +2657,30 @@ DROP TABLE nia_right_no_null;
statement ok
reset datafusion.optimizer.prefer_hash_join;
+
+# Regression test for #24139: CTE inside a correlated scalar subquery causing a schema error
+statement ok
+CREATE TABLE metrics (host VARCHAR, bytes_sent DOUBLE, ts BIGINT) AS VALUES
+ ('a', 100.0, 1),
+ ('b', 200.0, 2),
+ ('a', 300.0, 3),
+ ('b', 400.0, 4);
+
+query TR
+SELECT m.host, m.bytes_sent
+FROM metrics m
+WHERE m.bytes_sent > (
+ WITH avg_per_host AS (
+ SELECT avg(bytes_sent) AS avg_bytes
+ FROM metrics
+ WHERE host = m.host
+ )
+ SELECT avg_bytes FROM avg_per_host
+)
+ORDER BY m.host;
+----
+a 300
+b 400
+
+statement ok
+DROP TABLE metrics;