Apache DataFusion SQL Query Engine
fix: Optimize projections in recursive CTEs (#22476)
Nuno Faria710e9295eae6
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex b9f22a3f9e..acdbf71d05 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -35,9 +35,7 @@ use datafusion_expr::{use crate::optimize_projections::required_indices::RequiredIndices;use crate::utils::NamePreserver;-use datafusion_common::tree_node::{- Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion,-};+use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeContainer};/// Optimizer rule to prune unnecessary columns from intermediate schemas/// inside the [`LogicalPlan`]. This rule:@@ -373,29 +371,15 @@ fn optimize_projections(// These operators have no inputs, so stop the optimization process.return Ok(Transformed::no(plan));}- LogicalPlan::RecursiveQuery(recursive) => {- // Only allow subqueries that reference the current CTE; nested subqueries are not yet- // supported for projection pushdown for simplicity.- // TODO: be able to do projection pushdown on recursive CTEs with subqueries- if plan_contains_other_subqueries(- recursive.static_term.as_ref(),- &recursive.name,- ) || plan_contains_other_subqueries(- recursive.recursive_term.as_ref(),- &recursive.name,- ) {- return Ok(Transformed::no(plan));- }-- plan.inputs()- .into_iter()- .map(|input| {- indices- .clone()- .with_projection_beneficial()- .with_plan_exprs(&plan, input.schema())- })- .collect::<Result<Vec<_>>>()?+ LogicalPlan::RecursiveQuery(_) => {+ // optimize the static and recursive terms: treat each recursive CTE term like a+ // standalone subquery: optimize its internals, but do not push parent required indices+ // through the RecursiveQuery boundary, as this can otherwise lead to bugs+ // (see: https://github.com/apache/datafusion/issues/22249)+ return plan.map_children(|c| {+ let indices = RequiredIndices::new_for_all_exprs(&c);+ optimize_projections(c, config, indices)+ });}LogicalPlan::Join(join) => {let left_len = join.left.schema().fields().len();@@ -892,64 +876,6 @@ pub fn is_projection_unnecessary())}-/// Returns true if the plan subtree contains any subqueries that are not the-/// CTE reference itself. This treats any non-CTE [`LogicalPlan::SubqueryAlias`]-/// node (including aliased relations) as a blocker, along with expression-level-/// subqueries like scalar, EXISTS, or IN. These cases prevent projection-/// pushdown for now because we cannot safely reason about their column usage.-fn plan_contains_other_subqueries(plan: &LogicalPlan, cte_name: &str) -> bool {- if let LogicalPlan::SubqueryAlias(alias) = plan- && alias.alias.table() != cte_name- && !subquery_alias_targets_recursive_cte(alias.input.as_ref(), cte_name)- {- return true;- }-- let mut found = false;- plan.apply_expressions(|expr| {- if expr_contains_subquery(expr) {- found = true;- Ok(TreeNodeRecursion::Stop)- } else {- Ok(TreeNodeRecursion::Continue)- }- })- .expect("expression traversal never fails");- if found {- return true;- }-- plan.inputs()- .into_iter()- .any(|child| plan_contains_other_subqueries(child, cte_name))-}--fn expr_contains_subquery(expr: &Expr) -> bool {- expr.exists(|e| match e {- Expr::ScalarSubquery(_) | Expr::Exists(_) | Expr::InSubquery(_) => Ok(true),- _ => Ok(false),- })- // Safe unwrap since we are doing a simple boolean check- .unwrap()-}--fn subquery_alias_targets_recursive_cte(plan: &LogicalPlan, cte_name: &str) -> bool {- match plan {- LogicalPlan::TableScan(scan) => scan.table_name.table() == cte_name,- LogicalPlan::SubqueryAlias(alias) => {- subquery_alias_targets_recursive_cte(alias.input.as_ref(), cte_name)- }- _ => {- let inputs = plan.inputs();- if inputs.len() == 1 {- subquery_alias_targets_recursive_cte(inputs[0], cte_name)- } else {- false- }- }- }-}-#[cfg(test)]mod tests {use std::cmp::Ordering;diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rsindex a3c5ab7aa3..6fad39dc33 100644--- a/datafusion/optimizer/tests/optimizer_integration.rs+++ b/datafusion/optimizer/tests/optimizer_integration.rs@@ -56,8 +56,7 @@ fn init() {#[test]fn recursive_cte_with_nested_subquery() -> Result<()> {- // Covers bailout path in `plan_contains_other_subqueries`, ensuring nested subqueries- // within recursive CTE branches prevent projection pushdown.+ // projection optimization is applied to recursive CTEs even with nested subquerieslet sql = r#"WITH RECURSIVE numbers(id, level) AS (SELECT sub.id, sub.level FROM (@@ -79,17 +78,16 @@ fn recursive_cte_with_nested_subquery() -> Result<()> {SubqueryAlias: numbersProjection: sub.id AS id, sub.level AS levelRecursiveQuery: is_distinct=false- Projection: sub.id, sub.level- SubqueryAlias: sub- Projection: test.col_int32 AS id, Int64(1) AS level- TableScan: test+ SubqueryAlias: sub+ Projection: test.col_int32 AS id, Int64(1) AS level+ TableScan: test projection=[col_int32]Projection: t.col_int32, numbers.level + Int64(1)Inner Join: CAST(t.col_int32 AS Int64) = CAST(numbers.id AS Int64) + Int64(1)SubqueryAlias: tFilter: CAST(test.col_int32 AS Int64) IS NOT NULL- TableScan: test+ TableScan: test projection=[col_int32]Filter: CAST(numbers.id AS Int64) + Int64(1) IS NOT NULL- TableScan: numbers+ TableScan: numbers projection=[id, level]");@@ -527,12 +525,10 @@ fn select_correlated_predicate_subquery_with_uppercase_ident() {");}-#[test]-fn recursive_cte_projection_pushdown() -> Result<()> {- // Test that projection pushdown works with recursive CTEs by ensuring- // only the required columns are projected from the base table, even when- // the CTE definition includes unused columns+fn recursive_cte_outer_projection_pushdown() -> Result<()> {+ // projection optimization of a recursive CTE based on the outer query's projected columns is+ // not done as this can lead to bugs (see: https://github.com/apache/datafusion/issues/22249).let sql = "WITH RECURSIVE nodes AS (\SELECT col_int32 AS id, col_utf8 AS name, col_uint32 AS extra FROM test \UNION ALL \@@ -540,18 +536,19 @@ fn recursive_cte_projection_pushdown() -> Result<()> {) SELECT id FROM nodes";let plan = test_sql(sql)?;- // The optimizer successfully performs projection pushdown by only selecting the needed- // columns from the base table and recursive table, eliminating unused columns+ // col_int32, col_utf8, and col_uint32 and projected from test since they are used in the+ // recursive CTE, even though the outer query only requires col_int32assert_snapshot!(format!("{plan}"),@r"SubqueryAlias: nodes- RecursiveQuery: is_distinct=false- Projection: test.col_int32 AS id- TableScan: test projection=[col_int32]- Projection: CAST(CAST(nodes.id AS Int64) + Int64(1) AS Int32)- Filter: nodes.id < Int32(3)- TableScan: nodes projection=[id]+ Projection: id+ RecursiveQuery: is_distinct=false+ Projection: test.col_int32 AS id, test.col_utf8 AS name, test.col_uint32 AS extra+ TableScan: test projection=[col_int32, col_uint32, col_utf8]+ Projection: CAST(CAST(nodes.id AS Int64) + Int64(1) AS Int32), nodes.name, nodes.extra+ Filter: nodes.id < Int32(3)+ TableScan: nodes projection=[id, name, extra]");Ok(())@@ -570,47 +567,19 @@ fn recursive_cte_with_aliased_self_reference() -> Result<()> {format!("{plan}"),@r"SubqueryAlias: nodes- RecursiveQuery: is_distinct=false- Projection: test.col_int32 AS id- TableScan: test projection=[col_int32]- Projection: CAST(CAST(child.id AS Int64) + Int64(1) AS Int32)- SubqueryAlias: child- Filter: nodes.id < Int32(3)- TableScan: nodes projection=[id]+ Projection: id+ RecursiveQuery: is_distinct=false+ Projection: test.col_int32 AS id, test.col_utf8 AS name+ TableScan: test projection=[col_int32, col_utf8]+ Projection: CAST(CAST(child.id AS Int64) + Int64(1) AS Int32), child.name+ SubqueryAlias: child+ Filter: nodes.id < Int32(3)+ TableScan: nodes projection=[id, name]",);Ok(())}-#[test]-fn recursive_cte_with_unused_columns() -> Result<()> {- // Test projection pushdown with a recursive CTE where the base case- // includes columns that are never used in the recursive part or final result- let sql = "WITH RECURSIVE series AS (\- SELECT 1 AS n, col_utf8, col_uint32, col_date32 FROM test WHERE col_int32 = 1 \- UNION ALL \- SELECT n + 1, col_utf8, col_uint32, col_date32 FROM series WHERE n < 3\- ) SELECT n FROM series";- let plan = test_sql(sql)?;-- // The optimizer successfully performs projection pushdown by eliminating unused columns- // even when they're defined in the CTE but not actually needed- assert_snapshot!(- format!("{plan}"),- @r"- SubqueryAlias: series- RecursiveQuery: is_distinct=false- Projection: Int64(1) AS n- Filter: test.col_int32 = Int32(1)- TableScan: test projection=[col_int32]- Projection: series.n + Int64(1)- Filter: series.n < Int64(3)- TableScan: series projection=[n]- "- );- Ok(())-}-#[test]/// Asserts the minimal plan shape once projection pushdown succeeds for a recursive CTE./// Unlike the previous two tests that retain extra columns in either the base or recursivediff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex 8d85139766..0b93f6fc10 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -842,12 +842,12 @@ logical_plan03)----Projection: Int64(1) AS val04)------EmptyRelation: rows=105)----Projection: Int64(2) AS val-06)------Cross Join:-07)--------Filter: recursive_cte.val < Int64(2)-08)----------TableScan: recursive_cte-09)--------SubqueryAlias: sub_cte-10)----------Projection: Int64(2) AS val-11)------------EmptyRelation: rows=1+06)------Cross Join:+07)--------Projection:+08)----------Filter: recursive_cte.val < Int64(2)+09)------------TableScan: recursive_cte projection=[val]+10)--------SubqueryAlias: sub_cte+11)----------EmptyRelation: rows=1physical_plan01)RecursiveQueryExec: name=recursive_cte, is_distinct=false02)--ProjectionExec: expr=[1 as val]@@ -855,11 +855,10 @@ physical_plan04)--ProjectionExec: expr=[2 as val]05)----CrossJoinExec06)------CoalescePartitionsExec-07)--------FilterExec: val@0 < 2+07)--------FilterExec: val@0 < 2, projection=[]08)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=109)------------WorkTableExec: name=recursive_cte-10)------ProjectionExec: expr=[2 as val]-11)--------PlaceholderRowExec+10)------PlaceholderRowExec# Test issue: https://github.com/apache/datafusion/issues/9794# Non-recursive term and recursive term have different types@@ -1205,14 +1204,13 @@ EXPLAIN WITH RECURSIVE trans AS (logical_plan01)SubqueryAlias: trans02)--RecursiveQuery: is_distinct=true-03)----Projection: closure.start, closure.end-04)------TableScan: closure-05)----Projection: l.start, r.end-06)------Inner Join: l.end = r.start-07)--------SubqueryAlias: l-08)----------TableScan: trans-09)--------SubqueryAlias: r-10)----------TableScan: closure+03)----TableScan: closure projection=[start, end]+04)----Projection: l.start, r.end+05)------Inner Join: l.end = r.start+06)--------SubqueryAlias: l+07)----------TableScan: trans projection=[start, end]+08)--------SubqueryAlias: r+09)----------TableScan: closure projection=[start, end]physical_plan01)RecursiveQueryExec: name=trans, is_distinct=true02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/recursive_cte/closure.csv]]}, projection=[start, end], file_type=csv, has_header=true@@ -1499,3 +1497,128 @@ RESET datafusion.execution.enable_recursive_ctes;statement okRESET datafusion.sql_parser.enable_ident_normalization;+++# Test projection optimization in recursive CTEs++# https://github.com/apache/datafusion/issues/22249+query I+with recursive t(k, v) as (+ select 1 k, 10 v+ union all+ select 2, 20 from t where k = 1+)+select v+from t+order by 1;+----+10+20++# https://github.com/apache/datafusion/issues/22249+query I+with recursive t(k, v) as (+ select 1 k, 10 v+ union all+ select 2, 20 from t where v = 10+)+select v+from t+order by 1;+----+10+20++# Keep columns that are not selected by the outer query, but still affect+# recursive UNION distinctness.+query I+with recursive t(k, v) as (+ select 1 k, 10 v+ union+ select 2, 10 from t where v = 10+)+select v+from t+order by 1;+----+10+10++statement ok+copy (+ select i as k, i as v1, i as v2+ from generate_series(1, 3) t(i)+) to 'test_files/scratch/cte/test.parquet';++statement ok+create external table test stored as parquet location 'test_files/scratch/cte/test.parquet';++# check that both the static and recursive terms are optimized+query TT+explain+with recursive r as (+ select k, v1 -- only needs to project k and v1 from table test+ from test+ union all+ select k * 10, v1+ from r+ where k < ( -- only needs to project k and v2 from table test+ select v2+ from test+ where k = 2+ )+)+select *+from r+order by 1, 2;+----+logical_plan+01)Sort: r.k ASC NULLS LAST, r.v1 ASC NULLS LAST+02)--SubqueryAlias: r+03)----RecursiveQuery: is_distinct=false+04)------TableScan: test projection=[k, v1]+05)------Projection: r.k * Int64(10), r.v1+06)--------Filter: r.k < (<subquery>)+07)----------Subquery:+08)------------Projection: test.v2+09)--------------Filter: test.k = Int64(2)+10)----------------TableScan: test projection=[k, v2], partial_filters=[test.k = Int64(2)]+11)----------TableScan: r projection=[k, v1]+physical_plan+01)ScalarSubqueryExec: subqueries=1+02)--SortExec: expr=[k@0 ASC NULLS LAST, v1@1 ASC NULLS LAST], preserve_partitioning=[false]+03)----RecursiveQueryExec: name=r, is_distinct=false+04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/test.parquet]]}, projection=[CAST(k@0 AS Int64) as k, CAST(v1@1 AS Int64) as v1], output_ordering=[k@0 ASC NULLS LAST], file_type=parquet+05)------CoalescePartitionsExec+06)--------ProjectionExec: expr=[k@0 * 10 as k, v1@1 as v1]+07)----------FilterExec: k@0 < scalar_subquery(<pending>)+08)------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+09)--------------WorkTableExec: name=r+10)--FilterExec: k@0 = 2, projection=[v2@1]+11)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+12)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/cte/test.parquet]]}, projection=[k, v2], output_ordering=[k@0 ASC NULLS LAST], file_type=parquet, predicate=k@0 = 2, pruning_predicate=k_null_count@2 != row_count@3 AND k_min@0 <= 2 AND 2 <= k_max@1, required_guarantees=[k in (2)]++query II+with recursive r as (+ select k, v1+ from test+ union all+ select k * 10, v1+ from r+ where k < (+ select v2+ from test+ where k = 2+ )+)+select *+from r+order by 1, 2;+----+1 1+2 2+3 3+10 1++statement ok+drop table test;