Apache DataFusion SQL Query Engine
Implement nested recursive CTEs (#18956)
Thomas Tanonea2e22c74376
diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex 3e7c75b0c8..683dbb4e49 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -86,7 +86,7 @@ impl RecursiveQueryExec {is_distinct: bool,) -> Result<Self> {// Each recursive query needs its own work table- let work_table = Arc::new(WorkTable::new());+ let work_table = Arc::new(WorkTable::new(name.clone()));// Use the same work table for both the WorkTableExec and the recursive termlet recursive_term = assign_work_table(recursive_term, &work_table)?;let cache = Self::compute_properties(static_term.schema());@@ -380,8 +380,6 @@ fn assign_work_table(work_table_refs += 1;Ok(Transformed::yes(new_plan))}- } else if plan.as_any().is::<RecursiveQueryExec>() {- not_impl_err!("Recursive queries cannot be nested")} else {Ok(Transformed::no(plan))}diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rsindex ba7c98c264..87ca050e20 100644--- a/datafusion/physical-plan/src/work_table.rs+++ b/datafusion/physical-plan/src/work_table.rs@@ -58,13 +58,15 @@ impl ReservedBatches {#[derive(Debug)]pub struct WorkTable {batches: Mutex<Option<ReservedBatches>>,+ name: String,}impl WorkTable {/// Create a new work table.- pub(super) fn new() -> Self {+ pub(super) fn new(name: String) -> Self {Self {batches: Mutex::new(None),+ name,}}@@ -113,10 +115,10 @@ impl WorkTableExec {pub fn new(name: String, schema: SchemaRef) -> Self {let cache = Self::compute_properties(Arc::clone(&schema));Self {- name,+ name: name.clone(),schema,metrics: ExecutionPlanMetricsSet::new(),- work_table: Arc::new(WorkTable::new()),+ work_table: Arc::new(WorkTable::new(name)),cache,}}@@ -230,6 +232,10 @@ impl ExecutionPlan for WorkTableExec {// Down-cast to the expected state type; propagate `None` on failurelet work_table = state.downcast::<WorkTable>().ok()?;+ if work_table.name != self.name {+ return None; // Different table+ }+Some(Arc::new(Self {name: self.name.clone(),schema: Arc::clone(&self.schema),@@ -248,7 +254,7 @@ mod tests {#[test]fn test_work_table() {- let work_table = WorkTable::new();+ let work_table = WorkTable::new("test".into());// Can't take from empty work_tableassert!(work_table.take().is_err());diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.sltindex 853dbd2e24..3dac929387 100644--- a/datafusion/sqllogictest/test_files/cte.slt+++ b/datafusion/sqllogictest/test_files/cte.slt@@ -125,6 +125,25 @@ SELECT * FROM nodes34+# deduplicating recursive CTE with two variables works+query II+WITH RECURSIVE ranges AS (+ SELECT min, max from (VALUES (1, 1), (2, 2)) ranges(min, max)+ UNION+ SELECT min, max + 1 as max+ FROM ranges+ WHERE max < 4+)+SELECT * FROM ranges+----+1 1+2 2+1 2+2 3+1 3+2 4+1 4+# setupstatement okCREATE EXTERNAL TABLE balance STORED as CSV LOCATION '../core/tests/data/recursive_cte/balance.csv' OPTIONS ('format.has_header' 'true');@@ -647,21 +666,51 @@ ORDER BY3 1400 11 2700 2-#expect error from recursive CTE with nested recursive terms-query error DataFusion error: This feature is not implemented: Recursive queries cannot be nested+#nested recursive ctes+query IWITH RECURSIVE outer_cte AS (SELECT 1 as aUNION ALL (- WITH RECURSIVE nested_cte AS (+ WITH RECURSIVE nested_cte AS (SELECT 1 as aUNION ALL- SELECT a+2 as a- FROM nested_cte where a < 3- )- SELECT outer_cte.a +2- FROM outer_cte JOIN nested_cte USING(a)- WHERE nested_cte.a < 4- )+ SELECT a + 2 as a+ FROM nested_cte where a < 3+ )+ SELECT outer_cte.a + 2 as a+ FROM outer_cte JOIN nested_cte USING(a)+ WHERE nested_cte.a < 4+ )+)+SELECT a FROM outer_cte;+----+1+3+5++# Check that CTE name shadowing is returning an error+query error DataFusion error: Error during planning: WITH query name "outer_cte" specified more than once+WITH RECURSIVE outer_cte AS (+ SELECT 1 as a+ UNION ALL (+ WITH RECURSIVE nested_cte AS (+ SELECT 1 as a+ UNION ALL (+ WITH RECURSIVE outer_cte AS (+ SELECT 1 as a+ UNION ALL+ SELECT a + 2 as a+ FROM outer_cte where a < 3+ )+ SELECT nested_cte.a + outer_cte.a as a+ FROM nested_cte JOIN outer_cte USING(a)+ WHERE outer_cte_cte.a < 8+ )+ )+ SELECT outer_cte.a + nested_cte.a as a+ FROM outer_cte JOIN nested_cte USING(a)+ WHERE nested_cte.a < 8+ ))SELECT a FROM outer_cte;