Apache DataFusion SQL Query Engine
feat: make the materialized CTE nodes a first-class LogicalPlan variant (#26269)
Adrian Garcia Badaracco1e6017cdc44e
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 6db66e0e26..d1e5054cc8 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -94,7 +94,8 @@ use datafusion_expr::physical_planning_context::{use datafusion_expr::utils::{expr_to_columns, split_conjunction};use datafusion_expr::{Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,- FetchType, Filter, JoinType, Operator, RecursiveQuery, SkipType, StringifiedPlan,+ FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan,+ MaterializedCteScan, Operator, RecursiveQuery, SkipType, StringifiedPlan,WindowFrame, WindowFrameBound, WriteOp,};use datafusion_physical_expr::aggregate::{@@ -260,19 +261,14 @@ impl DefaultPhysicalPlanner {////// Returns `None` when `plan` has no materialized CTE.///-/// [`MaterializedCte`]: datafusion_expr::MaterializedCte/// [`ViewTable`]: crate::datasource::ViewTablefn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>> {- fn as_cte(plan: &LogicalPlan) -> Option<&datafusion_expr::MaterializedCte> {- match plan {- LogicalPlan::Extension(Extension { node }) => node.as_any().downcast_ref(),- _ => None,- }- }-let mut has_cte = false;plan.apply_with_subqueries(|node| {- if as_cte(node).is_some() {+ if matches!(+ node,+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_))+ ) {has_cte = true;return Ok(TreeNodeRecursion::Stop);}@@ -284,53 +280,40 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>plan.clone().transform_down_with_subqueries(|node| {- let Some(cte) = as_cte(&node) else {+ let LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) = node else {return Ok(Transformed::no(node));};let old_id = cte.id;- let new_id = datafusion_expr::MaterializedCteId::next();- let continuation = cte- .continuation- .clone()- .transform_down_with_subqueries(|node| {+ let new_id = MaterializedCteId::next();+ let continuation = Arc::unwrap_or_clone(cte.continuation)+ .transform_down_with_subqueries(|node| match node {// A nested occurrence of the same CTE owns the scans below// it, and is renumbered on its own.- if as_cte(&node).is_some_and(|nested| nested.id == old_id) {- return Ok(Transformed::new(- node,- false,- TreeNodeRecursion::Jump,- ));+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ ref nested,+ )) if nested.id == old_id => {+ Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))}- let LogicalPlan::Extension(Extension { node: scan }) = &node else {- return Ok(Transformed::no(node));- };- match scan- .as_any()- .downcast_ref::<datafusion_expr::MaterializedCteScan>()+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))+ if scan.id == old_id =>{- Some(scan) if scan.id == old_id => {- let scan = datafusion_expr::MaterializedCteScan {+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(+ MaterializedCtePlan::Scan(MaterializedCteScan {id: new_id,- ..scan.clone()- };- Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(scan),- })))- }- _ => Ok(Transformed::no(node)),+ ..scan+ }),+ )))}+ _ => Ok(Transformed::no(node)),})?.data;- let cte = datafusion_expr::MaterializedCte {- id: new_id,- name: cte.name.clone(),- cte: cte.cte.clone(),- continuation,- };- Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(cte),- })))+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(+ MaterializedCtePlan::Cte(MaterializedCte {+ id: new_id,+ continuation: Arc::new(continuation),+ ..cte+ }),+ )))}).map(|t| Some(t.data))}@@ -838,6 +821,16 @@ impl DefaultPhysicalPlanner {self.plan_describe(&Arc::clone(schema), output_schema)?}+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => {+ // Unbound until `bind_materialized_cte_scans` runs on the whole plan.+ Arc::new(MaterializedCteScanExec::new(+ scan.id.as_u64(),+ &scan.name,+ Arc::clone(scan.schema.inner()),+ session_state.config().target_partitions(),+ ))+ }+// 1 ChildLogicalPlan::Copy(CopyTo {input,@@ -1871,31 +1864,10 @@ impl DefaultPhysicalPlanner {)?)}- // N Children- LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,- LogicalPlan::Extension(Extension { node })- if node.as_any().is::<datafusion_expr::MaterializedCte>() =>- {- let cte = node- .as_any()- .downcast_ref::<datafusion_expr::MaterializedCte>()- .unwrap();+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {let [body, continuation] = children.two()?;// A CTE that nothing reads is never evaluated.- let mut is_referenced = false;- cte.continuation.apply_with_subqueries(|node| {- if let LogicalPlan::Extension(Extension { node }) = node- && let Some(scan) =- node.as_any()- .downcast_ref::<datafusion_expr::MaterializedCteScan>()- && scan.id == cte.id- {- is_referenced = true;- return Ok(TreeNodeRecursion::Stop);- }- Ok(TreeNodeRecursion::Continue)- })?;- if !is_referenced {+ if cte.scans()?.is_empty() {return Ok(continuation);}// The body is fully buffered before a scan yields a row, so an@@ -1910,21 +1882,9 @@ impl DefaultPhysicalPlanner {Arc::new(MaterializedCteBuffer::new(cte.id.as_u64(), &cte.name));Arc::new(MaterializedCteExec::new(body, continuation, buffer))}- LogicalPlan::Extension(Extension { node })- if node.as_any().is::<datafusion_expr::MaterializedCteScan>() =>- {- let scan = node- .as_any()- .downcast_ref::<datafusion_expr::MaterializedCteScan>()- .unwrap();- // Unbound until `bind_materialized_cte_scans` runs on the whole plan.- Arc::new(MaterializedCteScanExec::new(- scan.id.as_u64(),- &scan.name,- Arc::clone(scan.schema.inner()),- session_state.config().target_partitions(),- ))- }++ // N Children+ LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,LogicalPlan::Extension(Extension { node }) => {let mut maybe_plan = None;let children = children.vec();@@ -2730,7 +2690,8 @@ fn extract_dml_filters(| LogicalPlan::Ddl(_)| LogicalPlan::Copy(_)| LogicalPlan::Unnest(_)- | LogicalPlan::RecursiveQuery(_) => {+ | LogicalPlan::RecursiveQuery(_)+ | LogicalPlan::MaterializedCte(_) => {// No filters to extract from leaf/meta plans}// Plans with inputs (may contain filters in children)diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rsindex 843c06c6fd..80ab609892 100644--- a/datafusion/expr/src/logical_plan/display.rs+++ b/datafusion/expr/src/logical_plan/display.rs@@ -22,9 +22,10 @@ use std::fmt;use crate::{Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,- Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition,- Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest,- Values, Window, WriteOp, expr_vec_fmt,+ Join, Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan,+ Partitioning, Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias,+ TableProviderFilterPushDown, TableScan, Unnest, Values, Window, WriteOp,+ expr_vec_fmt,};use crate::dml::CopyTo;@@ -707,6 +708,23 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {"Detail": format!("{:?}", e.node))}+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {+ name,+ ..+ })) => {+ pg_fields!(+ "Node Type": "MaterializedCte",+ "Name": name,+ )+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(+ MaterializedCteScan { name, .. },+ )) => {+ pg_fields!(+ "Node Type": "MaterializedCteScan",+ "Name": name,+ )+ }LogicalPlan::DescribeTable(DescribeTable { .. }) => {pg_fields!("Node Type": "DescribeTable"diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rsindex 0614a8602c..fbc6cf4872 100644--- a/datafusion/expr/src/logical_plan/invariants.rs+++ b/datafusion/expr/src/logical_plan/invariants.rs@@ -15,6 +15,7 @@// specific language governing permissions and limitations// under the License.+use crate::logical_plan::MaterializedCtePlan;use datafusion_common::{DFSchemaRef, Result, assert_or_internal_err, plan_err,tree_node::{TreeNode, TreeNodeRecursion},@@ -355,6 +356,9 @@ fn check_inner_plan(inner_plan: &LogicalPlan) -> Result<()> {}},LogicalPlan::Extension(_) => Ok(()),+ // A materialized CTE is not checked, as when it was an extension+ // node. `PullUpCorrelatedExpr` rewrites its continuation, or stops.+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_)) => Ok(()),plan => check_no_outer_references(plan),}}diff --git a/datafusion/expr/src/logical_plan/materialized_cte.rs b/datafusion/expr/src/logical_plan/materialized_cte.rsindex aaf8f46dc6..037fc07b42 100644--- a/datafusion/expr/src/logical_plan/materialized_cte.rs+++ b/datafusion/expr/src/logical_plan/materialized_cte.rs@@ -17,11 +17,12 @@//! Logical nodes for `WITH x AS MATERIALIZED (...)`.//!-//! A materialized CTE is planned as one [`MaterializedCte`] node, which owns the-//! CTE body and the rest of the query (the "continuation"), and one+//! A materialized CTE is planned as one [`MaterializedCte`] node, which owns+//! the CTE body and the rest of the query (the "continuation"), and one//! [`MaterializedCteScan`] leaf for each reference to the CTE inside the-//! continuation. The body is therefore optimized and executed once, and the-//! scans read its buffered output.+//! continuation. Both are in the [`LogicalPlan::MaterializedCte`] variant,+//! as cases of [`MaterializedCtePlan`]. The body is therefore optimized and executed once, and+//! the scans read its buffered output.//!//! The scans are leaves, so filters and projections of one reference are not//! pushed into the shared body. This is the same optimization fence that@@ -30,11 +31,13 @@use std::cmp::Ordering;use std::fmt;use std::hash::Hash;+use std::sync::Arc;use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};-use datafusion_common::{DFSchemaRef, Result, assert_eq_or_internal_err};+use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{DFSchemaRef, Result};-use crate::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};+use crate::LogicalPlan;/// Identifies one materialized CTE and binds its scans to it.///@@ -62,91 +65,198 @@ impl fmt::Display for MaterializedCteId {}}+/// A node of a materialized CTE, in [`LogicalPlan::MaterializedCte`].+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]+pub enum MaterializedCtePlan {+ /// Computes the body of the CTE once and evaluates the continuation+ Cte(MaterializedCte),+ /// Reads the buffered output of the CTE. This is a leaf.+ Scan(MaterializedCteScan),+}+/// Computes `cte` once and evaluates `continuation`, in which every/// [`MaterializedCteScan`] with the same `id` reads the buffered output of/// `cte`. The output of this node is the output of `continuation`.+///+/// The node has two inputs with different roles, in this order:+///+/// 1. `cte`: the body of the CTE. It is shared by all the scans, so a rule+/// must not change its schema, and must not push into it a filter, limit+/// or projection that belongs to one scan only.+/// 2. `continuation`: the query that declared the CTE.#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct MaterializedCte {+ /// Binds the scans in `continuation` to this nodepub id: MaterializedCteId,+ /// Name of the CTE, used for display and error messages onlypub name: String,- pub cte: LogicalPlan,- pub continuation: LogicalPlan,+ /// The body of the CTE+ pub cte: Arc<LogicalPlan>,+ /// The query that declared the CTE and reads it+ pub continuation: Arc<LogicalPlan>,}-/// Orders by `id` and `name`. Nodes with the same `id` and `name` but a-/// different `cte` or `continuation` are not comparable, so `Some(Equal)` is-/// returned only for equal nodes.-impl PartialOrd for MaterializedCte {- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {- match self- .id- .cmp(&other.id)- .then_with(|| self.name.cmp(&other.name))- {- Ordering::Equal if self != other => None,- ord => Some(ord),+impl MaterializedCte {+ /// Create a new node.+ ///+ /// Returns an error if a scan of this CTE in `continuation` (also in a+ /// subquery expression) does not have the column names and types of `cte`.+ pub fn try_new(+ id: MaterializedCteId,+ name: impl Into<String>,+ cte: Arc<LogicalPlan>,+ continuation: Arc<LogicalPlan>,+ ) -> Result<Self> {+ let name = name.into();+ for scan in MaterializedCteScan::find_in(&continuation, id)? {+ scan.schema+ .has_equivalent_names_and_types(cte.schema())+ .map_err(|e| {+ e.context(format!(+ "A scan of the materialized CTE {name} does not have the schema of the CTE"+ ))+ })?;}+ Ok(Self {+ id,+ name,+ cte,+ continuation,+ })}-}-impl UserDefinedLogicalNodeCore for MaterializedCte {- fn name(&self) -> &str {- "MaterializedCte"+ /// The id that binds the scans in the continuation to this node.+ pub fn id(&self) -> MaterializedCteId {+ self.id}- fn inputs(&self) -> Vec<&LogicalPlan> {- vec![&self.cte, &self.continuation]+ /// The name of the CTE.+ pub fn name(&self) -> &str {+ &self.name}- fn schema(&self) -> &DFSchemaRef {- self.continuation.schema()+ /// The body of the CTE, which is computed once.+ pub fn body(&self) -> &Arc<LogicalPlan> {+ &self.cte+ }++ /// The query that declared the CTE. It gives this node its output.+ pub fn continuation(&self) -> &Arc<LogicalPlan> {+ &self.continuation}- fn expressions(&self) -> Vec<Expr> {- vec![]+ /// The schema of this node, which is the schema of the continuation.+ pub fn schema(&self) -> &DFSchemaRef {+ self.continuation.schema()}- fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "MaterializedCte: name={}", self.name)+ /// The scans in the continuation that read this CTE, also in subquery+ /// expressions. See [`MaterializedCteScan::find_in`].+ pub fn scans(&self) -> Result<Vec<MaterializedCteScan>> {+ MaterializedCteScan::find_in(&self.continuation, self.id)}- fn with_exprs_and_inputs(+ /// Return a node with the same id and name, and new inputs. The schemas+ /// of the scans are not checked again.+ pub fn with_new_inputs(&self,- exprs: Vec<Expr>,- mut inputs: Vec<LogicalPlan>,- ) -> Result<Self> {- assert_eq_or_internal_err!(exprs.len(), 0, "MaterializedCte takes no exprs");- assert_eq_or_internal_err!(inputs.len(), 2, "MaterializedCte takes 2 inputs");- let continuation = inputs.pop().unwrap();- let cte = inputs.pop().unwrap();- Ok(Self {+ cte: Arc<LogicalPlan>,+ continuation: Arc<LogicalPlan>,+ ) -> Self {+ Self {id: self.id,name: self.name.clone(),cte,continuation,- })+ }}+}- fn necessary_children_exprs(- &self,- output_columns: &[usize],- ) -> Option<Vec<Vec<usize>>> {- // The scans need every column of the body, since they are leaves whose- // schema is fixed at planning time. The continuation only needs the- // columns the parent asks for.- let all_cte_columns = (0..self.cte.schema().fields().len()).collect();- Some(vec![all_cte_columns, output_columns.to_vec()])+/// Orders by `id` and `name`. Nodes with the same `id` and `name` but a+/// different `cte` or `continuation` are not comparable, so `Some(Equal)` is+/// returned only for equal nodes.+impl PartialOrd for MaterializedCte {+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {+ match self+ .id+ .cmp(&other.id)+ .then_with(|| self.name.cmp(&other.name))+ {+ Ordering::Equal if self != other => None,+ ord => Some(ord),+ }}}/// Reads the buffered output of the [`MaterializedCte`] with the same `id`.+///+/// This is a leaf. Its schema is the schema of the CTE body and is fixed when+/// the scan is created.#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct MaterializedCteScan {+ /// The id of the [`MaterializedCte`] that this scan readspub id: MaterializedCteId,+ /// Name of the CTE, used for display and error messages onlypub name: String,+ /// The schema of the CTE bodypub schema: DFSchemaRef,}+impl MaterializedCteScan {+ /// Create a scan of the CTE `id`, whose body has the schema `schema`.+ pub fn new(+ id: MaterializedCteId,+ name: impl Into<String>,+ schema: DFSchemaRef,+ ) -> Self {+ Self {+ id,+ name: name.into(),+ schema,+ }+ }++ /// The id of the [`MaterializedCte`] that this scan reads.+ pub fn id(&self) -> MaterializedCteId {+ self.id+ }++ /// The name of the CTE.+ pub fn name(&self) -> &str {+ &self.name+ }++ /// The schema of this scan.+ pub fn schema(&self) -> &DFSchemaRef {+ &self.schema+ }++ /// Return a copy of each scan of the CTE `id` in `plan`, also in subquery+ /// expressions.+ ///+ /// The search does not go below a [`MaterializedCte`] with the same `id`+ /// in `plan`. Such a node is a separate occurrence of the CTE (the id is+ /// copied when a plan is cloned) and owns the scans below it.+ pub fn find_in(plan: &LogicalPlan, id: MaterializedCteId) -> Result<Vec<Self>> {+ let mut scans = vec![];+ plan.apply_with_subqueries(|node| match node {+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte))+ if cte.id == id =>+ {+ Ok(TreeNodeRecursion::Jump)+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))+ if scan.id == id =>+ {+ scans.push(scan.clone());+ Ok(TreeNodeRecursion::Continue)+ }+ _ => Ok(TreeNodeRecursion::Continue),+ })?;+ Ok(scans)+ }+}+/// Orders by `id` and `name`. Scans with the same `id` and `name` but a/// different `schema` are not comparable.impl PartialOrd for MaterializedCteScan {@@ -162,53 +272,39 @@ impl PartialOrd for MaterializedCteScan {}}-impl UserDefinedLogicalNodeCore for MaterializedCteScan {- fn name(&self) -> &str {- "MaterializedCteScan"- }-- fn inputs(&self) -> Vec<&LogicalPlan> {- vec![]- }-- fn schema(&self) -> &DFSchemaRef {- &self.schema- }+#[cfg(test)]+mod tests {+ use super::*;+ use crate::logical_plan::builder::LogicalPlanBuilder;+ use crate::{EmptyRelation, exists};+ use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::DFSchema;- fn expressions(&self) -> Vec<Expr> {- vec![]+ fn empty(produce_one_row: bool) -> Arc<LogicalPlan> {+ Arc::new(LogicalPlan::EmptyRelation(EmptyRelation {+ produce_one_row,+ schema: Arc::new(DFSchema::empty()),+ }))}- fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "MaterializedCteScan: name={}", self.name)+ fn schema(data_type: DataType) -> DFSchemaRef {+ let schema = Schema::new(vec![Field::new("a", data_type, true)]);+ Arc::new(DFSchema::try_from_qualified_schema("c", &schema).unwrap())}- fn with_exprs_and_inputs(- &self,- exprs: Vec<Expr>,- inputs: Vec<LogicalPlan>,- ) -> Result<Self> {- assert_eq_or_internal_err!(exprs.len(), 0, "MaterializedCteScan takes no exprs");- assert_eq_or_internal_err!(- inputs.len(),- 0,- "MaterializedCteScan takes no inputs"- );- Ok(self.clone())+ fn body(data_type: DataType) -> Arc<LogicalPlan> {+ Arc::new(LogicalPlan::EmptyRelation(EmptyRelation {+ produce_one_row: false,+ schema: schema(data_type),+ }))}-}-#[cfg(test)]-mod tests {- use super::*;- use crate::EmptyRelation;- use std::sync::Arc;-- fn empty(produce_one_row: bool) -> LogicalPlan {- LogicalPlan::EmptyRelation(EmptyRelation {- produce_one_row,- schema: Arc::new(datafusion_common::DFSchema::empty()),- })+ fn scan(id: MaterializedCteId, data_type: DataType) -> LogicalPlan {+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(MaterializedCteScan::new(+ id,+ "c",+ schema(data_type),+ )))}#[test]@@ -233,4 +329,76 @@ mod tests {};assert_eq!(a.partial_cmp(&c), Some(Ordering::Less));}++ #[test]+ fn try_new_checks_the_schema_of_the_scans() -> Result<()> {+ let id = MaterializedCteId::next();+ let cte = MaterializedCte::try_new(+ id,+ "c",+ body(DataType::Int32),+ Arc::new(scan(id, DataType::Int32)),+ )?;+ assert_eq!(cte.id(), id);+ assert_eq!(cte.name(), "c");+ assert_eq!(cte.schema(), cte.continuation().schema());+ assert_eq!(cte.scans()?.len(), 1);++ let plan = LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte));+ assert_eq!(plan.inputs().len(), 2);+ assert_eq!(+ plan.display_indent().to_string(),+ "MaterializedCte: name=c\n EmptyRelation: rows=0\n MaterializedCteScan: name=c"+ );++ let err = MaterializedCte::try_new(+ id,+ "c",+ body(DataType::Int32),+ Arc::new(scan(id, DataType::Utf8)),+ )+ .unwrap_err();+ assert!(+ err.to_string().contains(+ "A scan of the materialized CTE c does not have the schema of the CTE"+ ),+ "{err}"+ );++ // A scan of another CTE is not checked.+ MaterializedCte::try_new(+ id,+ "c",+ body(DataType::Int32),+ Arc::new(scan(MaterializedCteId::next(), DataType::Utf8)),+ )?;+ Ok(())+ }++ #[test]+ fn find_in_looks_in_subqueries_and_stops_at_the_same_cte() -> Result<()> {+ let id = MaterializedCteId::next();+ let other = MaterializedCteId::next();+ // The union holds one scan, one scan in a subquery expression, one+ // scan of another CTE, and a nested occurrence of the same CTE, which+ // owns its scan.+ let plan = LogicalPlanBuilder::from(scan(id, DataType::Int32))+ .filter(exists(Arc::new(scan(id, DataType::Int32))))?+ .union(scan(other, DataType::Int32))?+ .union(LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ MaterializedCte::try_new(+ id,+ "c",+ body(DataType::Int32),+ Arc::new(scan(id, DataType::Int32)),+ )?,+ )))?+ .build()?;+ assert_eq!(MaterializedCteScan::find_in(&plan, id)?.len(), 2);+ assert_eq!(MaterializedCteScan::find_in(&plan, other)?.len(), 1);+ assert!(+ MaterializedCteScan::find_in(&plan, MaterializedCteId::next())?.is_empty()+ );+ Ok(())+ }}diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rsindex 81483dae7f..794747c733 100644--- a/datafusion/expr/src/logical_plan/mod.rs+++ b/datafusion/expr/src/logical_plan/mod.rs@@ -61,4 +61,6 @@ pub use datafusion_common::format::ExplainFormat;pub use display::display_schema;pub use extension::{UserDefinedLogicalNode, UserDefinedLogicalNodeCore};-pub use materialized_cte::{MaterializedCte, MaterializedCteId, MaterializedCteScan};+pub use materialized_cte::{+ MaterializedCte, MaterializedCteId, MaterializedCtePlan, MaterializedCteScan,+};diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rsindex e0ccc5dc73..477e9c73d5 100644--- a/datafusion/expr/src/logical_plan/plan.rs+++ b/datafusion/expr/src/logical_plan/plan.rs@@ -39,7 +39,9 @@ use crate::expr_rewriter::{};use crate::logical_plan::display::{GraphvizVisitor, IndentVisitor};use crate::logical_plan::extension::UserDefinedLogicalNode;-use crate::logical_plan::materialized_cte::MaterializedCte;+use crate::logical_plan::materialized_cte::{+ MaterializedCte, MaterializedCtePlan, MaterializedCteScan,+};use crate::logical_plan::{DmlStatement, Statement, WriteOp};use crate::utils::{check_aggregate_and_window_nesting, check_no_window_functions,@@ -363,6 +365,9 @@ pub enum LogicalPlan {/// Match each left row with at most one ordered row from the right input./// This is used to implement SQL `ASOF JOIN`.AsOfJoin(AsOfJoin),+ /// A node of a SQL `WITH x AS MATERIALIZED (...)` CTE. See+ /// [`MaterializedCtePlan`].+ MaterializedCte(MaterializedCtePlan),}impl Default for LogicalPlan {@@ -419,6 +424,8 @@ impl LogicalPlan {LogicalPlan::Explain(explain) => &explain.schema,LogicalPlan::Analyze(analyze) => &analyze.schema,LogicalPlan::Extension(extension) => extension.node.schema(),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => cte.schema(),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => &scan.schema,LogicalPlan::Union(Union { schema, .. }) => schema,LogicalPlan::DescribeTable(DescribeTable { output_schema, .. }) => {output_schema@@ -537,6 +544,12 @@ impl LogicalPlan {LogicalPlan::Subquery(Subquery { subquery, .. }) => vec![subquery],LogicalPlan::SubqueryAlias(SubqueryAlias { input, .. }) => vec![input],LogicalPlan::Extension(extension) => extension.node.inputs(),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {+ cte,+ continuation,+ ..+ })) => vec![cte, continuation],+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => vec![],LogicalPlan::Union(Union { inputs, .. }) => {inputs.iter().map(|arc| arc.as_ref()).collect()}@@ -673,12 +686,11 @@ impl LogicalPlan {.map_or(Ok(None), |v| v.map(Some))}// A materialized CTE outputs the output of its continuation.- LogicalPlan::Extension(Extension { node }) => {- match node.as_any().downcast_ref::<MaterializedCte>() {- Some(cte) => cte.continuation.head_output_expr(),- None => Ok(None),- }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {+ cte.continuation.head_output_expr()}+ LogicalPlan::Extension(_)+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => Ok(None),LogicalPlan::Subquery(_) => Ok(None),LogicalPlan::EmptyRelation(_)| LogicalPlan::Statement(_)@@ -811,6 +823,10 @@ impl LogicalPlan {}) => SubqueryAlias::try_new(input, alias).map(LogicalPlan::SubqueryAlias),LogicalPlan::Limit(_) => Ok(self),LogicalPlan::Ddl(_) => Ok(self),+ // A `Cte` takes the schema of its continuation, and does not store+ // it. A `Scan` keeps the schema of the CTE body, which also keeps+ // its schema.+ LogicalPlan::MaterializedCte(_) => Ok(self),LogicalPlan::Extension(Extension { node }) => {// todo make an API that does not require cloning// This requires a copy of the extension nodes expressions and inputs@@ -1228,6 +1244,18 @@ impl LogicalPlan {LogicalPlan::Extension(e) => Ok(LogicalPlan::Extension(Extension {node: e.node.with_exprs_and_inputs(expr, inputs)?,})),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {+ self.assert_no_expressions(expr)?;+ let (body, continuation) = self.only_two_inputs(inputs)?;+ Ok(LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ cte.with_new_inputs(Arc::new(body), Arc::new(continuation)),+ )))+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => {+ self.assert_no_expressions(expr)?;+ self.assert_no_inputs(inputs)?;+ Ok(self.clone())+ }LogicalPlan::Union(Union { schema, .. }) => {self.assert_no_expressions(expr)?;let input_schema = inputs[0].schema();@@ -1603,7 +1631,8 @@ impl LogicalPlan {| LogicalPlan::Copy(_)| LogicalPlan::DescribeTable(_)| LogicalPlan::Statement(_)- | LogicalPlan::Extension(_) => None,+ | LogicalPlan::Extension(_)+ | LogicalPlan::MaterializedCte(_) => None,}}@@ -1680,7 +1709,8 @@ impl LogicalPlan {| LogicalPlan::Copy(_)| LogicalPlan::DescribeTable(_)| LogicalPlan::Statement(_)- | LogicalPlan::Extension(_) => 0,+ | LogicalPlan::Extension(_)+ | LogicalPlan::MaterializedCte(_) => 0,}}@@ -1713,6 +1743,7 @@ impl LogicalPlan {LogicalPlan::Explain(_) => Ok(None),LogicalPlan::Analyze(_) => Ok(None),LogicalPlan::Extension(_) => Ok(None),+ LogicalPlan::MaterializedCte(_) => Ok(None),LogicalPlan::Distinct(_) => Ok(None),LogicalPlan::Dml(_) => Ok(None),LogicalPlan::Ddl(_) => Ok(None),@@ -1752,6 +1783,7 @@ impl LogicalPlan {LogicalPlan::Explain(_) => Ok(None),LogicalPlan::Analyze(_) => Ok(None),LogicalPlan::Extension(_) => Ok(None),+ LogicalPlan::MaterializedCte(_) => Ok(None),LogicalPlan::Distinct(_) => Ok(None),LogicalPlan::Dml(_) => Ok(None),LogicalPlan::Ddl(_) => Ok(None),@@ -2498,6 +2530,16 @@ impl LogicalPlan {LogicalPlan::Analyze { .. } => write!(f, "Analyze"),LogicalPlan::Union(_) => write!(f, "Union"),LogicalPlan::Extension(e) => e.node.fmt_for_explain(f),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ MaterializedCte { name, .. },+ )) => {+ write!(f, "MaterializedCte: name={name}")+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(+ MaterializedCteScan { name, .. },+ )) => {+ write!(f, "MaterializedCteScan: name={name}")+ }LogicalPlan::DescribeTable(DescribeTable { .. }) => {write!(f, "DescribeTable")}diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rsindex 9e961a1f11..a4739b3024 100644--- a/datafusion/expr/src/logical_plan/tree_node.rs+++ b/datafusion/expr/src/logical_plan/tree_node.rs@@ -43,10 +43,10 @@ use crate::logical_plan::plan::RangePartitioning;use crate::{Aggregate, Analyze, AsOfJoin, AsOfMatch, CreateMemoryTable, CreateView, DdlStatement,Distinct, DistinctOn, DmlStatement, Execute, Explain, Expr, Extension, Filter, Join,- Limit, LogicalPlan, Partitioning, Prepare, Projection, RecursiveQuery, Repartition,- Sort, Statement, Subquery, SubqueryAlias, TableScan, Union, Unnest,- UserDefinedLogicalNode, Values, Window, WriteOp, builder::unnest_with_options,- dml::CopyTo,+ Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, Partitioning, Prepare,+ Projection, RecursiveQuery, Repartition, Sort, Statement, Subquery, SubqueryAlias,+ TableScan, Union, Unnest, UserDefinedLogicalNode, Values, Window, WriteOp,+ builder::unnest_with_options, dml::CopyTo,};use datafusion_common::tree_node::TreeNodeRefContainer;@@ -377,6 +377,25 @@ impl TreeNode for LogicalPlan {})},),+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {+ id,+ name,+ cte,+ continuation,+ })) => {+ (cte, continuation)+ .map_elements(f)?+ .update_data(|(cte, continuation)| {+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ MaterializedCte {+ id,+ name,+ cte,+ continuation,+ },+ ))+ })+ }LogicalPlan::Statement(stmt) => match stmt {Statement::Prepare(p) => p.input@@ -389,6 +408,7 @@ impl TreeNode for LogicalPlan {LogicalPlan::TableScan { .. }| LogicalPlan::EmptyRelation { .. }| LogicalPlan::Values { .. }+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))| LogicalPlan::DescribeTable(_) => Transformed::no(self),})}@@ -516,6 +536,7 @@ impl LogicalPlan {// plans without expressionsLogicalPlan::EmptyRelation(_)| LogicalPlan::RecursiveQuery(_)+ | LogicalPlan::MaterializedCte(_)| LogicalPlan::Subquery(_)| LogicalPlan::SubqueryAlias(_)| LogicalPlan::Analyze(_)@@ -803,6 +824,7 @@ impl LogicalPlan {// plans without expressionsLogicalPlan::EmptyRelation(_)| LogicalPlan::RecursiveQuery(_)+ | LogicalPlan::MaterializedCte(_)| LogicalPlan::Subquery(_)| LogicalPlan::SubqueryAlias(_)| LogicalPlan::Analyze(_)diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rsindex bd3aea11ae..abc756879a 100644--- a/datafusion/optimizer/src/analyzer/type_coercion.rs+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs@@ -43,9 +43,7 @@ use datafusion_expr::expr::{};use datafusion_expr::expr_rewriter::coerce_plan_expr_for_schema;use datafusion_expr::expr_schema::{cast_subquery, cast_subquery_columns};-use datafusion_expr::logical_plan::{- Extension, MaterializedCte, MaterializedCteId, MaterializedCteScan, Subquery,-};+use datafusion_expr::logical_plan::Subquery;use datafusion_expr::type_coercion::binary::{comparison_coercion_with_session_timezone, like_coercion, regex_coercion,type_union_coercion,@@ -60,7 +58,8 @@ use datafusion_expr::type_coercion::{use datafusion_expr::utils::merge_schema;use datafusion_expr::{AsOfJoin, AsOfMatch, Cast, DmlStatement, Expr, ExprSchemable, Join, Limit,- LogicalPlan, Operator, Projection, Union, ValueOrLambda, WindowFrame,+ LogicalPlan, MaterializedCte, MaterializedCteId, MaterializedCtePlan,+ MaterializedCteScan, Operator, Projection, Union, ValueOrLambda, WindowFrame,WindowFrameBound, WindowFrameUnits, WriteOp, is_false, is_not_false, is_not_true,is_not_unknown, is_true, is_unknown, lit, not,};@@ -135,38 +134,32 @@ fn coerce_plan_and_subqueries(static EMPTY_SCHEMA: LazyLock<DFSchema> = LazyLock::new(DFSchema::empty);plan.transform_down_up_with_subqueries(- |plan| {- let cte = match &plan {- LogicalPlan::Extension(Extension { node }) => {- node.as_any().downcast_ref::<MaterializedCte>().cloned()- }- _ => None,- };- let Some(cte) = cte else {- return Ok(Transformed::no(plan));- };+ |plan| match plan {// Coercion can change the types of the body, for example the type// of a `NULL` column of a `UNION`. Coerce the body first, and give// its new schema to the scans before the continuation is coerced.- let body = coerce_plan_and_subqueries(cte.cte, config)?;- let continuation = with_materialized_cte_scan_schema(- cte.continuation,- cte.id,- body.schema(),- )?;- let continuation = coerce_plan_and_subqueries(continuation, config)?;- Ok(Transformed::new(- LogicalPlan::Extension(Extension {- node: Arc::new(MaterializedCte {- id: cte.id,- name: cte.name,- cte: body,- continuation,- }),- }),- true,- TreeNodeRecursion::Jump,- ))+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {+ let body =+ coerce_plan_and_subqueries(Arc::unwrap_or_clone(cte.cte), config)?;+ let continuation = with_materialized_cte_scan_schema(+ Arc::unwrap_or_clone(cte.continuation),+ cte.id,+ body.schema(),+ )?;+ let continuation = coerce_plan_and_subqueries(continuation, config)?;+ Ok(Transformed::new(+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ MaterializedCte {+ cte: Arc::new(body),+ continuation: Arc::new(continuation),+ ..cte+ },+ )),+ true,+ TreeNodeRecursion::Jump,+ ))+ }+ _ => Ok(Transformed::no(plan)),},|plan| {analyze_internal(@@ -188,42 +181,32 @@ fn with_materialized_cte_scan_schema(body_schema: &DFSchemaRef,) -> Result<LogicalPlan> {continuation- .transform_down_with_subqueries(|plan| {- let LogicalPlan::Extension(Extension { node }) = &plan else {- return Ok(Transformed::no(plan));- };+ .transform_down_with_subqueries(|node| match node {// A nested occurrence of the same CTE owns the scans below it.- if node- .as_any()- .downcast_ref::<MaterializedCte>()- .is_some_and(|nested| nested.id == id)+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref nested))+ if nested.id == id =>{- return Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump));+ Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))}- let Some(scan) =- node.as_any()- .downcast_ref::<MaterializedCteScan>()- .filter(|scan| {- scan.id == id && scan.schema.inner() != body_schema.inner()- })- else {- return Ok(Transformed::no(plan));- };- let qualifiers = scan.schema.iter().map(|(q, _)| q.cloned()).collect();- let schema = DFSchema::from_field_specific_qualified_schema(- qualifiers,- body_schema.inner(),- )?- .with_functional_dependencies(- body_schema.functional_dependencies().clone(),- )?;- Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(MaterializedCteScan {- id: scan.id,- name: scan.name.clone(),- schema: Arc::new(schema),- }),- })))+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))+ if scan.id == id && scan.schema.inner() != body_schema.inner() =>+ {+ let qualifiers = scan.schema.iter().map(|(q, _)| q.cloned()).collect();+ let schema = DFSchema::from_field_specific_qualified_schema(+ qualifiers,+ body_schema.inner(),+ )?+ .with_functional_dependencies(+ body_schema.functional_dependencies().clone(),+ )?;+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(+ MaterializedCtePlan::Scan(MaterializedCteScan {+ schema: Arc::new(schema),+ ..scan+ }),+ )))+ }+ _ => Ok(Transformed::no(node)),}).map(|t| t.data)}diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rsindex e0d43d7c15..2c5a55cc65 100644--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs@@ -586,7 +586,8 @@ impl OptimizerRule for CommonSubexprEliminate {| LogicalPlan::Dml(_)| LogicalPlan::Copy(_)| LogicalPlan::Unnest(_)- | LogicalPlan::RecursiveQuery(_) => {+ | LogicalPlan::RecursiveQuery(_)+ | LogicalPlan::MaterializedCte(_) => {// This rule handles recursion itself in a `ApplyOrder::TopDown` like// manner. Process uncorrelated subqueries in expressions// (e.g., Expr::ScalarSubquery), then direct children.diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rsindex 246639b389..6253c90ebb 100644--- a/datafusion/optimizer/src/decorrelate.rs+++ b/datafusion/optimizer/src/decorrelate.rs@@ -39,9 +39,9 @@ use datafusion_expr::logical_plan::{Join, JoinType};use datafusion_expr::simplify::SimplifyContext;use datafusion_expr::utils::{collect_subquery_cols, conjunction, split_conjunction};use datafusion_expr::{- Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension,- FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType,- UserDefinedLogicalNodeCore, Window, expr, lit,+ Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, FetchType,+ LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCtePlan, Operator,+ SkipType, Window, expr, lit,};/// This struct rewrite the sub query plan by pull up the correlated@@ -208,35 +208,25 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {LogicalPlan::Subquery(_) => {Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump))}- LogicalPlan::Extension(Extension { node })- if node.as_any().is::<MaterializedCte>() =>+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref cte))+ if !cte.cte.all_out_ref_exprs().is_empty() =>{- let cte = node.as_any().downcast_ref::<MaterializedCte>().unwrap();- if !cte.cte.all_out_ref_exprs().is_empty() {- // A correlated body would have to run once per outer row.- return Ok(- self.stop_pull_up(LogicalPlan::Extension(Extension { node }))- );- }+ // A correlated body would have to run once per outer row.+ Ok(self.stop_pull_up(plan))+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {// Only the continuation can hold correlated expressions. Rewrite// it alone, so that the body shared by every scan keeps its schema.- let continuation = cte.continuation.clone().rewrite(self)?;- if !continuation.transformed {- return Ok(Transformed::new(- LogicalPlan::Extension(Extension { node }),- false,- TreeNodeRecursion::Jump,- ));- }- let cte = cte.with_exprs_and_inputs(- vec![],- vec![cte.cte.clone(), continuation.data],- )?;+ let continuation =+ Arc::unwrap_or_clone(cte.continuation).rewrite(self)?;Ok(Transformed::new(- LogicalPlan::Extension(Extension {- node: Arc::new(cte),- }),- true,+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(+ MaterializedCte {+ continuation: Arc::new(continuation.data),+ ..cte+ },+ )),+ continuation.transformed,TreeNodeRecursion::Jump,))}@@ -350,6 +340,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {| LogicalPlan::SubqueryAlias(_)| LogicalPlan::TableScan(_)| LogicalPlan::EmptyRelation(_)+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))| LogicalPlan::Values(_) => {if plan.contains_outer_reference() {// the unsupported cases, the plan expressions contain outdiff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rsindex 8fdb5775c9..6cf4047713 100644--- a/datafusion/optimizer/src/optimize_projections/mod.rs+++ b/datafusion/optimizer/src/optimize_projections/mod.rs@@ -32,6 +32,7 @@ mod required_indices;use crate::optimizer::ApplyOrder;use crate::{OptimizerConfig, OptimizerRule};+use datafusion_expr::logical_plan::MaterializedCtePlan;use std::sync::Arc;use datafusion_common::{@@ -375,8 +376,18 @@ fn optimize_projections(.collect()}}+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {+ // The scans are leaves whose schema is the schema of the body, so+ // the body keeps every column. The output is the output of the+ // continuation, so the continuation needs what the parent needs.+ vec![+ RequiredIndices::new_for_all_exprs(&cte.cte),+ RequiredIndices::new_from_indices(indices.into_inner()),+ ]+ }LogicalPlan::EmptyRelation(_)| LogicalPlan::Values(_)+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))| LogicalPlan::DescribeTable(_) => {// These operators have no inputs, so stop the optimization process.return Ok(Transformed::no(plan));diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rsindex 94583d545d..e3bf8eff17 100644--- a/datafusion/optimizer/src/optimizer.rs+++ b/datafusion/optimizer/src/optimizer.rs@@ -36,9 +36,9 @@ use datafusion_expr::dml::CopyTo;use datafusion_expr::logical_plan::LogicalPlan;use datafusion_expr::{Aggregate, Analyze, CreateMemoryTable, CreateView, DdlStatement, Distinct,- DistinctOn, DmlStatement, Explain, Expr, Extension, Filter, Join, Limit, Projection,- RecursiveQuery, Repartition, Sort, Statement, Subquery, SubqueryAlias, Union, Unnest,- Window,+ DistinctOn, DmlStatement, Explain, Expr, Extension, Filter, Join, Limit,+ MaterializedCte, MaterializedCtePlan, Projection, RecursiveQuery, Repartition, Sort,+ Statement, Subquery, SubqueryAlias, Union, Unnest, Window,};use crate::common_subexpr_eliminate::CommonSubexprEliminate;@@ -443,6 +443,15 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(let r = f(Arc::make_mut(recursive_term))?;s || r}+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {+ cte,+ continuation,+ ..+ })) => {+ let b = f(Arc::make_mut(cte))?;+ let c = f(Arc::make_mut(continuation))?;+ b || c+ }LogicalPlan::Statement(Statement::Prepare(p)) => f(Arc::make_mut(&mut p.input))?,LogicalPlan::Extension(Extension { node }) => {let inputs = node.inputs();@@ -471,6 +480,7 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(LogicalPlan::TableScan { .. }| LogicalPlan::EmptyRelation { .. }| LogicalPlan::Values { .. }+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))| LogicalPlan::DescribeTable(_)| LogicalPlan::Ddl(DdlStatement::CreateExternalTable(_))| LogicalPlan::Ddl(DdlStatement::CreateCatalogSchema(_))diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rsindex ccde8aa07f..01a4fb0e9b 100644--- a/datafusion/optimizer/src/push_down_filter.rs+++ b/datafusion/optimizer/src/push_down_filter.rs@@ -39,6 +39,7 @@//!//! [`PushDownLeafProjections`]: crate::extract_leaf_expressions::PushDownLeafProjections+use datafusion_expr::logical_plan::MaterializedCtePlan;use std::collections::{HashMap, HashSet};use std::sync::Arc;@@ -62,9 +63,8 @@ use datafusion_expr::utils::{conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,};use datafusion_expr::{- BinaryExpr, Distinct, EmptyRelation, Expr, Extension, Filter, MaterializedCte,- Operator, Projection, TableProviderFilterPushDown, UserDefinedLogicalNodeCore, and,- or,+ BinaryExpr, Distinct, EmptyRelation, Expr, Filter, Operator, Projection,+ TableProviderFilterPushDown, and, or,};use crate::extract_leaf_expressions::is_pure_extraction_projection;@@ -1403,22 +1403,22 @@ impl OptimizerRule for PushDownFilter {LogicalPlan::TableScan(scan),)))}- LogicalPlan::Extension(Extension { node })- if node.as_any().is::<MaterializedCte>() =>- {+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(mut cte)) => {// The output is the output of the continuation, so the filter// moves onto the continuation. The body is shared by every scan// of the CTE and is never filtered for one of them.- let cte = node.as_any().downcast_ref::<MaterializedCte>().unwrap();- let continuation = LogicalPlan::Filter(Filter::new(+ cte.continuation = Arc::new(LogicalPlan::Filter(Filter::new(filter.predicate,- Arc::new(cte.continuation.clone()),- ));- let cte = cte- .with_exprs_and_inputs(vec![], vec![cte.cte.clone(), continuation])?;- Ok(Transformed::yes(LogicalPlan::Extension(Extension {- node: Arc::new(cte),- })))+ cte.continuation,+ )));+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(+ MaterializedCtePlan::Cte(cte),+ )))+ }+ // A scan reads the shared body, so the filter stays above it.+ scan @ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => {+ filter.input = Arc::new(scan);+ Ok(Transformed::no(LogicalPlan::Filter(filter)))}LogicalPlan::Extension(extension_plan) => {// This check prevents the Filter from being removed when the extension node has no children,diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rsindex d86b27ff3c..9037fb8772 100644--- a/datafusion/proto/src/logical_plan/mod.rs+++ b/datafusion/proto/src/logical_plan/mod.rs@@ -2411,6 +2411,11 @@ impl AsLogicalPlan for LogicalPlanNode {LogicalPlan::DescribeTable(_) => Err(proto_error("LogicalPlan serde is not yet implemented for DescribeTable",)),+ LogicalPlan::MaterializedCte(_) => {+ not_impl_err!(+ "LogicalPlan serde is not yet implemented for MATERIALIZED CTEs"+ )+ }LogicalPlan::RecursiveQuery(recursive) => {let static_term = LogicalPlanNode::try_from_logical_plan(recursive.static_term.as_ref(),diff --git a/datafusion/sql/src/cte.rs b/datafusion/sql/src/cte.rsindex b6fbd21822..e5ff1b6367 100644--- a/datafusion/sql/src/cte.rs+++ b/datafusion/sql/src/cte.rs@@ -25,8 +25,8 @@ use datafusion_common::{tree_node::{TreeNode, TreeNodeRecursion},};use datafusion_expr::{- Extension, LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCteId,- MaterializedCteScan, TableSource,+ LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCteId,+ MaterializedCtePlan, MaterializedCteScan, TableSource,};use sqlparser::ast::{CteAsMaterialized, Ident, Query, SetExpr, SetOperator, With};@@ -40,15 +40,14 @@ pub(crate) struct PendingMaterializedCte {impl PendingMaterializedCte {/// Wrap `continuation` so that its scans of this CTE read one shared result.- pub(crate) fn wrap(self, continuation: LogicalPlan) -> LogicalPlan {- LogicalPlan::Extension(Extension {- node: Arc::new(MaterializedCte {- id: self.id,- name: self.name,- cte: self.cte,- continuation,- }),- })+ pub(crate) fn wrap(self, continuation: LogicalPlan) -> Result<LogicalPlan> {+ MaterializedCte::try_new(+ self.id,+ self.name,+ Arc::new(self.cte),+ Arc::new(continuation),+ )+ .map(|cte| LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)))}}@@ -101,13 +100,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {// of the body. The body is already planned, so a reference// of a recursive CTE to itself stays a work table scan.let id = MaterializedCteId::next();- let scan = LogicalPlan::Extension(Extension {- node: Arc::new(MaterializedCteScan {+ let scan = LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(+ MaterializedCteScan::new(id,- name: cte_name.clone(),- schema: Arc::clone(final_plan.schema()),- }),- });+ cte_name.clone(),+ Arc::clone(final_plan.schema()),+ ),+ ));materialized.push(PendingMaterializedCte {id,name: cte_name.clone(),diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rsindex a943abe2f8..00651cd0b1 100644--- a/datafusion/sql/src/query.rs+++ b/datafusion/sql/src/query.rs@@ -102,10 +102,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {return not_impl_err!("MATERIALIZED CTEs are not supported with SELECT INTO");}// A later CTE may read an earlier one, so the first CTE is the outermost.- Ok(materialized_ctes+ materialized_ctes.into_iter().rev()- .fold(plan, |plan, cte| cte.wrap(plan)))+ .try_fold(plan, |plan, cte| cte.wrap(plan))}/// Apply pipe operators to a plandiff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rsindex 83ced616e0..de6ba9443c 100644--- a/datafusion/sql/src/unparser/plan.rs+++ b/datafusion/sql/src/unparser/plan.rs@@ -207,6 +207,9 @@ impl Unparser<'_> {| LogicalPlan::DescribeTable(_)| LogicalPlan::RecursiveQuery(_)| LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"),+ LogicalPlan::MaterializedCte(_) => {+ not_impl_err!("Unparsing a MATERIALIZED CTE is not supported: {plan:?}")+ }}}diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rsindex 15f59919a2..8b02e09f4c 100644--- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs+++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs@@ -37,7 +37,7 @@ pub use sort_rel::*;use crate::logical_plan::producer::SubstraitProducer;use datafusion::common::not_impl_err;-use datafusion::logical_expr::LogicalPlan;+use datafusion::logical_expr::{LogicalPlan, MaterializedCtePlan};use substrait::proto::Rel;pub fn to_substrait_rel(@@ -77,5 +77,11 @@ pub fn to_substrait_rel(LogicalPlan::RecursiveQuery(plan) => {not_impl_err!("Unsupported plan type: {plan:?}")?}+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(plan)) => {+ not_impl_err!("Substrait MATERIALIZED CTE is not supported: {plan:?}")?+ }+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(plan)) => {+ not_impl_err!("Substrait MATERIALIZED CTE is not supported: {plan:?}")?+ }}}diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.mdindex 7b6bc6a99e..e927d3ec4a 100644--- a/docs/source/library-user-guide/upgrading/56.0.0.md+++ b/docs/source/library-user-guide/upgrading/56.0.0.md@@ -1233,6 +1233,20 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' planSee [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.+### `LogicalPlan` has the new variant `MaterializedCte`++`WITH x AS MATERIALIZED (...)` is planned with the new variant `LogicalPlan::MaterializedCte(MaterializedCtePlan)`. `MaterializedCtePlan` has two cases:++- `MaterializedCtePlan::Cte` has two inputs: the body of the CTE, which is computed once, and the continuation (the query that declared the CTE). Its output is the output of the continuation.+- `MaterializedCtePlan::Scan` is a leaf. It reads the buffered output of the `MaterializedCtePlan::Cte` that has the same id.++#### Who is affected++- Code that has an exhaustive `match` on `LogicalPlan` must add an arm for the new variant.+- A custom optimizer rule that moves a filter, limit or projection below a node must not move it into the body of a `MaterializedCte`, because all the scans of the CTE share the body. The continuation has no such limit.++`MaterializedCteScan::find_in` returns the scans of one CTE in a plan, also in subquery expressions.+### Extension `PhysicalExpr`s can decode by name instead of through a codec`PhysicalExtensionExprNode` has a new optional `expr_name` field. An extension expression that sets it is decoded by a decoder registered on the session, not by a `PhysicalExtensionCodec`. This is the same mechanism extension `ExecutionPlan`s use (see above), and it uses the same `ProtoDecoderRegistry`.