Skip to content

Apache DataFusion SQL Query Engine

feat: make the materialized CTE nodes a first-class LogicalPlan variant (#26269)

Adrian Garcia Badaracco1e6017cdc44e
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 6db66e0e26..d1e5054cc8 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -94,7 +94,8 @@ use datafusion_expr::physical_planning_context::{
use datafusion_expr::utils::{expr_to_columns, split_conjunction};
use datafusion_expr::{
Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension,
- FetchType, Filter, JoinType, Operator, RecursiveQuery, SkipType, StringifiedPlan,
+ FetchType, Filter, JoinType, MaterializedCte, MaterializedCteId, MaterializedCtePlan,
+ MaterializedCteScan, Operator, RecursiveQuery, SkipType, StringifiedPlan,
WindowFrame, WindowFrameBound, WriteOp,
};
use datafusion_physical_expr::aggregate::{
@@ -260,19 +261,14 @@ impl DefaultPhysicalPlanner {
///
/// Returns `None` when `plan` has no materialized CTE.
///
-/// [`MaterializedCte`]: datafusion_expr::MaterializedCte
/// [`ViewTable`]: crate::datasource::ViewTable
fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>> {
- fn as_cte(plan: &LogicalPlan) -> Option<&datafusion_expr::MaterializedCte> {
- match plan {
- LogicalPlan::Extension(Extension { node }) => node.as_any().downcast_ref(),
- _ => None,
- }
- }
-
let mut has_cte = false;
plan.apply_with_subqueries(|node| {
- if as_cte(node).is_some() {
+ if matches!(
+ node,
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_))
+ ) {
has_cte = true;
return Ok(TreeNodeRecursion::Stop);
}
@@ -284,53 +280,40 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
plan.clone()
.transform_down_with_subqueries(|node| {
- let Some(cte) = as_cte(&node) else {
+ let LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) = node else {
return Ok(Transformed::no(node));
};
let old_id = cte.id;
- let new_id = datafusion_expr::MaterializedCteId::next();
- let continuation = cte
- .continuation
- .clone()
- .transform_down_with_subqueries(|node| {
+ let new_id = MaterializedCteId::next();
+ let continuation = Arc::unwrap_or_clone(cte.continuation)
+ .transform_down_with_subqueries(|node| match node {
// A nested occurrence of the same CTE owns the scans below
// it, and is renumbered on its own.
- if as_cte(&node).is_some_and(|nested| nested.id == old_id) {
- return Ok(Transformed::new(
- node,
- false,
- TreeNodeRecursion::Jump,
- ));
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ ref nested,
+ )) if nested.id == old_id => {
+ Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
}
- let LogicalPlan::Extension(Extension { node: scan }) = &node else {
- return Ok(Transformed::no(node));
- };
- match scan
- .as_any()
- .downcast_ref::<datafusion_expr::MaterializedCteScan>()
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))
+ if scan.id == old_id =>
{
- Some(scan) if scan.id == old_id => {
- let scan = datafusion_expr::MaterializedCteScan {
+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(
+ MaterializedCtePlan::Scan(MaterializedCteScan {
id: new_id,
- ..scan.clone()
- };
- Ok(Transformed::yes(LogicalPlan::Extension(Extension {
- node: Arc::new(scan),
- })))
- }
- _ => Ok(Transformed::no(node)),
+ ..scan
+ }),
+ )))
}
+ _ => Ok(Transformed::no(node)),
})?
.data;
- let cte = datafusion_expr::MaterializedCte {
- id: new_id,
- name: cte.name.clone(),
- cte: cte.cte.clone(),
- continuation,
- };
- Ok(Transformed::yes(LogicalPlan::Extension(Extension {
- node: Arc::new(cte),
- })))
+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(
+ MaterializedCtePlan::Cte(MaterializedCte {
+ id: new_id,
+ continuation: Arc::new(continuation),
+ ..cte
+ }),
+ )))
})
.map(|t| Some(t.data))
}
@@ -838,6 +821,16 @@ impl DefaultPhysicalPlanner {
self.plan_describe(&Arc::clone(schema), output_schema)?
}
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => {
+ // Unbound until `bind_materialized_cte_scans` runs on the whole plan.
+ Arc::new(MaterializedCteScanExec::new(
+ scan.id.as_u64(),
+ &scan.name,
+ Arc::clone(scan.schema.inner()),
+ session_state.config().target_partitions(),
+ ))
+ }
+
// 1 Child
LogicalPlan::Copy(CopyTo {
input,
@@ -1871,31 +1864,10 @@ impl DefaultPhysicalPlanner {
)?)
}
- // N Children
- LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
- LogicalPlan::Extension(Extension { node })
- if node.as_any().is::<datafusion_expr::MaterializedCte>() =>
- {
- let cte = node
- .as_any()
- .downcast_ref::<datafusion_expr::MaterializedCte>()
- .unwrap();
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
let [body, continuation] = children.two()?;
// A CTE that nothing reads is never evaluated.
- let mut is_referenced = false;
- cte.continuation.apply_with_subqueries(|node| {
- if let LogicalPlan::Extension(Extension { node }) = node
- && let Some(scan) =
- node.as_any()
- .downcast_ref::<datafusion_expr::MaterializedCteScan>()
- && scan.id == cte.id
- {
- is_referenced = true;
- return Ok(TreeNodeRecursion::Stop);
- }
- Ok(TreeNodeRecursion::Continue)
- })?;
- if !is_referenced {
+ if cte.scans()?.is_empty() {
return Ok(continuation);
}
// The body is fully buffered before a scan yields a row, so an
@@ -1910,21 +1882,9 @@ impl DefaultPhysicalPlanner {
Arc::new(MaterializedCteBuffer::new(cte.id.as_u64(), &cte.name));
Arc::new(MaterializedCteExec::new(body, continuation, buffer))
}
- LogicalPlan::Extension(Extension { node })
- if node.as_any().is::<datafusion_expr::MaterializedCteScan>() =>
- {
- let scan = node
- .as_any()
- .downcast_ref::<datafusion_expr::MaterializedCteScan>()
- .unwrap();
- // Unbound until `bind_materialized_cte_scans` runs on the whole plan.
- Arc::new(MaterializedCteScanExec::new(
- scan.id.as_u64(),
- &scan.name,
- Arc::clone(scan.schema.inner()),
- session_state.config().target_partitions(),
- ))
- }
+
+ // N Children
+ LogicalPlan::Union(_) => UnionExec::try_new(children.vec())?,
LogicalPlan::Extension(Extension { node }) => {
let mut maybe_plan = None;
let children = children.vec();
@@ -2730,7 +2690,8 @@ fn extract_dml_filters(
| LogicalPlan::Ddl(_)
| LogicalPlan::Copy(_)
| LogicalPlan::Unnest(_)
- | LogicalPlan::RecursiveQuery(_) => {
+ | LogicalPlan::RecursiveQuery(_)
+ | LogicalPlan::MaterializedCte(_) => {
// No filters to extract from leaf/meta plans
}
// Plans with inputs (may contain filters in children)
diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs
index 843c06c6fd..80ab609892 100644
--- a/datafusion/expr/src/logical_plan/display.rs
+++ b/datafusion/expr/src/logical_plan/display.rs
@@ -22,9 +22,10 @@ use std::fmt;
use crate::{
Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter,
- Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition,
- Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest,
- Values, Window, WriteOp, expr_vec_fmt,
+ Join, Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, MaterializedCteScan,
+ Partitioning, Projection, RecursiveQuery, Repartition, Sort, Subquery, SubqueryAlias,
+ TableProviderFilterPushDown, TableScan, Unnest, Values, Window, WriteOp,
+ expr_vec_fmt,
};
use crate::dml::CopyTo;
@@ -707,6 +708,23 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> {
"Detail": format!("{:?}", e.node)
)
}
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
+ name,
+ ..
+ })) => {
+ pg_fields!(
+ "Node Type": "MaterializedCte",
+ "Name": name,
+ )
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(
+ MaterializedCteScan { name, .. },
+ )) => {
+ pg_fields!(
+ "Node Type": "MaterializedCteScan",
+ "Name": name,
+ )
+ }
LogicalPlan::DescribeTable(DescribeTable { .. }) => {
pg_fields!(
"Node Type": "DescribeTable"
diff --git a/datafusion/expr/src/logical_plan/invariants.rs b/datafusion/expr/src/logical_plan/invariants.rs
index 0614a8602c..fbc6cf4872 100644
--- a/datafusion/expr/src/logical_plan/invariants.rs
+++ b/datafusion/expr/src/logical_plan/invariants.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use crate::logical_plan::MaterializedCtePlan;
use datafusion_common::{
DFSchemaRef, Result, assert_or_internal_err, plan_err,
tree_node::{TreeNode, TreeNodeRecursion},
@@ -355,6 +356,9 @@ fn check_inner_plan(inner_plan: &LogicalPlan) -> Result<()> {
}
},
LogicalPlan::Extension(_) => Ok(()),
+ // A materialized CTE is not checked, as when it was an extension
+ // node. `PullUpCorrelatedExpr` rewrites its continuation, or stops.
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(_)) => Ok(()),
plan => check_no_outer_references(plan),
}
}
diff --git a/datafusion/expr/src/logical_plan/materialized_cte.rs b/datafusion/expr/src/logical_plan/materialized_cte.rs
index aaf8f46dc6..037fc07b42 100644
--- a/datafusion/expr/src/logical_plan/materialized_cte.rs
+++ b/datafusion/expr/src/logical_plan/materialized_cte.rs
@@ -17,11 +17,12 @@
//! Logical nodes for `WITH x AS MATERIALIZED (...)`.
//!
-//! A materialized CTE is planned as one [`MaterializedCte`] node, which owns the
-//! CTE body and the rest of the query (the "continuation"), and one
+//! A materialized CTE is planned as one [`MaterializedCte`] node, which owns
+//! the CTE body and the rest of the query (the "continuation"), and one
//! [`MaterializedCteScan`] leaf for each reference to the CTE inside the
-//! continuation. The body is therefore optimized and executed once, and the
-//! scans read its buffered output.
+//! continuation. Both are in the [`LogicalPlan::MaterializedCte`] variant,
+//! as cases of [`MaterializedCtePlan`]. The body is therefore optimized and executed once, and
+//! the scans read its buffered output.
//!
//! The scans are leaves, so filters and projections of one reference are not
//! pushed into the shared body. This is the same optimization fence that
@@ -30,11 +31,13 @@
use std::cmp::Ordering;
use std::fmt;
use std::hash::Hash;
+use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering};
-use datafusion_common::{DFSchemaRef, Result, assert_eq_or_internal_err};
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::{DFSchemaRef, Result};
-use crate::{Expr, LogicalPlan, UserDefinedLogicalNodeCore};
+use crate::LogicalPlan;
/// Identifies one materialized CTE and binds its scans to it.
///
@@ -62,91 +65,198 @@ impl fmt::Display for MaterializedCteId {
}
}
+/// A node of a materialized CTE, in [`LogicalPlan::MaterializedCte`].
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]
+pub enum MaterializedCtePlan {
+ /// Computes the body of the CTE once and evaluates the continuation
+ Cte(MaterializedCte),
+ /// Reads the buffered output of the CTE. This is a leaf.
+ Scan(MaterializedCteScan),
+}
+
/// Computes `cte` once and evaluates `continuation`, in which every
/// [`MaterializedCteScan`] with the same `id` reads the buffered output of
/// `cte`. The output of this node is the output of `continuation`.
+///
+/// The node has two inputs with different roles, in this order:
+///
+/// 1. `cte`: the body of the CTE. It is shared by all the scans, so a rule
+/// must not change its schema, and must not push into it a filter, limit
+/// or projection that belongs to one scan only.
+/// 2. `continuation`: the query that declared the CTE.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct MaterializedCte {
+ /// Binds the scans in `continuation` to this node
pub id: MaterializedCteId,
+ /// Name of the CTE, used for display and error messages only
pub name: String,
- pub cte: LogicalPlan,
- pub continuation: LogicalPlan,
+ /// The body of the CTE
+ pub cte: Arc<LogicalPlan>,
+ /// The query that declared the CTE and reads it
+ pub continuation: Arc<LogicalPlan>,
}
-/// Orders by `id` and `name`. Nodes with the same `id` and `name` but a
-/// different `cte` or `continuation` are not comparable, so `Some(Equal)` is
-/// returned only for equal nodes.
-impl PartialOrd for MaterializedCte {
- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
- match self
- .id
- .cmp(&other.id)
- .then_with(|| self.name.cmp(&other.name))
- {
- Ordering::Equal if self != other => None,
- ord => Some(ord),
+impl MaterializedCte {
+ /// Create a new node.
+ ///
+ /// Returns an error if a scan of this CTE in `continuation` (also in a
+ /// subquery expression) does not have the column names and types of `cte`.
+ pub fn try_new(
+ id: MaterializedCteId,
+ name: impl Into<String>,
+ cte: Arc<LogicalPlan>,
+ continuation: Arc<LogicalPlan>,
+ ) -> Result<Self> {
+ let name = name.into();
+ for scan in MaterializedCteScan::find_in(&continuation, id)? {
+ scan.schema
+ .has_equivalent_names_and_types(cte.schema())
+ .map_err(|e| {
+ e.context(format!(
+ "A scan of the materialized CTE {name} does not have the schema of the CTE"
+ ))
+ })?;
}
+ Ok(Self {
+ id,
+ name,
+ cte,
+ continuation,
+ })
}
-}
-impl UserDefinedLogicalNodeCore for MaterializedCte {
- fn name(&self) -> &str {
- "MaterializedCte"
+ /// The id that binds the scans in the continuation to this node.
+ pub fn id(&self) -> MaterializedCteId {
+ self.id
}
- fn inputs(&self) -> Vec<&LogicalPlan> {
- vec![&self.cte, &self.continuation]
+ /// The name of the CTE.
+ pub fn name(&self) -> &str {
+ &self.name
}
- fn schema(&self) -> &DFSchemaRef {
- self.continuation.schema()
+ /// The body of the CTE, which is computed once.
+ pub fn body(&self) -> &Arc<LogicalPlan> {
+ &self.cte
+ }
+
+ /// The query that declared the CTE. It gives this node its output.
+ pub fn continuation(&self) -> &Arc<LogicalPlan> {
+ &self.continuation
}
- fn expressions(&self) -> Vec<Expr> {
- vec![]
+ /// The schema of this node, which is the schema of the continuation.
+ pub fn schema(&self) -> &DFSchemaRef {
+ self.continuation.schema()
}
- fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "MaterializedCte: name={}", self.name)
+ /// The scans in the continuation that read this CTE, also in subquery
+ /// expressions. See [`MaterializedCteScan::find_in`].
+ pub fn scans(&self) -> Result<Vec<MaterializedCteScan>> {
+ MaterializedCteScan::find_in(&self.continuation, self.id)
}
- fn with_exprs_and_inputs(
+ /// Return a node with the same id and name, and new inputs. The schemas
+ /// of the scans are not checked again.
+ pub fn with_new_inputs(
&self,
- exprs: Vec<Expr>,
- mut inputs: Vec<LogicalPlan>,
- ) -> Result<Self> {
- assert_eq_or_internal_err!(exprs.len(), 0, "MaterializedCte takes no exprs");
- assert_eq_or_internal_err!(inputs.len(), 2, "MaterializedCte takes 2 inputs");
- let continuation = inputs.pop().unwrap();
- let cte = inputs.pop().unwrap();
- Ok(Self {
+ cte: Arc<LogicalPlan>,
+ continuation: Arc<LogicalPlan>,
+ ) -> Self {
+ Self {
id: self.id,
name: self.name.clone(),
cte,
continuation,
- })
+ }
}
+}
- fn necessary_children_exprs(
- &self,
- output_columns: &[usize],
- ) -> Option<Vec<Vec<usize>>> {
- // The scans need every column of the body, since they are leaves whose
- // schema is fixed at planning time. The continuation only needs the
- // columns the parent asks for.
- let all_cte_columns = (0..self.cte.schema().fields().len()).collect();
- Some(vec![all_cte_columns, output_columns.to_vec()])
+/// Orders by `id` and `name`. Nodes with the same `id` and `name` but a
+/// different `cte` or `continuation` are not comparable, so `Some(Equal)` is
+/// returned only for equal nodes.
+impl PartialOrd for MaterializedCte {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ match self
+ .id
+ .cmp(&other.id)
+ .then_with(|| self.name.cmp(&other.name))
+ {
+ Ordering::Equal if self != other => None,
+ ord => Some(ord),
+ }
}
}
/// Reads the buffered output of the [`MaterializedCte`] with the same `id`.
+///
+/// This is a leaf. Its schema is the schema of the CTE body and is fixed when
+/// the scan is created.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct MaterializedCteScan {
+ /// The id of the [`MaterializedCte`] that this scan reads
pub id: MaterializedCteId,
+ /// Name of the CTE, used for display and error messages only
pub name: String,
+ /// The schema of the CTE body
pub schema: DFSchemaRef,
}
+impl MaterializedCteScan {
+ /// Create a scan of the CTE `id`, whose body has the schema `schema`.
+ pub fn new(
+ id: MaterializedCteId,
+ name: impl Into<String>,
+ schema: DFSchemaRef,
+ ) -> Self {
+ Self {
+ id,
+ name: name.into(),
+ schema,
+ }
+ }
+
+ /// The id of the [`MaterializedCte`] that this scan reads.
+ pub fn id(&self) -> MaterializedCteId {
+ self.id
+ }
+
+ /// The name of the CTE.
+ pub fn name(&self) -> &str {
+ &self.name
+ }
+
+ /// The schema of this scan.
+ pub fn schema(&self) -> &DFSchemaRef {
+ &self.schema
+ }
+
+ /// Return a copy of each scan of the CTE `id` in `plan`, also in subquery
+ /// expressions.
+ ///
+ /// The search does not go below a [`MaterializedCte`] with the same `id`
+ /// in `plan`. Such a node is a separate occurrence of the CTE (the id is
+ /// copied when a plan is cloned) and owns the scans below it.
+ pub fn find_in(plan: &LogicalPlan, id: MaterializedCteId) -> Result<Vec<Self>> {
+ let mut scans = vec![];
+ plan.apply_with_subqueries(|node| match node {
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte))
+ if cte.id == id =>
+ {
+ Ok(TreeNodeRecursion::Jump)
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))
+ if scan.id == id =>
+ {
+ scans.push(scan.clone());
+ Ok(TreeNodeRecursion::Continue)
+ }
+ _ => Ok(TreeNodeRecursion::Continue),
+ })?;
+ Ok(scans)
+ }
+}
+
/// Orders by `id` and `name`. Scans with the same `id` and `name` but a
/// different `schema` are not comparable.
impl PartialOrd for MaterializedCteScan {
@@ -162,53 +272,39 @@ impl PartialOrd for MaterializedCteScan {
}
}
-impl UserDefinedLogicalNodeCore for MaterializedCteScan {
- fn name(&self) -> &str {
- "MaterializedCteScan"
- }
-
- fn inputs(&self) -> Vec<&LogicalPlan> {
- vec![]
- }
-
- fn schema(&self) -> &DFSchemaRef {
- &self.schema
- }
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::logical_plan::builder::LogicalPlanBuilder;
+ use crate::{EmptyRelation, exists};
+ use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::DFSchema;
- fn expressions(&self) -> Vec<Expr> {
- vec![]
+ fn empty(produce_one_row: bool) -> Arc<LogicalPlan> {
+ Arc::new(LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row,
+ schema: Arc::new(DFSchema::empty()),
+ }))
}
- fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "MaterializedCteScan: name={}", self.name)
+ fn schema(data_type: DataType) -> DFSchemaRef {
+ let schema = Schema::new(vec![Field::new("a", data_type, true)]);
+ Arc::new(DFSchema::try_from_qualified_schema("c", &schema).unwrap())
}
- fn with_exprs_and_inputs(
- &self,
- exprs: Vec<Expr>,
- inputs: Vec<LogicalPlan>,
- ) -> Result<Self> {
- assert_eq_or_internal_err!(exprs.len(), 0, "MaterializedCteScan takes no exprs");
- assert_eq_or_internal_err!(
- inputs.len(),
- 0,
- "MaterializedCteScan takes no inputs"
- );
- Ok(self.clone())
+ fn body(data_type: DataType) -> Arc<LogicalPlan> {
+ Arc::new(LogicalPlan::EmptyRelation(EmptyRelation {
+ produce_one_row: false,
+ schema: schema(data_type),
+ }))
}
-}
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::EmptyRelation;
- use std::sync::Arc;
-
- fn empty(produce_one_row: bool) -> LogicalPlan {
- LogicalPlan::EmptyRelation(EmptyRelation {
- produce_one_row,
- schema: Arc::new(datafusion_common::DFSchema::empty()),
- })
+ fn scan(id: MaterializedCteId, data_type: DataType) -> LogicalPlan {
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(MaterializedCteScan::new(
+ id,
+ "c",
+ schema(data_type),
+ )))
}
#[test]
@@ -233,4 +329,76 @@ mod tests {
};
assert_eq!(a.partial_cmp(&c), Some(Ordering::Less));
}
+
+ #[test]
+ fn try_new_checks_the_schema_of_the_scans() -> Result<()> {
+ let id = MaterializedCteId::next();
+ let cte = MaterializedCte::try_new(
+ id,
+ "c",
+ body(DataType::Int32),
+ Arc::new(scan(id, DataType::Int32)),
+ )?;
+ assert_eq!(cte.id(), id);
+ assert_eq!(cte.name(), "c");
+ assert_eq!(cte.schema(), cte.continuation().schema());
+ assert_eq!(cte.scans()?.len(), 1);
+
+ let plan = LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte));
+ assert_eq!(plan.inputs().len(), 2);
+ assert_eq!(
+ plan.display_indent().to_string(),
+ "MaterializedCte: name=c\n EmptyRelation: rows=0\n MaterializedCteScan: name=c"
+ );
+
+ let err = MaterializedCte::try_new(
+ id,
+ "c",
+ body(DataType::Int32),
+ Arc::new(scan(id, DataType::Utf8)),
+ )
+ .unwrap_err();
+ assert!(
+ err.to_string().contains(
+ "A scan of the materialized CTE c does not have the schema of the CTE"
+ ),
+ "{err}"
+ );
+
+ // A scan of another CTE is not checked.
+ MaterializedCte::try_new(
+ id,
+ "c",
+ body(DataType::Int32),
+ Arc::new(scan(MaterializedCteId::next(), DataType::Utf8)),
+ )?;
+ Ok(())
+ }
+
+ #[test]
+ fn find_in_looks_in_subqueries_and_stops_at_the_same_cte() -> Result<()> {
+ let id = MaterializedCteId::next();
+ let other = MaterializedCteId::next();
+ // The union holds one scan, one scan in a subquery expression, one
+ // scan of another CTE, and a nested occurrence of the same CTE, which
+ // owns its scan.
+ let plan = LogicalPlanBuilder::from(scan(id, DataType::Int32))
+ .filter(exists(Arc::new(scan(id, DataType::Int32))))?
+ .union(scan(other, DataType::Int32))?
+ .union(LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ MaterializedCte::try_new(
+ id,
+ "c",
+ body(DataType::Int32),
+ Arc::new(scan(id, DataType::Int32)),
+ )?,
+ )))?
+ .build()?;
+ assert_eq!(MaterializedCteScan::find_in(&plan, id)?.len(), 2);
+ assert_eq!(MaterializedCteScan::find_in(&plan, other)?.len(), 1);
+ assert!(
+ MaterializedCteScan::find_in(&plan, MaterializedCteId::next())?.is_empty()
+ );
+ Ok(())
+ }
}
diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rs
index 81483dae7f..794747c733 100644
--- a/datafusion/expr/src/logical_plan/mod.rs
+++ b/datafusion/expr/src/logical_plan/mod.rs
@@ -61,4 +61,6 @@ pub use datafusion_common::format::ExplainFormat;
pub use display::display_schema;
pub use extension::{UserDefinedLogicalNode, UserDefinedLogicalNodeCore};
-pub use materialized_cte::{MaterializedCte, MaterializedCteId, MaterializedCteScan};
+pub use materialized_cte::{
+ MaterializedCte, MaterializedCteId, MaterializedCtePlan, MaterializedCteScan,
+};
diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs
index e0ccc5dc73..477e9c73d5 100644
--- a/datafusion/expr/src/logical_plan/plan.rs
+++ b/datafusion/expr/src/logical_plan/plan.rs
@@ -39,7 +39,9 @@ use crate::expr_rewriter::{
};
use crate::logical_plan::display::{GraphvizVisitor, IndentVisitor};
use crate::logical_plan::extension::UserDefinedLogicalNode;
-use crate::logical_plan::materialized_cte::MaterializedCte;
+use crate::logical_plan::materialized_cte::{
+ MaterializedCte, MaterializedCtePlan, MaterializedCteScan,
+};
use crate::logical_plan::{DmlStatement, Statement, WriteOp};
use crate::utils::{
check_aggregate_and_window_nesting, check_no_window_functions,
@@ -363,6 +365,9 @@ pub enum LogicalPlan {
/// Match each left row with at most one ordered row from the right input.
/// This is used to implement SQL `ASOF JOIN`.
AsOfJoin(AsOfJoin),
+ /// A node of a SQL `WITH x AS MATERIALIZED (...)` CTE. See
+ /// [`MaterializedCtePlan`].
+ MaterializedCte(MaterializedCtePlan),
}
impl Default for LogicalPlan {
@@ -419,6 +424,8 @@ impl LogicalPlan {
LogicalPlan::Explain(explain) => &explain.schema,
LogicalPlan::Analyze(analyze) => &analyze.schema,
LogicalPlan::Extension(extension) => extension.node.schema(),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => cte.schema(),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan)) => &scan.schema,
LogicalPlan::Union(Union { schema, .. }) => schema,
LogicalPlan::DescribeTable(DescribeTable { output_schema, .. }) => {
output_schema
@@ -537,6 +544,12 @@ impl LogicalPlan {
LogicalPlan::Subquery(Subquery { subquery, .. }) => vec![subquery],
LogicalPlan::SubqueryAlias(SubqueryAlias { input, .. }) => vec![input],
LogicalPlan::Extension(extension) => extension.node.inputs(),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
+ cte,
+ continuation,
+ ..
+ })) => vec![cte, continuation],
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => vec![],
LogicalPlan::Union(Union { inputs, .. }) => {
inputs.iter().map(|arc| arc.as_ref()).collect()
}
@@ -673,12 +686,11 @@ impl LogicalPlan {
.map_or(Ok(None), |v| v.map(Some))
}
// A materialized CTE outputs the output of its continuation.
- LogicalPlan::Extension(Extension { node }) => {
- match node.as_any().downcast_ref::<MaterializedCte>() {
- Some(cte) => cte.continuation.head_output_expr(),
- None => Ok(None),
- }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
+ cte.continuation.head_output_expr()
}
+ LogicalPlan::Extension(_)
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => Ok(None),
LogicalPlan::Subquery(_) => Ok(None),
LogicalPlan::EmptyRelation(_)
| LogicalPlan::Statement(_)
@@ -811,6 +823,10 @@ impl LogicalPlan {
}) => SubqueryAlias::try_new(input, alias).map(LogicalPlan::SubqueryAlias),
LogicalPlan::Limit(_) => Ok(self),
LogicalPlan::Ddl(_) => Ok(self),
+ // A `Cte` takes the schema of its continuation, and does not store
+ // it. A `Scan` keeps the schema of the CTE body, which also keeps
+ // its schema.
+ LogicalPlan::MaterializedCte(_) => Ok(self),
LogicalPlan::Extension(Extension { node }) => {
// todo make an API that does not require cloning
// This requires a copy of the extension nodes expressions and inputs
@@ -1228,6 +1244,18 @@ impl LogicalPlan {
LogicalPlan::Extension(e) => Ok(LogicalPlan::Extension(Extension {
node: e.node.with_exprs_and_inputs(expr, inputs)?,
})),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
+ self.assert_no_expressions(expr)?;
+ let (body, continuation) = self.only_two_inputs(inputs)?;
+ Ok(LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ cte.with_new_inputs(Arc::new(body), Arc::new(continuation)),
+ )))
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => {
+ self.assert_no_expressions(expr)?;
+ self.assert_no_inputs(inputs)?;
+ Ok(self.clone())
+ }
LogicalPlan::Union(Union { schema, .. }) => {
self.assert_no_expressions(expr)?;
let input_schema = inputs[0].schema();
@@ -1603,7 +1631,8 @@ impl LogicalPlan {
| LogicalPlan::Copy(_)
| LogicalPlan::DescribeTable(_)
| LogicalPlan::Statement(_)
- | LogicalPlan::Extension(_) => None,
+ | LogicalPlan::Extension(_)
+ | LogicalPlan::MaterializedCte(_) => None,
}
}
@@ -1680,7 +1709,8 @@ impl LogicalPlan {
| LogicalPlan::Copy(_)
| LogicalPlan::DescribeTable(_)
| LogicalPlan::Statement(_)
- | LogicalPlan::Extension(_) => 0,
+ | LogicalPlan::Extension(_)
+ | LogicalPlan::MaterializedCte(_) => 0,
}
}
@@ -1713,6 +1743,7 @@ impl LogicalPlan {
LogicalPlan::Explain(_) => Ok(None),
LogicalPlan::Analyze(_) => Ok(None),
LogicalPlan::Extension(_) => Ok(None),
+ LogicalPlan::MaterializedCte(_) => Ok(None),
LogicalPlan::Distinct(_) => Ok(None),
LogicalPlan::Dml(_) => Ok(None),
LogicalPlan::Ddl(_) => Ok(None),
@@ -1752,6 +1783,7 @@ impl LogicalPlan {
LogicalPlan::Explain(_) => Ok(None),
LogicalPlan::Analyze(_) => Ok(None),
LogicalPlan::Extension(_) => Ok(None),
+ LogicalPlan::MaterializedCte(_) => Ok(None),
LogicalPlan::Distinct(_) => Ok(None),
LogicalPlan::Dml(_) => Ok(None),
LogicalPlan::Ddl(_) => Ok(None),
@@ -2498,6 +2530,16 @@ impl LogicalPlan {
LogicalPlan::Analyze { .. } => write!(f, "Analyze"),
LogicalPlan::Union(_) => write!(f, "Union"),
LogicalPlan::Extension(e) => e.node.fmt_for_explain(f),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ MaterializedCte { name, .. },
+ )) => {
+ write!(f, "MaterializedCte: name={name}")
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(
+ MaterializedCteScan { name, .. },
+ )) => {
+ write!(f, "MaterializedCteScan: name={name}")
+ }
LogicalPlan::DescribeTable(DescribeTable { .. }) => {
write!(f, "DescribeTable")
}
diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs
index 9e961a1f11..a4739b3024 100644
--- a/datafusion/expr/src/logical_plan/tree_node.rs
+++ b/datafusion/expr/src/logical_plan/tree_node.rs
@@ -43,10 +43,10 @@ use crate::logical_plan::plan::RangePartitioning;
use crate::{
Aggregate, Analyze, AsOfJoin, AsOfMatch, CreateMemoryTable, CreateView, DdlStatement,
Distinct, DistinctOn, DmlStatement, Execute, Explain, Expr, Extension, Filter, Join,
- Limit, LogicalPlan, Partitioning, Prepare, Projection, RecursiveQuery, Repartition,
- Sort, Statement, Subquery, SubqueryAlias, TableScan, Union, Unnest,
- UserDefinedLogicalNode, Values, Window, WriteOp, builder::unnest_with_options,
- dml::CopyTo,
+ Limit, LogicalPlan, MaterializedCte, MaterializedCtePlan, Partitioning, Prepare,
+ Projection, RecursiveQuery, Repartition, Sort, Statement, Subquery, SubqueryAlias,
+ TableScan, Union, Unnest, UserDefinedLogicalNode, Values, Window, WriteOp,
+ builder::unnest_with_options, dml::CopyTo,
};
use datafusion_common::tree_node::TreeNodeRefContainer;
@@ -377,6 +377,25 @@ impl TreeNode for LogicalPlan {
})
},
),
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
+ id,
+ name,
+ cte,
+ continuation,
+ })) => {
+ (cte, continuation)
+ .map_elements(f)?
+ .update_data(|(cte, continuation)| {
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ MaterializedCte {
+ id,
+ name,
+ cte,
+ continuation,
+ },
+ ))
+ })
+ }
LogicalPlan::Statement(stmt) => match stmt {
Statement::Prepare(p) => p
.input
@@ -389,6 +408,7 @@ impl TreeNode for LogicalPlan {
LogicalPlan::TableScan { .. }
| LogicalPlan::EmptyRelation { .. }
| LogicalPlan::Values { .. }
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))
| LogicalPlan::DescribeTable(_) => Transformed::no(self),
})
}
@@ -516,6 +536,7 @@ impl LogicalPlan {
// plans without expressions
LogicalPlan::EmptyRelation(_)
| LogicalPlan::RecursiveQuery(_)
+ | LogicalPlan::MaterializedCte(_)
| LogicalPlan::Subquery(_)
| LogicalPlan::SubqueryAlias(_)
| LogicalPlan::Analyze(_)
@@ -803,6 +824,7 @@ impl LogicalPlan {
// plans without expressions
LogicalPlan::EmptyRelation(_)
| LogicalPlan::RecursiveQuery(_)
+ | LogicalPlan::MaterializedCte(_)
| LogicalPlan::Subquery(_)
| LogicalPlan::SubqueryAlias(_)
| LogicalPlan::Analyze(_)
diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs
index bd3aea11ae..abc756879a 100644
--- a/datafusion/optimizer/src/analyzer/type_coercion.rs
+++ b/datafusion/optimizer/src/analyzer/type_coercion.rs
@@ -43,9 +43,7 @@ use datafusion_expr::expr::{
};
use datafusion_expr::expr_rewriter::coerce_plan_expr_for_schema;
use datafusion_expr::expr_schema::{cast_subquery, cast_subquery_columns};
-use datafusion_expr::logical_plan::{
- Extension, MaterializedCte, MaterializedCteId, MaterializedCteScan, Subquery,
-};
+use datafusion_expr::logical_plan::Subquery;
use datafusion_expr::type_coercion::binary::{
comparison_coercion_with_session_timezone, like_coercion, regex_coercion,
type_union_coercion,
@@ -60,7 +58,8 @@ use datafusion_expr::type_coercion::{
use datafusion_expr::utils::merge_schema;
use datafusion_expr::{
AsOfJoin, AsOfMatch, Cast, DmlStatement, Expr, ExprSchemable, Join, Limit,
- LogicalPlan, Operator, Projection, Union, ValueOrLambda, WindowFrame,
+ LogicalPlan, MaterializedCte, MaterializedCteId, MaterializedCtePlan,
+ MaterializedCteScan, Operator, Projection, Union, ValueOrLambda, WindowFrame,
WindowFrameBound, WindowFrameUnits, WriteOp, is_false, is_not_false, is_not_true,
is_not_unknown, is_true, is_unknown, lit, not,
};
@@ -135,38 +134,32 @@ fn coerce_plan_and_subqueries(
static EMPTY_SCHEMA: LazyLock<DFSchema> = LazyLock::new(DFSchema::empty);
plan.transform_down_up_with_subqueries(
- |plan| {
- let cte = match &plan {
- LogicalPlan::Extension(Extension { node }) => {
- node.as_any().downcast_ref::<MaterializedCte>().cloned()
- }
- _ => None,
- };
- let Some(cte) = cte else {
- return Ok(Transformed::no(plan));
- };
+ |plan| match plan {
// Coercion can change the types of the body, for example the type
// of a `NULL` column of a `UNION`. Coerce the body first, and give
// its new schema to the scans before the continuation is coerced.
- let body = coerce_plan_and_subqueries(cte.cte, config)?;
- let continuation = with_materialized_cte_scan_schema(
- cte.continuation,
- cte.id,
- body.schema(),
- )?;
- let continuation = coerce_plan_and_subqueries(continuation, config)?;
- Ok(Transformed::new(
- LogicalPlan::Extension(Extension {
- node: Arc::new(MaterializedCte {
- id: cte.id,
- name: cte.name,
- cte: body,
- continuation,
- }),
- }),
- true,
- TreeNodeRecursion::Jump,
- ))
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
+ let body =
+ coerce_plan_and_subqueries(Arc::unwrap_or_clone(cte.cte), config)?;
+ let continuation = with_materialized_cte_scan_schema(
+ Arc::unwrap_or_clone(cte.continuation),
+ cte.id,
+ body.schema(),
+ )?;
+ let continuation = coerce_plan_and_subqueries(continuation, config)?;
+ Ok(Transformed::new(
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ MaterializedCte {
+ cte: Arc::new(body),
+ continuation: Arc::new(continuation),
+ ..cte
+ },
+ )),
+ true,
+ TreeNodeRecursion::Jump,
+ ))
+ }
+ _ => Ok(Transformed::no(plan)),
},
|plan| {
analyze_internal(
@@ -188,42 +181,32 @@ fn with_materialized_cte_scan_schema(
body_schema: &DFSchemaRef,
) -> Result<LogicalPlan> {
continuation
- .transform_down_with_subqueries(|plan| {
- let LogicalPlan::Extension(Extension { node }) = &plan else {
- return Ok(Transformed::no(plan));
- };
+ .transform_down_with_subqueries(|node| match node {
// A nested occurrence of the same CTE owns the scans below it.
- if node
- .as_any()
- .downcast_ref::<MaterializedCte>()
- .is_some_and(|nested| nested.id == id)
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref nested))
+ if nested.id == id =>
{
- return Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump));
+ Ok(Transformed::new(node, false, TreeNodeRecursion::Jump))
}
- let Some(scan) =
- node.as_any()
- .downcast_ref::<MaterializedCteScan>()
- .filter(|scan| {
- scan.id == id && scan.schema.inner() != body_schema.inner()
- })
- else {
- return Ok(Transformed::no(plan));
- };
- let qualifiers = scan.schema.iter().map(|(q, _)| q.cloned()).collect();
- let schema = DFSchema::from_field_specific_qualified_schema(
- qualifiers,
- body_schema.inner(),
- )?
- .with_functional_dependencies(
- body_schema.functional_dependencies().clone(),
- )?;
- Ok(Transformed::yes(LogicalPlan::Extension(Extension {
- node: Arc::new(MaterializedCteScan {
- id: scan.id,
- name: scan.name.clone(),
- schema: Arc::new(schema),
- }),
- })))
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(scan))
+ if scan.id == id && scan.schema.inner() != body_schema.inner() =>
+ {
+ let qualifiers = scan.schema.iter().map(|(q, _)| q.cloned()).collect();
+ let schema = DFSchema::from_field_specific_qualified_schema(
+ qualifiers,
+ body_schema.inner(),
+ )?
+ .with_functional_dependencies(
+ body_schema.functional_dependencies().clone(),
+ )?;
+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(
+ MaterializedCtePlan::Scan(MaterializedCteScan {
+ schema: Arc::new(schema),
+ ..scan
+ }),
+ )))
+ }
+ _ => Ok(Transformed::no(node)),
})
.map(|t| t.data)
}
diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs
index e0d43d7c15..2c5a55cc65 100644
--- a/datafusion/optimizer/src/common_subexpr_eliminate.rs
+++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs
@@ -586,7 +586,8 @@ impl OptimizerRule for CommonSubexprEliminate {
| LogicalPlan::Dml(_)
| LogicalPlan::Copy(_)
| LogicalPlan::Unnest(_)
- | LogicalPlan::RecursiveQuery(_) => {
+ | LogicalPlan::RecursiveQuery(_)
+ | LogicalPlan::MaterializedCte(_) => {
// This rule handles recursion itself in a `ApplyOrder::TopDown` like
// manner. Process uncorrelated subqueries in expressions
// (e.g., Expr::ScalarSubquery), then direct children.
diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs
index 246639b389..6253c90ebb 100644
--- a/datafusion/optimizer/src/decorrelate.rs
+++ b/datafusion/optimizer/src/decorrelate.rs
@@ -39,9 +39,9 @@ use datafusion_expr::logical_plan::{Join, JoinType};
use datafusion_expr::simplify::SimplifyContext;
use datafusion_expr::utils::{collect_subquery_cols, conjunction, split_conjunction};
use datafusion_expr::{
- Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, Extension,
- FetchType, LogicalPlan, LogicalPlanBuilder, MaterializedCte, Operator, SkipType,
- UserDefinedLogicalNodeCore, Window, expr, lit,
+ Aggregate, BinaryExpr, Cast, Distinct, EmptyRelation, Expr, ExprSchemable, FetchType,
+ LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCtePlan, Operator,
+ SkipType, Window, expr, lit,
};
/// This struct rewrite the sub query plan by pull up the correlated
@@ -208,35 +208,25 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
LogicalPlan::Subquery(_) => {
Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump))
}
- LogicalPlan::Extension(Extension { node })
- if node.as_any().is::<MaterializedCte>() =>
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(ref cte))
+ if !cte.cte.all_out_ref_exprs().is_empty() =>
{
- let cte = node.as_any().downcast_ref::<MaterializedCte>().unwrap();
- if !cte.cte.all_out_ref_exprs().is_empty() {
- // A correlated body would have to run once per outer row.
- return Ok(
- self.stop_pull_up(LogicalPlan::Extension(Extension { node }))
- );
- }
+ // A correlated body would have to run once per outer row.
+ Ok(self.stop_pull_up(plan))
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
// Only the continuation can hold correlated expressions. Rewrite
// it alone, so that the body shared by every scan keeps its schema.
- let continuation = cte.continuation.clone().rewrite(self)?;
- if !continuation.transformed {
- return Ok(Transformed::new(
- LogicalPlan::Extension(Extension { node }),
- false,
- TreeNodeRecursion::Jump,
- ));
- }
- let cte = cte.with_exprs_and_inputs(
- vec![],
- vec![cte.cte.clone(), continuation.data],
- )?;
+ let continuation =
+ Arc::unwrap_or_clone(cte.continuation).rewrite(self)?;
Ok(Transformed::new(
- LogicalPlan::Extension(Extension {
- node: Arc::new(cte),
- }),
- true,
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(
+ MaterializedCte {
+ continuation: Arc::new(continuation.data),
+ ..cte
+ },
+ )),
+ continuation.transformed,
TreeNodeRecursion::Jump,
))
}
@@ -350,6 +340,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
| LogicalPlan::SubqueryAlias(_)
| LogicalPlan::TableScan(_)
| LogicalPlan::EmptyRelation(_)
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))
| LogicalPlan::Values(_) => {
if plan.contains_outer_reference() {
// the unsupported cases, the plan expressions contain out
diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs
index 8fdb5775c9..6cf4047713 100644
--- a/datafusion/optimizer/src/optimize_projections/mod.rs
+++ b/datafusion/optimizer/src/optimize_projections/mod.rs
@@ -32,6 +32,7 @@ mod required_indices;
use crate::optimizer::ApplyOrder;
use crate::{OptimizerConfig, OptimizerRule};
+use datafusion_expr::logical_plan::MaterializedCtePlan;
use std::sync::Arc;
use datafusion_common::{
@@ -375,8 +376,18 @@ fn optimize_projections(
.collect()
}
}
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)) => {
+ // The scans are leaves whose schema is the schema of the body, so
+ // the body keeps every column. The output is the output of the
+ // continuation, so the continuation needs what the parent needs.
+ vec![
+ RequiredIndices::new_for_all_exprs(&cte.cte),
+ RequiredIndices::new_from_indices(indices.into_inner()),
+ ]
+ }
LogicalPlan::EmptyRelation(_)
| LogicalPlan::Values(_)
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))
| LogicalPlan::DescribeTable(_) => {
// These operators have no inputs, so stop the optimization process.
return Ok(Transformed::no(plan));
diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs
index 94583d545d..e3bf8eff17 100644
--- a/datafusion/optimizer/src/optimizer.rs
+++ b/datafusion/optimizer/src/optimizer.rs
@@ -36,9 +36,9 @@ use datafusion_expr::dml::CopyTo;
use datafusion_expr::logical_plan::LogicalPlan;
use datafusion_expr::{
Aggregate, Analyze, CreateMemoryTable, CreateView, DdlStatement, Distinct,
- DistinctOn, DmlStatement, Explain, Expr, Extension, Filter, Join, Limit, Projection,
- RecursiveQuery, Repartition, Sort, Statement, Subquery, SubqueryAlias, Union, Unnest,
- Window,
+ DistinctOn, DmlStatement, Explain, Expr, Extension, Filter, Join, Limit,
+ MaterializedCte, MaterializedCtePlan, Projection, RecursiveQuery, Repartition, Sort,
+ Statement, Subquery, SubqueryAlias, Union, Unnest, Window,
};
use crate::common_subexpr_eliminate::CommonSubexprEliminate;
@@ -443,6 +443,15 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(
let r = f(Arc::make_mut(recursive_term))?;
s || r
}
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(MaterializedCte {
+ cte,
+ continuation,
+ ..
+ })) => {
+ let b = f(Arc::make_mut(cte))?;
+ let c = f(Arc::make_mut(continuation))?;
+ b || c
+ }
LogicalPlan::Statement(Statement::Prepare(p)) => f(Arc::make_mut(&mut p.input))?,
LogicalPlan::Extension(Extension { node }) => {
let inputs = node.inputs();
@@ -471,6 +480,7 @@ fn map_children_mut<F: FnMut(&mut LogicalPlan) -> Result<bool>>(
LogicalPlan::TableScan { .. }
| LogicalPlan::EmptyRelation { .. }
| LogicalPlan::Values { .. }
+ | LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_))
| LogicalPlan::DescribeTable(_)
| LogicalPlan::Ddl(DdlStatement::CreateExternalTable(_))
| LogicalPlan::Ddl(DdlStatement::CreateCatalogSchema(_))
diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs
index ccde8aa07f..01a4fb0e9b 100644
--- a/datafusion/optimizer/src/push_down_filter.rs
+++ b/datafusion/optimizer/src/push_down_filter.rs
@@ -39,6 +39,7 @@
//!
//! [`PushDownLeafProjections`]: crate::extract_leaf_expressions::PushDownLeafProjections
+use datafusion_expr::logical_plan::MaterializedCtePlan;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -62,9 +63,8 @@ use datafusion_expr::utils::{
conjunction, expr_to_columns, split_conjunction, split_conjunction_owned,
};
use datafusion_expr::{
- BinaryExpr, Distinct, EmptyRelation, Expr, Extension, Filter, MaterializedCte,
- Operator, Projection, TableProviderFilterPushDown, UserDefinedLogicalNodeCore, and,
- or,
+ BinaryExpr, Distinct, EmptyRelation, Expr, Filter, Operator, Projection,
+ TableProviderFilterPushDown, and, or,
};
use crate::extract_leaf_expressions::is_pure_extraction_projection;
@@ -1403,22 +1403,22 @@ impl OptimizerRule for PushDownFilter {
LogicalPlan::TableScan(scan),
)))
}
- LogicalPlan::Extension(Extension { node })
- if node.as_any().is::<MaterializedCte>() =>
- {
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(mut cte)) => {
// The output is the output of the continuation, so the filter
// moves onto the continuation. The body is shared by every scan
// of the CTE and is never filtered for one of them.
- let cte = node.as_any().downcast_ref::<MaterializedCte>().unwrap();
- let continuation = LogicalPlan::Filter(Filter::new(
+ cte.continuation = Arc::new(LogicalPlan::Filter(Filter::new(
filter.predicate,
- Arc::new(cte.continuation.clone()),
- ));
- let cte = cte
- .with_exprs_and_inputs(vec![], vec![cte.cte.clone(), continuation])?;
- Ok(Transformed::yes(LogicalPlan::Extension(Extension {
- node: Arc::new(cte),
- })))
+ cte.continuation,
+ )));
+ Ok(Transformed::yes(LogicalPlan::MaterializedCte(
+ MaterializedCtePlan::Cte(cte),
+ )))
+ }
+ // A scan reads the shared body, so the filter stays above it.
+ scan @ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(_)) => {
+ filter.input = Arc::new(scan);
+ Ok(Transformed::no(LogicalPlan::Filter(filter)))
}
LogicalPlan::Extension(extension_plan) => {
// This check prevents the Filter from being removed when the extension node has no children,
diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs
index d86b27ff3c..9037fb8772 100644
--- a/datafusion/proto/src/logical_plan/mod.rs
+++ b/datafusion/proto/src/logical_plan/mod.rs
@@ -2411,6 +2411,11 @@ impl AsLogicalPlan for LogicalPlanNode {
LogicalPlan::DescribeTable(_) => Err(proto_error(
"LogicalPlan serde is not yet implemented for DescribeTable",
)),
+ LogicalPlan::MaterializedCte(_) => {
+ not_impl_err!(
+ "LogicalPlan serde is not yet implemented for MATERIALIZED CTEs"
+ )
+ }
LogicalPlan::RecursiveQuery(recursive) => {
let static_term = LogicalPlanNode::try_from_logical_plan(
recursive.static_term.as_ref(),
diff --git a/datafusion/sql/src/cte.rs b/datafusion/sql/src/cte.rs
index b6fbd21822..e5ff1b6367 100644
--- a/datafusion/sql/src/cte.rs
+++ b/datafusion/sql/src/cte.rs
@@ -25,8 +25,8 @@ use datafusion_common::{
tree_node::{TreeNode, TreeNodeRecursion},
};
use datafusion_expr::{
- Extension, LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCteId,
- MaterializedCteScan, TableSource,
+ LogicalPlan, LogicalPlanBuilder, MaterializedCte, MaterializedCteId,
+ MaterializedCtePlan, MaterializedCteScan, TableSource,
};
use sqlparser::ast::{CteAsMaterialized, Ident, Query, SetExpr, SetOperator, With};
@@ -40,15 +40,14 @@ pub(crate) struct PendingMaterializedCte {
impl PendingMaterializedCte {
/// Wrap `continuation` so that its scans of this CTE read one shared result.
- pub(crate) fn wrap(self, continuation: LogicalPlan) -> LogicalPlan {
- LogicalPlan::Extension(Extension {
- node: Arc::new(MaterializedCte {
- id: self.id,
- name: self.name,
- cte: self.cte,
- continuation,
- }),
- })
+ pub(crate) fn wrap(self, continuation: LogicalPlan) -> Result<LogicalPlan> {
+ MaterializedCte::try_new(
+ self.id,
+ self.name,
+ Arc::new(self.cte),
+ Arc::new(continuation),
+ )
+ .map(|cte| LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(cte)))
}
}
@@ -101,13 +100,13 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
// of the body. The body is already planned, so a reference
// of a recursive CTE to itself stays a work table scan.
let id = MaterializedCteId::next();
- let scan = LogicalPlan::Extension(Extension {
- node: Arc::new(MaterializedCteScan {
+ let scan = LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(
+ MaterializedCteScan::new(
id,
- name: cte_name.clone(),
- schema: Arc::clone(final_plan.schema()),
- }),
- });
+ cte_name.clone(),
+ Arc::clone(final_plan.schema()),
+ ),
+ ));
materialized.push(PendingMaterializedCte {
id,
name: cte_name.clone(),
diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs
index a943abe2f8..00651cd0b1 100644
--- a/datafusion/sql/src/query.rs
+++ b/datafusion/sql/src/query.rs
@@ -102,10 +102,10 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
return not_impl_err!("MATERIALIZED CTEs are not supported with SELECT INTO");
}
// A later CTE may read an earlier one, so the first CTE is the outermost.
- Ok(materialized_ctes
+ materialized_ctes
.into_iter()
.rev()
- .fold(plan, |plan, cte| cte.wrap(plan)))
+ .try_fold(plan, |plan, cte| cte.wrap(plan))
}
/// Apply pipe operators to a plan
diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs
index 83ced616e0..de6ba9443c 100644
--- a/datafusion/sql/src/unparser/plan.rs
+++ b/datafusion/sql/src/unparser/plan.rs
@@ -207,6 +207,9 @@ impl Unparser<'_> {
| LogicalPlan::DescribeTable(_)
| LogicalPlan::RecursiveQuery(_)
| LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"),
+ LogicalPlan::MaterializedCte(_) => {
+ not_impl_err!("Unparsing a MATERIALIZED CTE is not supported: {plan:?}")
+ }
}
}
diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs
index 15f59919a2..8b02e09f4c 100644
--- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs
+++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs
@@ -37,7 +37,7 @@ pub use sort_rel::*;
use crate::logical_plan::producer::SubstraitProducer;
use datafusion::common::not_impl_err;
-use datafusion::logical_expr::LogicalPlan;
+use datafusion::logical_expr::{LogicalPlan, MaterializedCtePlan};
use substrait::proto::Rel;
pub fn to_substrait_rel(
@@ -77,5 +77,11 @@ pub fn to_substrait_rel(
LogicalPlan::RecursiveQuery(plan) => {
not_impl_err!("Unsupported plan type: {plan:?}")?
}
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Cte(plan)) => {
+ not_impl_err!("Substrait MATERIALIZED CTE is not supported: {plan:?}")?
+ }
+ LogicalPlan::MaterializedCte(MaterializedCtePlan::Scan(plan)) => {
+ not_impl_err!("Substrait MATERIALIZED CTE is not supported: {plan:?}")?
+ }
}
}
diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md
index 7b6bc6a99e..e927d3ec4a 100644
--- a/docs/source/library-user-guide/upgrading/56.0.0.md
+++ b/docs/source/library-user-guide/upgrading/56.0.0.md
@@ -1233,6 +1233,20 @@ A registered plan needs no `PhysicalExtensionCodec` at all, and two crates' plan
See [issue #24625](https://github.com/apache/datafusion/issues/24625) for details.
+### `LogicalPlan` has the new variant `MaterializedCte`
+
+`WITH x AS MATERIALIZED (...)` is planned with the new variant `LogicalPlan::MaterializedCte(MaterializedCtePlan)`. `MaterializedCtePlan` has two cases:
+
+- `MaterializedCtePlan::Cte` has two inputs: the body of the CTE, which is computed once, and the continuation (the query that declared the CTE). Its output is the output of the continuation.
+- `MaterializedCtePlan::Scan` is a leaf. It reads the buffered output of the `MaterializedCtePlan::Cte` that has the same id.
+
+#### Who is affected
+
+- Code that has an exhaustive `match` on `LogicalPlan` must add an arm for the new variant.
+- A custom optimizer rule that moves a filter, limit or projection below a node must not move it into the body of a `MaterializedCte`, because all the scans of the CTE share the body. The continuation has no such limit.
+
+`MaterializedCteScan::find_in` returns the scans of one CTE in a plan, also in subquery expressions.
+
### Extension `PhysicalExpr`s can decode by name instead of through a codec
`PhysicalExtensionExprNode` has a new optional `expr_name` field. An extension expression that sets it is decoded by a decoder registered on the session, not by a `PhysicalExtensionCodec`. This is the same mechanism extension `ExecutionPlan`s use (see above), and it uses the same `ProtoDecoderRegistry`.