Skip to content

Apache DataFusion SQL Query Engine

feat: report statistics of materialized CTE scans (#26271)

Adrian Garcia Badaraccod48480e227f2
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index d1e5054cc8..5f2a2ca8ad 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -72,8 +72,8 @@ use datafusion_common::display::ToStringifiedPlan;
use datafusion_common::format::ExplainAnalyzeCategories;
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::{
- DFSchema, DFSchemaRef, ScalarValue, exec_err, internal_datafusion_err, internal_err,
- not_impl_err, plan_err,
+ DFSchema, DFSchemaRef, ScalarValue, Statistics, exec_err, internal_datafusion_err,
+ internal_err, not_impl_err, plan_err,
};
use datafusion_common::{
TableReference, assert_eq_or_internal_err, assert_or_internal_err,
@@ -112,7 +112,7 @@ use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::recursive_query::RecursiveQueryExec;
use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};
-use datafusion_physical_plan::statistics::StatisticsContext;
+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};
use datafusion_physical_plan::unnest::ListUnnest;
use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};
@@ -239,8 +239,6 @@ impl DefaultPhysicalPlanner {
let plan = self
.create_initial_plan(logical_plan, session_state)
.await?;
- let plan = bind_materialized_cte_scans(plan)?;
-
self.optimize_physical_plan(plan, session_state, |_, _| {})
}
}
@@ -318,56 +316,126 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>
.map(|t| Some(t.data))
}
-/// Point every [`MaterializedCteScanExec`] at the buffer of the
-/// [`MaterializedCteExec`] with the same id.
+/// Points every [`MaterializedCteScanExec`] at the buffer of the
+/// [`MaterializedCteExec`] with the same id, and gives it the statistics of
+/// the CTE body.
///
/// [`renumber_materialized_ctes`] makes the ids unique before the physical
/// plan is created.
///
/// A scan that is already bound was planned and bound by a separate planning
-/// run, such as the `scan` of a [`ViewTable`], and is left as is. Binding it
-/// again would count its partitions twice in the buffer.
+/// run, such as the `scan` of a [`ViewTable`], and is not bound again. Binding
+/// it again would count its partitions twice in the buffer.
///
/// Scans are planned before the node that owns their CTE, and a scan inside a
/// scalar subquery is planned in a separate subtree, so the binding is done
-/// once on the whole initial plan.
+/// on the whole initial plan.
///
/// [`ViewTable`]: crate::datasource::ViewTable
+struct MaterializedCteBinder {
+ ctes: HashMap<u64, Arc<dyn ExecutionPlan>>,
+ /// The statistics of the CTE bodies. `None` while they are computed.
+ statistics: HashMap<u64, Option<Arc<Statistics>>>,
+ context: StatisticsContext,
+}
+
+impl MaterializedCteBinder {
+ fn new(plan: &Arc<dyn ExecutionPlan>, session_state: &dyn Session) -> Result<Self> {
+ let mut ctes = HashMap::new();
+ plan.apply(|node| {
+ if let Some(cte) = node.downcast_ref::<MaterializedCteExec>()
+ && ctes.insert(cte.buffer().id(), Arc::clone(node)).is_some()
+ {
+ return internal_err!(
+ "Two MaterializedCteExec nodes have the id {}",
+ cte.buffer().id()
+ );
+ }
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ let context = match session_state.statistics_registry() {
+ Some(registry) => StatisticsContext::new_with_registry(registry.clone()),
+ None => StatisticsContext::new(),
+ };
+ Ok(Self {
+ ctes,
+ statistics: HashMap::new(),
+ context,
+ })
+ }
+
+ fn cte(&self, id: u64) -> Option<&MaterializedCteExec> {
+ self.ctes
+ .get(&id)
+ .and_then(|cte| cte.downcast_ref::<MaterializedCteExec>())
+ }
+
+ /// Give every scan in `plan` whose CTE is in `self.ctes` the statistics
+ /// of the CTE body. If `bind` is true, also bind the scans that are not
+ /// bound.
+ fn bind(
+ &mut self,
+ plan: Arc<dyn ExecutionPlan>,
+ bind: bool,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ plan.transform_up(|node| {
+ let Some(scan) = node.downcast_ref::<MaterializedCteScanExec>() else {
+ return Ok(Transformed::no(node));
+ };
+ let Some(cte) = self.cte(scan.id()) else {
+ if scan.is_bound() {
+ return Ok(Transformed::no(node));
+ }
+ return internal_err!("MaterializedCteScanExec {} has no CTE", scan.id());
+ };
+ let bound =
+ (bind && !scan.is_bound()).then(|| scan.bind(Arc::clone(cte.buffer())));
+ let scan = bound.as_ref().unwrap_or(scan);
+ let scan = match self.body_statistics(scan.id())? {
+ Some(statistics) => scan.with_body_statistics(statistics),
+ None => match bound {
+ Some(bound) => bound,
+ None => return Ok(Transformed::no(node)),
+ },
+ };
+ Ok(Transformed::yes(Arc::new(scan) as Arc<dyn ExecutionPlan>))
+ })
+ .map(|t| t.data)
+ }
+
+ /// The statistics of the body of the CTE `id`, or `None` while they are
+ /// computed.
+ ///
+ /// The body can read other CTEs. Its scans get their statistics first,
+ /// in a copy of the body that is used only here.
+ fn body_statistics(&mut self, id: u64) -> Result<Option<Arc<Statistics>>> {
+ if let Some(statistics) = self.statistics.get(&id) {
+ return Ok(statistics.clone());
+ }
+ let Some(cte) = self.cte(id) else {
+ return Ok(None);
+ };
+ let body = Arc::clone(cte.body());
+ self.statistics.insert(id, None);
+ let body = self.bind(body, false)?;
+ let statistics = self
+ .context
+ .compute(body.as_ref(), &StatisticsArgs::new())?;
+ self.statistics.insert(id, Some(Arc::clone(&statistics)));
+ Ok(Some(statistics))
+ }
+}
+
+/// See [`MaterializedCteBinder`].
fn bind_materialized_cte_scans(
plan: Arc<dyn ExecutionPlan>,
+ session_state: &dyn Session,
) -> Result<Arc<dyn ExecutionPlan>> {
- let mut buffers = HashMap::new();
- plan.apply(|node| {
- if let Some(cte) = node.downcast_ref::<MaterializedCteExec>()
- && buffers
- .insert(cte.buffer().id(), Arc::clone(cte.buffer()))
- .is_some()
- {
- return internal_err!(
- "Two MaterializedCteExec nodes have the id {}",
- cte.buffer().id()
- );
- }
- Ok(TreeNodeRecursion::Continue)
- })?;
- if buffers.is_empty() {
+ let mut binder = MaterializedCteBinder::new(&plan, session_state)?;
+ if binder.ctes.is_empty() {
return Ok(plan);
}
- plan.transform_up(|node| {
- let Some(scan) = node.downcast_ref::<MaterializedCteScanExec>() else {
- return Ok(Transformed::no(node));
- };
- if scan.is_bound() {
- return Ok(Transformed::no(node));
- }
- let Some(buffer) = buffers.get(&scan.id()) else {
- return internal_err!("MaterializedCteScanExec {} has no CTE", scan.id());
- };
- Ok(Transformed::yes(
- Arc::new(scan.bind(Arc::clone(buffer))) as Arc<dyn ExecutionPlan>
- ))
- })
- .map(|t| t.data)
+ binder.bind(plan, true)
}
#[derive(Debug)]
@@ -3301,7 +3369,15 @@ impl DefaultPhysicalPlanner {
/// Optimize a physical plan by applying each physical optimizer,
/// calling observer(plan, optimizer after each one)
- #[expect(clippy::needless_pass_by_value)]
+ ///
+ /// Before the optimizers run, each [`MaterializedCteScanExec`] is bound
+ /// to the buffer of its [`MaterializedCteExec`] and gets the statistics
+ /// of the CTE body, as it is before the optimizers run. A rewrite of the
+ /// body does not change its rows, so these statistics stay valid, but
+ /// they can be less precise than the statistics of the optimized body.
+ ///
+ /// TODO: snapshot the statistics after the body is optimized, see
+ /// <https://github.com/apache/datafusion/issues/26424>.
pub fn optimize_physical_plan<F>(
&self,
plan: Arc<dyn ExecutionPlan>,
@@ -3311,6 +3387,7 @@ impl DefaultPhysicalPlanner {
where
F: FnMut(&dyn ExecutionPlan, &dyn PhysicalOptimizerRule),
{
+ let plan = bind_materialized_cte_scans(plan, session_state)?;
let optimizers = session_state.physical_optimizers();
debug!(
"Input physical plan:\n{}\n",
diff --git a/datafusion/physical-plan/src/materialized_cte.rs b/datafusion/physical-plan/src/materialized_cte.rs
index 194cd6825b..de23309f99 100644
--- a/datafusion/physical-plan/src/materialized_cte.rs
+++ b/datafusion/physical-plan/src/materialized_cte.rs
@@ -39,6 +39,10 @@
//! files until it can. An in-memory batch that is larger than the batches on
//! disk is written in pieces, so that the move does not raise the memory that
//! each reader needs.
+//!
+//! A scan is a leaf, and the body is not its child, so a scan cannot compute
+//! the statistics of the body. The planner copies them to the scan with
+//! [`MaterializedCteScanExec::with_body_statistics`].
use std::fmt;
use std::pin::Pin;
@@ -513,6 +517,11 @@ impl MaterializedCteExec {
pub fn buffer(&self) -> &Arc<MaterializedCteBuffer> {
&self.buffer
}
+
+ /// The plan of the CTE body.
+ pub fn body(&self) -> &Arc<dyn ExecutionPlan> {
+ &self.body
+ }
}
impl DisplayAs for MaterializedCteExec {
@@ -663,6 +672,13 @@ pub struct MaterializedCteScanExec {
id: u64,
name: String,
schema: SchemaRef,
+ /// The statistics of all partitions of the body.
+ ///
+ /// These are valid only because the scan always returns every row of
+ /// the body. If an operator that changes the rows (for example a limit
+ /// or a filter) is ever pushed down into this scan, these statistics
+ /// must be updated to match.
+ statistics: Option<Arc<Statistics>>,
buffer: Option<Arc<MaterializedCteBuffer>>,
metrics: ExecutionPlanMetricsSet,
cache: Arc<PlanProperties>,
@@ -687,6 +703,7 @@ impl MaterializedCteScanExec {
id,
name: name.into(),
schema,
+ statistics: None,
buffer: None,
metrics: ExecutionPlanMetricsSet::new(),
cache: Arc::new(cache),
@@ -703,12 +720,30 @@ impl MaterializedCteScanExec {
id: self.id,
name: self.name.clone(),
schema: Arc::clone(&self.schema),
+ statistics: self.statistics.clone(),
buffer: Some(buffer),
metrics: ExecutionPlanMetricsSet::new(),
cache: Arc::clone(&self.cache),
}
}
+ /// Copy this scan, with the statistics of all partitions of the CTE
+ /// body.
+ ///
+ /// The scan returns every row of the body, so the statistics stay
+ /// correct when the body is rewritten to a plan that gives the same rows.
+ pub fn with_body_statistics(&self, statistics: Arc<Statistics>) -> Self {
+ Self {
+ id: self.id,
+ name: self.name.clone(),
+ schema: Arc::clone(&self.schema),
+ statistics: Some(statistics),
+ buffer: self.buffer.clone(),
+ metrics: ExecutionPlanMetricsSet::new(),
+ cache: Arc::clone(&self.cache),
+ }
+ }
+
/// The id of the [`MaterializedCteBuffer`] this scan reads.
pub fn id(&self) -> u64 {
self.id
@@ -801,19 +836,28 @@ impl ExecutionPlan for MaterializedCteScanExec {
Some(self.metrics.clone_inner())
}
- /// The statistics are unknown: a leaf cannot ask the body for its
- /// statistics through [`StatisticsArgs`].
+ /// Returns the statistics that the planner copied from the CTE body.
+ /// See [`Self::with_body_statistics`].
///
- /// This is a known limitation, and it can make a plan worse than the
- /// plan of the inlined CTE. For example, `JoinSelection` does not know
- /// that a scan is small, and can put the other, larger input of a hash
- /// join on the build side.
+ /// The statistics of one partition are known only when the scan has one
+ /// partition.
+ ///
+ /// TODO: support multiple partitions, see
+ /// <https://github.com/apache/datafusion/issues/26424>.
fn statistics_from_inputs(
&self,
_input_stats: &[Arc<Statistics>],
- _args: &StatisticsArgs,
+ args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
- Ok(Arc::new(Statistics::new_unknown(&self.schema)))
+ match &self.statistics {
+ Some(statistics)
+ if args.partition().is_none()
+ || self.cache.partitioning.partition_count() == 1 =>
+ {
+ Ok(Arc::clone(statistics))
+ }
+ _ => Ok(Arc::new(Statistics::new_unknown(&self.schema))),
+ }
}
}
@@ -823,12 +867,89 @@ mod tests {
use crate::common::collect;
use crate::empty::EmptyExec;
use crate::filter_pushdown::PushedDown;
+ use crate::statistics::StatisticsContext;
use crate::test::TestMemoryExec;
use arrow::array::{Array, Int32Array};
use arrow::datatypes::{DataType, Field, Schema};
+ use datafusion_common::stats::Precision;
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
use datafusion_physical_expr::expressions::{col, lit};
+ #[test]
+ fn scan_reports_body_statistics() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let batch = RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],
+ )?;
+ let body = TestMemoryExec::try_new_exec(
+ &[vec![batch.clone()], vec![batch]],
+ Arc::clone(&schema),
+ None,
+ )?;
+ let context = StatisticsContext::new();
+ let body_statistics = context.compute(body.as_ref(), &StatisticsArgs::new())?;
+ assert_eq!(body_statistics.num_rows, Precision::Exact(6));
+
+ let buffer = Arc::new(MaterializedCteBuffer::new(0, "c"));
+ let scan = MaterializedCteScanExec::new(0, "c", schema, 4).bind(buffer);
+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;
+ assert_eq!(statistics.num_rows, Precision::Absent);
+
+ let scan = scan.with_body_statistics(body_statistics);
+ assert!(scan.is_bound());
+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;
+ assert_eq!(statistics.num_rows, Precision::Exact(6));
+ // The 4 scan partitions do not map to the 2 body partitions.
+ let args = StatisticsArgs::new().with_partition(Some(0));
+ let statistics = context.compute(&scan, &args)?;
+ assert_eq!(statistics.num_rows, Precision::Absent);
+ Ok(())
+ }
+
+ #[test]
+ fn scan_reports_body_column_statistics() -> Result<()> {
+ use crate::test::exec::StatisticsExec;
+ use datafusion_common::{ColumnStatistics, ScalarValue};
+
+ let schema = Schema::new(vec![
+ Field::new("a", DataType::Int32, true),
+ Field::new("b", DataType::Int32, false),
+ ]);
+ let column = |min: i32, max: i32, nulls: usize, distinct: usize| {
+ ColumnStatistics::new_unknown()
+ .with_min_value(Precision::Exact(ScalarValue::Int32(Some(min))))
+ .with_max_value(Precision::Exact(ScalarValue::Int32(Some(max))))
+ .with_null_count(Precision::Exact(nulls))
+ .with_distinct_count(Precision::Inexact(distinct))
+ .with_byte_size(Precision::Exact(400))
+ };
+ let body_statistics = Statistics {
+ num_rows: Precision::Exact(100),
+ total_byte_size: Precision::Exact(800),
+ column_statistics: vec![column(1, 50, 7, 40), column(-3, 3, 0, 7)],
+ };
+ let body = StatisticsExec::new(body_statistics.clone(), schema.clone());
+ let context = StatisticsContext::new();
+ let computed = context.compute(&body, &StatisticsArgs::new())?;
+ assert_eq!(*computed, body_statistics);
+
+ let buffer = Arc::new(MaterializedCteBuffer::new(0, "c"));
+ let scan = MaterializedCteScanExec::new(0, "c", Arc::new(schema.clone()), 1)
+ .bind(buffer)
+ .with_body_statistics(computed);
+ // Every column keeps its min, max, null count, distinct count and
+ // byte size, in the order of the scan schema.
+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;
+ assert_eq!(*statistics, body_statistics);
+ // A scan with one partition reads all rows of the body in that
+ // partition, so the statistics of partition 0 are the same.
+ let args = StatisticsArgs::new().with_partition(Some(0));
+ let statistics = context.compute(&scan, &args)?;
+ assert_eq!(*statistics, body_statistics);
+ Ok(())
+ }
+
#[test]
fn parent_filters_go_to_the_continuation_only() -> Result<()> {
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
diff --git a/datafusion/sqllogictest/test_files/cte_materialized.slt b/datafusion/sqllogictest/test_files/cte_materialized.slt
index cae4a397e8..f7e2bd10e3 100644
--- a/datafusion/sqllogictest/test_files/cte_materialized.slt
+++ b/datafusion/sqllogictest/test_files/cte_materialized.slt
@@ -93,24 +93,23 @@ SELECT k FROM c WHERE v2 < 30
# referenced three times, and each scan replays all 4 rows.
query TT
EXPLAIN ANALYZE WITH c AS MATERIALIZED (SELECT k, v FROM t)
-SELECT count(*) FROM (
+SELECT sum(k) FROM (
SELECT k FROM c UNION ALL SELECT k FROM c UNION ALL SELECT k FROM c
)
----
Plan with Metrics
01)MaterializedCteExec: name=c, metrics=[spill_count=0, <slt:ignore>buffered_rows=4]
02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[<slt:ignore>]
-03)--ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]
-04)----AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-05)------CoalescePartitionsExec, metrics=[<slt:ignore>]
-06)--------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]
-07)----------UnionExec, metrics=[<slt:ignore>]
-08)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]
-09)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
-10)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]
-11)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
-12)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]
-13)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
+03)--AggregateExec: mode=Final, gby=[], aggr=[sum(k)], metrics=[<slt:ignore>]
+04)----CoalescePartitionsExec, metrics=[<slt:ignore>]
+05)------AggregateExec: mode=Partial, gby=[], aggr=[sum(k)], metrics=[<slt:ignore>]
+06)--------UnionExec, metrics=[<slt:ignore>]
+07)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]
+08)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
+09)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]
+10)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
+11)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]
+12)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]
# For comparison, the inlined CTE scans the source once per reference.
query TT
@@ -402,10 +401,12 @@ logical_plan
physical_plan
01)MaterializedCteExec: name=r
02)--DataSourceExec: partitions=1, partition_sizes=[1]
-03)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[k@0, v@1]
-04)----FilterExec: v@1 = 10
-05)------DataSourceExec: partitions=1, partition_sizes=[1]
-06)----MaterializedCteScanExec: name=r
+03)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[k@1, v@2]
+04)----CoalescePartitionsExec
+05)------MaterializedCteScanExec: name=r
+06)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1
+07)------FilterExec: v@1 = 10
+08)--------DataSourceExec: partitions=1, partition_sizes=[1]
query II
SELECT * FROM (
@@ -469,9 +470,9 @@ SELECT count(*) FROM (SELECT s FROM c UNION ALL SELECT s FROM c)
statement ok
RESET datafusion.runtime.memory_limit
-# Known limitation: a reference to a materialized CTE has unknown statistics.
-# The join below builds the hash table on the 100000 rows of `huge`. When the
-# CTE is inlined, the hash table is built on the 5 rows of `small`.
+# A reference to a materialized CTE has the statistics of the CTE body. The
+# join below builds the hash table on the 5 rows of the CTE, not on the 100000
+# rows of `huge`, as it does when the CTE is inlined.
statement ok
CREATE TABLE small AS SELECT value AS k FROM generate_series(1, 5)
@@ -498,8 +499,8 @@ physical_plan
03)--ProjectionExec: expr=[count(Int64(1))@0 as count(*)]
04)----AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))]
05)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[]
-06)--------DataSourceExec: partitions=1, partition_sizes=[13]
-07)--------MaterializedCteScanExec: name=c
+06)--------MaterializedCteScanExec: name=c
+07)--------DataSourceExec: partitions=1, partition_sizes=[13]
query TT
EXPLAIN WITH c AS NOT MATERIALIZED (SELECT k FROM small)
@@ -520,12 +521,135 @@ physical_plan
04)------DataSourceExec: partitions=1, partition_sizes=[1]
05)------DataSourceExec: partitions=1, partition_sizes=[13]
+query I
+WITH c AS MATERIALIZED (SELECT k FROM small)
+SELECT count(*) FROM huge JOIN c ON huge.k = c.k
+----
+5
+
+# The statistics show in the plan. The scan of `c` has the row count of the
+# body.
+statement ok
+SET datafusion.explain.show_statistics = true
+
+statement ok
+SET datafusion.explain.physical_plan_only = true
+
+query TT
+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small) SELECT k + 1 FROM c
+----
+physical_plan
+01)MaterializedCteExec: name=c, statistics=[Rows=Exact(5), Bytes=Exact(40), [(Col[0]:)]]
+02)--DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]
+03)--ProjectionExec: expr=[k@0 + 1 as c.k + Int64(1)], statistics=[Rows=Exact(5), Bytes=Exact(40), [(Col[0]:)]]
+04)----MaterializedCteScanExec: name=c, statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]
+
+# The scan also has the column statistics of the body. The filter of the body
+# gives `k` a max of 3. The outer filter uses this max to bound `k` to [3, 3],
+# the same bounds as the inlined CTE.
+query TT
+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small WHERE k <= 3)
+SELECT k FROM c WHERE k > 2
+----
+physical_plan
+01)MaterializedCteExec: name=c, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]
+02)--FilterExec: k@0 <= 3, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Max=Inexact(Int64(3)) Null=Exact(0))]]
+03)----DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]
+04)--FilterExec: k@0 > 2, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]
+05)----MaterializedCteScanExec: name=c, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Max=Inexact(Int64(3)) Null=Exact(0))]]
+
+query TT
+EXPLAIN WITH c AS NOT MATERIALIZED (SELECT k FROM small WHERE k <= 3)
+SELECT k FROM c WHERE k > 2
+----
+physical_plan
+01)FilterExec: k@0 <= 3 AND k@0 > 2, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]
+02)--DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]
+
+statement ok
+RESET datafusion.explain.show_statistics
+
+# An exact row count of the body answers count(*), and the body does not run.
+query TT
+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small) SELECT count(*) FROM c
+----
+physical_plan
+01)MaterializedCteExec: name=c
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--ProjectionExec: expr=[5 as count(*)]
+04)----PlaceholderRowExec
+
+query I
+WITH c AS MATERIALIZED (SELECT k FROM small) SELECT count(*) FROM c
+----
+5
+
+# A body that reads another CTE gets the statistics of that CTE first.
+query TT
+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small),
+ d AS MATERIALIZED (SELECT k FROM c)
+SELECT count(*) FROM d
+----
+physical_plan
+01)MaterializedCteExec: name=c
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--MaterializedCteExec: name=d
+04)----MaterializedCteScanExec: name=c
+05)----ProjectionExec: expr=[5 as count(*)]
+06)------PlaceholderRowExec
+
statement ok
DROP TABLE small
statement ok
DROP TABLE huge
+statement ok
+SET datafusion.execution.target_partitions = 4
+
+statement ok
+CREATE TABLE p AS SELECT value AS k, value % 10 AS g FROM generate_series(1, 1000)
+
+# A reference inside a scalar subquery is bound to the CTE.
+query II
+WITH c AS MATERIALIZED (SELECT g, count(*) AS n FROM p GROUP BY g)
+SELECT (SELECT count(*) FROM (SELECT g FROM c GROUP BY g)), max(n) FROM c
+----
+10 100
+
+# A view is planned on its own, and its plan is then optimized again as a part
+# of the outer plan.
+statement ok
+CREATE VIEW pv AS
+WITH c AS MATERIALIZED (SELECT g, count(*) AS n FROM p GROUP BY g)
+SELECT g, sum(n) AS s FROM c GROUP BY g
+
+query II rowsort
+SELECT a.g, a.s + b.s FROM pv a JOIN pv b ON a.g = b.g
+----
+0 200
+1 200
+2 200
+3 200
+4 200
+5 200
+6 200
+7 200
+8 200
+9 200
+
+statement ok
+DROP VIEW pv
+
+statement ok
+RESET datafusion.explain.physical_plan_only
+
+statement ok
+RESET datafusion.execution.target_partitions
+
+statement ok
+DROP TABLE p
+
# Setting a runtime option also sets this option to false.
statement ok
RESET datafusion.catalog.create_default_catalog_and_schema