Apache DataFusion SQL Query Engine
feat: report statistics of materialized CTE scans (#26271)
Adrian Garcia Badaraccod48480e227f2
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex d1e5054cc8..5f2a2ca8ad 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -72,8 +72,8 @@ use datafusion_common::display::ToStringifiedPlan;use datafusion_common::format::ExplainAnalyzeCategories;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};use datafusion_common::{- DFSchema, DFSchemaRef, ScalarValue, exec_err, internal_datafusion_err, internal_err,- not_impl_err, plan_err,+ DFSchema, DFSchemaRef, ScalarValue, Statistics, exec_err, internal_datafusion_err,+ internal_err, not_impl_err, plan_err,};use datafusion_common::{TableReference, assert_eq_or_internal_err, assert_or_internal_err,@@ -112,7 +112,7 @@ use datafusion_physical_plan::operator_statistics::StatisticsRegistry;use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::recursive_query::RecursiveQueryExec;use datafusion_physical_plan::scalar_subquery::{ScalarSubqueryExec, ScalarSubqueryLink};-use datafusion_physical_plan::statistics::StatisticsContext;+use datafusion_physical_plan::statistics::{StatisticsArgs, StatisticsContext};use datafusion_physical_plan::unnest::ListUnnest;use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};@@ -239,8 +239,6 @@ impl DefaultPhysicalPlanner {let plan = self.create_initial_plan(logical_plan, session_state).await?;- let plan = bind_materialized_cte_scans(plan)?;-self.optimize_physical_plan(plan, session_state, |_, _| {})}}@@ -318,56 +316,126 @@ fn renumber_materialized_ctes(plan: &LogicalPlan) -> Result<Option<LogicalPlan>>.map(|t| Some(t.data))}-/// Point every [`MaterializedCteScanExec`] at the buffer of the-/// [`MaterializedCteExec`] with the same id.+/// Points every [`MaterializedCteScanExec`] at the buffer of the+/// [`MaterializedCteExec`] with the same id, and gives it the statistics of+/// the CTE body.////// [`renumber_materialized_ctes`] makes the ids unique before the physical/// plan is created.////// A scan that is already bound was planned and bound by a separate planning-/// run, such as the `scan` of a [`ViewTable`], and is left as is. Binding it-/// again would count its partitions twice in the buffer.+/// run, such as the `scan` of a [`ViewTable`], and is not bound again. Binding+/// it again would count its partitions twice in the buffer.////// Scans are planned before the node that owns their CTE, and a scan inside a/// scalar subquery is planned in a separate subtree, so the binding is done-/// once on the whole initial plan.+/// on the whole initial plan.////// [`ViewTable`]: crate::datasource::ViewTable+struct MaterializedCteBinder {+ ctes: HashMap<u64, Arc<dyn ExecutionPlan>>,+ /// The statistics of the CTE bodies. `None` while they are computed.+ statistics: HashMap<u64, Option<Arc<Statistics>>>,+ context: StatisticsContext,+}++impl MaterializedCteBinder {+ fn new(plan: &Arc<dyn ExecutionPlan>, session_state: &dyn Session) -> Result<Self> {+ let mut ctes = HashMap::new();+ plan.apply(|node| {+ if let Some(cte) = node.downcast_ref::<MaterializedCteExec>()+ && ctes.insert(cte.buffer().id(), Arc::clone(node)).is_some()+ {+ return internal_err!(+ "Two MaterializedCteExec nodes have the id {}",+ cte.buffer().id()+ );+ }+ Ok(TreeNodeRecursion::Continue)+ })?;+ let context = match session_state.statistics_registry() {+ Some(registry) => StatisticsContext::new_with_registry(registry.clone()),+ None => StatisticsContext::new(),+ };+ Ok(Self {+ ctes,+ statistics: HashMap::new(),+ context,+ })+ }++ fn cte(&self, id: u64) -> Option<&MaterializedCteExec> {+ self.ctes+ .get(&id)+ .and_then(|cte| cte.downcast_ref::<MaterializedCteExec>())+ }++ /// Give every scan in `plan` whose CTE is in `self.ctes` the statistics+ /// of the CTE body. If `bind` is true, also bind the scans that are not+ /// bound.+ fn bind(+ &mut self,+ plan: Arc<dyn ExecutionPlan>,+ bind: bool,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ plan.transform_up(|node| {+ let Some(scan) = node.downcast_ref::<MaterializedCteScanExec>() else {+ return Ok(Transformed::no(node));+ };+ let Some(cte) = self.cte(scan.id()) else {+ if scan.is_bound() {+ return Ok(Transformed::no(node));+ }+ return internal_err!("MaterializedCteScanExec {} has no CTE", scan.id());+ };+ let bound =+ (bind && !scan.is_bound()).then(|| scan.bind(Arc::clone(cte.buffer())));+ let scan = bound.as_ref().unwrap_or(scan);+ let scan = match self.body_statistics(scan.id())? {+ Some(statistics) => scan.with_body_statistics(statistics),+ None => match bound {+ Some(bound) => bound,+ None => return Ok(Transformed::no(node)),+ },+ };+ Ok(Transformed::yes(Arc::new(scan) as Arc<dyn ExecutionPlan>))+ })+ .map(|t| t.data)+ }++ /// The statistics of the body of the CTE `id`, or `None` while they are+ /// computed.+ ///+ /// The body can read other CTEs. Its scans get their statistics first,+ /// in a copy of the body that is used only here.+ fn body_statistics(&mut self, id: u64) -> Result<Option<Arc<Statistics>>> {+ if let Some(statistics) = self.statistics.get(&id) {+ return Ok(statistics.clone());+ }+ let Some(cte) = self.cte(id) else {+ return Ok(None);+ };+ let body = Arc::clone(cte.body());+ self.statistics.insert(id, None);+ let body = self.bind(body, false)?;+ let statistics = self+ .context+ .compute(body.as_ref(), &StatisticsArgs::new())?;+ self.statistics.insert(id, Some(Arc::clone(&statistics)));+ Ok(Some(statistics))+ }+}++/// See [`MaterializedCteBinder`].fn bind_materialized_cte_scans(plan: Arc<dyn ExecutionPlan>,+ session_state: &dyn Session,) -> Result<Arc<dyn ExecutionPlan>> {- let mut buffers = HashMap::new();- plan.apply(|node| {- if let Some(cte) = node.downcast_ref::<MaterializedCteExec>()- && buffers- .insert(cte.buffer().id(), Arc::clone(cte.buffer()))- .is_some()- {- return internal_err!(- "Two MaterializedCteExec nodes have the id {}",- cte.buffer().id()- );- }- Ok(TreeNodeRecursion::Continue)- })?;- if buffers.is_empty() {+ let mut binder = MaterializedCteBinder::new(&plan, session_state)?;+ if binder.ctes.is_empty() {return Ok(plan);}- plan.transform_up(|node| {- let Some(scan) = node.downcast_ref::<MaterializedCteScanExec>() else {- return Ok(Transformed::no(node));- };- if scan.is_bound() {- return Ok(Transformed::no(node));- }- let Some(buffer) = buffers.get(&scan.id()) else {- return internal_err!("MaterializedCteScanExec {} has no CTE", scan.id());- };- Ok(Transformed::yes(- Arc::new(scan.bind(Arc::clone(buffer))) as Arc<dyn ExecutionPlan>- ))- })- .map(|t| t.data)+ binder.bind(plan, true)}#[derive(Debug)]@@ -3301,7 +3369,15 @@ impl DefaultPhysicalPlanner {/// Optimize a physical plan by applying each physical optimizer,/// calling observer(plan, optimizer after each one)- #[expect(clippy::needless_pass_by_value)]+ ///+ /// Before the optimizers run, each [`MaterializedCteScanExec`] is bound+ /// to the buffer of its [`MaterializedCteExec`] and gets the statistics+ /// of the CTE body, as it is before the optimizers run. A rewrite of the+ /// body does not change its rows, so these statistics stay valid, but+ /// they can be less precise than the statistics of the optimized body.+ ///+ /// TODO: snapshot the statistics after the body is optimized, see+ /// <https://github.com/apache/datafusion/issues/26424>.pub fn optimize_physical_plan<F>(&self,plan: Arc<dyn ExecutionPlan>,@@ -3311,6 +3387,7 @@ impl DefaultPhysicalPlanner {whereF: FnMut(&dyn ExecutionPlan, &dyn PhysicalOptimizerRule),{+ let plan = bind_materialized_cte_scans(plan, session_state)?;let optimizers = session_state.physical_optimizers();debug!("Input physical plan:\n{}\n",diff --git a/datafusion/physical-plan/src/materialized_cte.rs b/datafusion/physical-plan/src/materialized_cte.rsindex 194cd6825b..de23309f99 100644--- a/datafusion/physical-plan/src/materialized_cte.rs+++ b/datafusion/physical-plan/src/materialized_cte.rs@@ -39,6 +39,10 @@//! files until it can. An in-memory batch that is larger than the batches on//! disk is written in pieces, so that the move does not raise the memory that//! each reader needs.+//!+//! A scan is a leaf, and the body is not its child, so a scan cannot compute+//! the statistics of the body. The planner copies them to the scan with+//! [`MaterializedCteScanExec::with_body_statistics`].use std::fmt;use std::pin::Pin;@@ -513,6 +517,11 @@ impl MaterializedCteExec {pub fn buffer(&self) -> &Arc<MaterializedCteBuffer> {&self.buffer}++ /// The plan of the CTE body.+ pub fn body(&self) -> &Arc<dyn ExecutionPlan> {+ &self.body+ }}impl DisplayAs for MaterializedCteExec {@@ -663,6 +672,13 @@ pub struct MaterializedCteScanExec {id: u64,name: String,schema: SchemaRef,+ /// The statistics of all partitions of the body.+ ///+ /// These are valid only because the scan always returns every row of+ /// the body. If an operator that changes the rows (for example a limit+ /// or a filter) is ever pushed down into this scan, these statistics+ /// must be updated to match.+ statistics: Option<Arc<Statistics>>,buffer: Option<Arc<MaterializedCteBuffer>>,metrics: ExecutionPlanMetricsSet,cache: Arc<PlanProperties>,@@ -687,6 +703,7 @@ impl MaterializedCteScanExec {id,name: name.into(),schema,+ statistics: None,buffer: None,metrics: ExecutionPlanMetricsSet::new(),cache: Arc::new(cache),@@ -703,12 +720,30 @@ impl MaterializedCteScanExec {id: self.id,name: self.name.clone(),schema: Arc::clone(&self.schema),+ statistics: self.statistics.clone(),buffer: Some(buffer),metrics: ExecutionPlanMetricsSet::new(),cache: Arc::clone(&self.cache),}}+ /// Copy this scan, with the statistics of all partitions of the CTE+ /// body.+ ///+ /// The scan returns every row of the body, so the statistics stay+ /// correct when the body is rewritten to a plan that gives the same rows.+ pub fn with_body_statistics(&self, statistics: Arc<Statistics>) -> Self {+ Self {+ id: self.id,+ name: self.name.clone(),+ schema: Arc::clone(&self.schema),+ statistics: Some(statistics),+ buffer: self.buffer.clone(),+ metrics: ExecutionPlanMetricsSet::new(),+ cache: Arc::clone(&self.cache),+ }+ }+/// The id of the [`MaterializedCteBuffer`] this scan reads.pub fn id(&self) -> u64 {self.id@@ -801,19 +836,28 @@ impl ExecutionPlan for MaterializedCteScanExec {Some(self.metrics.clone_inner())}- /// The statistics are unknown: a leaf cannot ask the body for its- /// statistics through [`StatisticsArgs`].+ /// Returns the statistics that the planner copied from the CTE body.+ /// See [`Self::with_body_statistics`].///- /// This is a known limitation, and it can make a plan worse than the- /// plan of the inlined CTE. For example, `JoinSelection` does not know- /// that a scan is small, and can put the other, larger input of a hash- /// join on the build side.+ /// The statistics of one partition are known only when the scan has one+ /// partition.+ ///+ /// TODO: support multiple partitions, see+ /// <https://github.com/apache/datafusion/issues/26424>.fn statistics_from_inputs(&self,_input_stats: &[Arc<Statistics>],- _args: &StatisticsArgs,+ args: &StatisticsArgs,) -> Result<Arc<Statistics>> {- Ok(Arc::new(Statistics::new_unknown(&self.schema)))+ match &self.statistics {+ Some(statistics)+ if args.partition().is_none()+ || self.cache.partitioning.partition_count() == 1 =>+ {+ Ok(Arc::clone(statistics))+ }+ _ => Ok(Arc::new(Statistics::new_unknown(&self.schema))),+ }}}@@ -823,12 +867,89 @@ mod tests {use crate::common::collect;use crate::empty::EmptyExec;use crate::filter_pushdown::PushedDown;+ use crate::statistics::StatisticsContext;use crate::test::TestMemoryExec;use arrow::array::{Array, Int32Array};use arrow::datatypes::{DataType, Field, Schema};+ use datafusion_common::stats::Precision;use datafusion_execution::runtime_env::RuntimeEnvBuilder;use datafusion_physical_expr::expressions::{col, lit};+ #[test]+ fn scan_reports_body_statistics() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let batch = RecordBatch::try_new(+ Arc::clone(&schema),+ vec![Arc::new(Int32Array::from(vec![3, 1, 2]))],+ )?;+ let body = TestMemoryExec::try_new_exec(+ &[vec![batch.clone()], vec![batch]],+ Arc::clone(&schema),+ None,+ )?;+ let context = StatisticsContext::new();+ let body_statistics = context.compute(body.as_ref(), &StatisticsArgs::new())?;+ assert_eq!(body_statistics.num_rows, Precision::Exact(6));++ let buffer = Arc::new(MaterializedCteBuffer::new(0, "c"));+ let scan = MaterializedCteScanExec::new(0, "c", schema, 4).bind(buffer);+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;+ assert_eq!(statistics.num_rows, Precision::Absent);++ let scan = scan.with_body_statistics(body_statistics);+ assert!(scan.is_bound());+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;+ assert_eq!(statistics.num_rows, Precision::Exact(6));+ // The 4 scan partitions do not map to the 2 body partitions.+ let args = StatisticsArgs::new().with_partition(Some(0));+ let statistics = context.compute(&scan, &args)?;+ assert_eq!(statistics.num_rows, Precision::Absent);+ Ok(())+ }++ #[test]+ fn scan_reports_body_column_statistics() -> Result<()> {+ use crate::test::exec::StatisticsExec;+ use datafusion_common::{ColumnStatistics, ScalarValue};++ let schema = Schema::new(vec![+ Field::new("a", DataType::Int32, true),+ Field::new("b", DataType::Int32, false),+ ]);+ let column = |min: i32, max: i32, nulls: usize, distinct: usize| {+ ColumnStatistics::new_unknown()+ .with_min_value(Precision::Exact(ScalarValue::Int32(Some(min))))+ .with_max_value(Precision::Exact(ScalarValue::Int32(Some(max))))+ .with_null_count(Precision::Exact(nulls))+ .with_distinct_count(Precision::Inexact(distinct))+ .with_byte_size(Precision::Exact(400))+ };+ let body_statistics = Statistics {+ num_rows: Precision::Exact(100),+ total_byte_size: Precision::Exact(800),+ column_statistics: vec![column(1, 50, 7, 40), column(-3, 3, 0, 7)],+ };+ let body = StatisticsExec::new(body_statistics.clone(), schema.clone());+ let context = StatisticsContext::new();+ let computed = context.compute(&body, &StatisticsArgs::new())?;+ assert_eq!(*computed, body_statistics);++ let buffer = Arc::new(MaterializedCteBuffer::new(0, "c"));+ let scan = MaterializedCteScanExec::new(0, "c", Arc::new(schema.clone()), 1)+ .bind(buffer)+ .with_body_statistics(computed);+ // Every column keeps its min, max, null count, distinct count and+ // byte size, in the order of the scan schema.+ let statistics = context.compute(&scan, &StatisticsArgs::new())?;+ assert_eq!(*statistics, body_statistics);+ // A scan with one partition reads all rows of the body in that+ // partition, so the statistics of partition 0 are the same.+ let args = StatisticsArgs::new().with_partition(Some(0));+ let statistics = context.compute(&scan, &args)?;+ assert_eq!(*statistics, body_statistics);+ Ok(())+ }+#[test]fn parent_filters_go_to_the_continuation_only() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));diff --git a/datafusion/sqllogictest/test_files/cte_materialized.slt b/datafusion/sqllogictest/test_files/cte_materialized.sltindex cae4a397e8..f7e2bd10e3 100644--- a/datafusion/sqllogictest/test_files/cte_materialized.slt+++ b/datafusion/sqllogictest/test_files/cte_materialized.slt@@ -93,24 +93,23 @@ SELECT k FROM c WHERE v2 < 30# referenced three times, and each scan replays all 4 rows.query TTEXPLAIN ANALYZE WITH c AS MATERIALIZED (SELECT k, v FROM t)-SELECT count(*) FROM (+SELECT sum(k) FROM (SELECT k FROM c UNION ALL SELECT k FROM c UNION ALL SELECT k FROM c)----Plan with Metrics01)MaterializedCteExec: name=c, metrics=[spill_count=0, <slt:ignore>buffered_rows=4]02)--DataSourceExec: partitions=1, partition_sizes=[1], metrics=[<slt:ignore>]-03)--ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[<slt:ignore>]-04)----AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-05)------CoalescePartitionsExec, metrics=[<slt:ignore>]-06)--------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))], metrics=[<slt:ignore>]-07)----------UnionExec, metrics=[<slt:ignore>]-08)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]-09)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]-10)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]-11)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]-12)------------ProjectionExec: expr=[], metrics=[<slt:ignore>]-13)--------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]+03)--AggregateExec: mode=Final, gby=[], aggr=[sum(k)], metrics=[<slt:ignore>]+04)----CoalescePartitionsExec, metrics=[<slt:ignore>]+05)------AggregateExec: mode=Partial, gby=[], aggr=[sum(k)], metrics=[<slt:ignore>]+06)--------UnionExec, metrics=[<slt:ignore>]+07)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]+08)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]+09)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]+10)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]+11)----------ProjectionExec: expr=[k@0 as k], metrics=[<slt:ignore>]+12)------------MaterializedCteScanExec: name=c, metrics=[output_rows=4, <slt:ignore>]# For comparison, the inlined CTE scans the source once per reference.query TT@@ -402,10 +401,12 @@ logical_planphysical_plan01)MaterializedCteExec: name=r02)--DataSourceExec: partitions=1, partition_sizes=[1]-03)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[k@0, v@1]-04)----FilterExec: v@1 = 10-05)------DataSourceExec: partitions=1, partition_sizes=[1]-06)----MaterializedCteScanExec: name=r+03)--HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[k@1, v@2]+04)----CoalescePartitionsExec+05)------MaterializedCteScanExec: name=r+06)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1+07)------FilterExec: v@1 = 10+08)--------DataSourceExec: partitions=1, partition_sizes=[1]query IISELECT * FROM (@@ -469,9 +470,9 @@ SELECT count(*) FROM (SELECT s FROM c UNION ALL SELECT s FROM c)statement okRESET datafusion.runtime.memory_limit-# Known limitation: a reference to a materialized CTE has unknown statistics.-# The join below builds the hash table on the 100000 rows of `huge`. When the-# CTE is inlined, the hash table is built on the 5 rows of `small`.+# A reference to a materialized CTE has the statistics of the CTE body. The+# join below builds the hash table on the 5 rows of the CTE, not on the 100000+# rows of `huge`, as it does when the CTE is inlined.statement okCREATE TABLE small AS SELECT value AS k FROM generate_series(1, 5)@@ -498,8 +499,8 @@ physical_plan03)--ProjectionExec: expr=[count(Int64(1))@0 as count(*)]04)----AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))]05)------HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(k@0, k@0)], projection=[]-06)--------DataSourceExec: partitions=1, partition_sizes=[13]-07)--------MaterializedCteScanExec: name=c+06)--------MaterializedCteScanExec: name=c+07)--------DataSourceExec: partitions=1, partition_sizes=[13]query TTEXPLAIN WITH c AS NOT MATERIALIZED (SELECT k FROM small)@@ -520,12 +521,135 @@ physical_plan04)------DataSourceExec: partitions=1, partition_sizes=[1]05)------DataSourceExec: partitions=1, partition_sizes=[13]+query I+WITH c AS MATERIALIZED (SELECT k FROM small)+SELECT count(*) FROM huge JOIN c ON huge.k = c.k+----+5++# The statistics show in the plan. The scan of `c` has the row count of the+# body.+statement ok+SET datafusion.explain.show_statistics = true++statement ok+SET datafusion.explain.physical_plan_only = true++query TT+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small) SELECT k + 1 FROM c+----+physical_plan+01)MaterializedCteExec: name=c, statistics=[Rows=Exact(5), Bytes=Exact(40), [(Col[0]:)]]+02)--DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]+03)--ProjectionExec: expr=[k@0 + 1 as c.k + Int64(1)], statistics=[Rows=Exact(5), Bytes=Exact(40), [(Col[0]:)]]+04)----MaterializedCteScanExec: name=c, statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]++# The scan also has the column statistics of the body. The filter of the body+# gives `k` a max of 3. The outer filter uses this max to bound `k` to [3, 3],+# the same bounds as the inlined CTE.+query TT+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small WHERE k <= 3)+SELECT k FROM c WHERE k > 2+----+physical_plan+01)MaterializedCteExec: name=c, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]+02)--FilterExec: k@0 <= 3, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Max=Inexact(Int64(3)) Null=Exact(0))]]+03)----DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]+04)--FilterExec: k@0 > 2, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]+05)----MaterializedCteScanExec: name=c, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Max=Inexact(Int64(3)) Null=Exact(0))]]++query TT+EXPLAIN WITH c AS NOT MATERIALIZED (SELECT k FROM small WHERE k <= 3)+SELECT k FROM c WHERE k > 2+----+physical_plan+01)FilterExec: k@0 <= 3 AND k@0 > 2, statistics=[Rows=Inexact(5), Bytes=Inexact(65632), [(Col[0]: Min=Exact(Int64(3)) Max=Exact(Int64(3)) Null=Exact(0) Distinct=Inexact(1))]]+02)--DataSourceExec: partitions=1, partition_sizes=[1], statistics=[Rows=Exact(5), Bytes=Exact(65632), [(Col[0]: Null=Exact(0))]]++statement ok+RESET datafusion.explain.show_statistics++# An exact row count of the body answers count(*), and the body does not run.+query TT+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small) SELECT count(*) FROM c+----+physical_plan+01)MaterializedCteExec: name=c+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--ProjectionExec: expr=[5 as count(*)]+04)----PlaceholderRowExec++query I+WITH c AS MATERIALIZED (SELECT k FROM small) SELECT count(*) FROM c+----+5++# A body that reads another CTE gets the statistics of that CTE first.+query TT+EXPLAIN WITH c AS MATERIALIZED (SELECT k FROM small),+ d AS MATERIALIZED (SELECT k FROM c)+SELECT count(*) FROM d+----+physical_plan+01)MaterializedCteExec: name=c+02)--DataSourceExec: partitions=1, partition_sizes=[1]+03)--MaterializedCteExec: name=d+04)----MaterializedCteScanExec: name=c+05)----ProjectionExec: expr=[5 as count(*)]+06)------PlaceholderRowExec+statement okDROP TABLE smallstatement okDROP TABLE huge+statement ok+SET datafusion.execution.target_partitions = 4++statement ok+CREATE TABLE p AS SELECT value AS k, value % 10 AS g FROM generate_series(1, 1000)++# A reference inside a scalar subquery is bound to the CTE.+query II+WITH c AS MATERIALIZED (SELECT g, count(*) AS n FROM p GROUP BY g)+SELECT (SELECT count(*) FROM (SELECT g FROM c GROUP BY g)), max(n) FROM c+----+10 100++# A view is planned on its own, and its plan is then optimized again as a part+# of the outer plan.+statement ok+CREATE VIEW pv AS+WITH c AS MATERIALIZED (SELECT g, count(*) AS n FROM p GROUP BY g)+SELECT g, sum(n) AS s FROM c GROUP BY g++query II rowsort+SELECT a.g, a.s + b.s FROM pv a JOIN pv b ON a.g = b.g+----+0 200+1 200+2 200+3 200+4 200+5 200+6 200+7 200+8 200+9 200++statement ok+DROP VIEW pv++statement ok+RESET datafusion.explain.physical_plan_only++statement ok+RESET datafusion.execution.target_partitions++statement ok+DROP TABLE p+# Setting a runtime option also sets this option to false.statement okRESET datafusion.catalog.create_default_catalog_and_schema