Apache DataFusion SQL Query Engine
refactor(physical-plan): Simplify `ExecutionPlan` API with `replace_children` (#23903)
Justin O'Dwyercaa3cb4af1bb
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex a5a38edf0b..6f176f8b46 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -36,8 +36,8 @@ use datafusion::physical_expr::EquivalenceProperties;use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion::physical_plan::memory::MemoryStream;use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,- SendableRecordBatchStream, project_schema,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,+ PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream, project_schema,};use datafusion::prelude::*;@@ -268,13 +268,24 @@ impl ExecutionPlan for CustomExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rsindex 6decb84b55..89eff74e0d 100644--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs@@ -39,7 +39,8 @@ use datafusion::execution::{SendableRecordBatchStream, TaskContext};use datafusion::logical_expr::LogicalPlanBuilder;use datafusion::physical_plan::stream::RecordBatchStreamAdapter;use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions,};use datafusion::prelude::*;use futures::stream::{StreamExt, TryStreamExt};@@ -237,9 +238,10 @@ impl ExecutionPlan for BufferingExecutionPlan {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {if children.len() == 1 {Ok(Arc::new(BufferingExecutionPlan::new(@@ -251,6 +253,16 @@ impl ExecutionPlan for BufferingExecutionPlan {}}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rsindex d5197fe61b..51c1bc7c55 100644--- a/datafusion-examples/examples/proto/composed_extension_codec.rs+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs@@ -39,6 +39,7 @@ use datafusion::common::Result;use datafusion::common::internal_err;use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::execution::TaskContext;+use datafusion::physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use datafusion::physical_plan::{DisplayAs, ExecutionPlan};use datafusion::prelude::SessionContext;use datafusion_proto::physical_plan::{@@ -111,13 +112,24 @@ impl ExecutionPlan for ParentExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unreachable!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -198,13 +210,24 @@ impl ExecutionPlan for ChildExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unreachable!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex 388175ee3a..7a8f533ac3 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -100,7 +100,6 @@ use futures::{use rand::{Rng, SeedableRng, rngs::StdRng};use tonic::async_trait;-use datafusion::optimizer::simplify_expressions::simplify_literal::parse_literal;use datafusion::{catalog::Session,execution::{@@ -116,6 +115,10 @@ use datafusion::{physical_planner::{DefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner},prelude::*,};+use datafusion::{+ optimizer::simplify_expressions::simplify_literal::parse_literal,+ physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions},+};use datafusion_common::{DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,@@ -698,9 +701,10 @@ impl ExecutionPlan for SampleExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(Self::try_new(children.swap_remove(0),@@ -710,6 +714,16 @@ impl ExecutionPlan for SampleExec {)?))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex ef5669a3a1..4cf96cb364 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -45,8 +45,8 @@ use datafusion_physical_expr::{use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr,- PlanProperties, collect_partitioned,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,+ PhysicalExpr, PlanProperties, ReplaceChildrenOptions, collect_partitioned,};use datafusion_session::Session;@@ -572,13 +572,24 @@ impl ExecutionPlan for DmlResultExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 1202f08a56..3c1e7b5078 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -3323,6 +3323,7 @@ mod tests {use datafusion_functions_aggregate::expr_fn::sum;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};+ use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use datafusion_session::QueryPlanner;#[derive(Debug)]@@ -4932,9 +4933,10 @@ mod tests {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {if children.is_empty() {Ok(self)@@ -4943,6 +4945,16 @@ mod tests {}}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -5102,12 +5114,22 @@ digraph {fn name(&self) -> &str {"always ok"}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(Self(children)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn schema(&self) -> SchemaRef {Arc::new(Schema::empty())}@@ -5157,12 +5179,22 @@ digraph {fn schema(&self) -> SchemaRef {Arc::new(Schema::empty())}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {unimplemented!()}@@ -5216,10 +5248,16 @@ digraph {// ok planlet ok_node: Arc<dyn ExecutionPlan> = Arc::new(OkExtensionNode(vec![]));let child = Arc::clone(&ok_node);- let ok_plan = Arc::clone(&ok_node).with_new_children(vec![- Arc::clone(&child).with_new_children(vec![Arc::clone(&child)])?,- Arc::clone(&child),- ])?;+ let ok_plan = Arc::clone(&ok_node).replace_children(+ vec![+ Arc::clone(&child).replace_children(+ vec![Arc::clone(&child)],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?,+ Arc::clone(&child),+ ],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;// Test: check should pass with same schemalet equal_schema = ok_plan.schema();@@ -5251,10 +5289,16 @@ digraph {// Test: should fail when descendent extension node failslet failing_node: Arc<dyn ExecutionPlan> = Arc::new(InvariantFailsExtensionNode);- let invalid_plan = ok_node.with_new_children(vec![- Arc::clone(&child).with_new_children(vec![Arc::clone(&failing_node)])?,- Arc::clone(&child),- ])?;+ let invalid_plan = ok_node.replace_children(+ vec![+ Arc::clone(&child).replace_children(+ vec![Arc::clone(&failing_node)],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?,+ Arc::clone(&child),+ ],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let result = OptimizationInvariantChecker::new(&rule).check(&invalid_plan, &ok_plan.schema());if cfg!(debug_assertions) {@@ -5287,12 +5331,22 @@ digraph {fn schema(&self) -> SchemaRef {Arc::new(Schema::empty())}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}@@ -5339,10 +5393,16 @@ digraph {let failing_node: Arc<dyn ExecutionPlan> = Arc::new(ExecutableInvariantFails);let ok_node: Arc<dyn ExecutionPlan> = Arc::new(OkExtensionNode(vec![]));let child = Arc::clone(&ok_node);- let plan = ok_node.with_new_children(vec![- Arc::clone(&child).with_new_children(vec![Arc::clone(&failing_node)])?,- Arc::clone(&child),- ])?;+ let plan = ok_node.replace_children(+ vec![+ Arc::clone(&child).replace_children(+ vec![Arc::clone(&failing_node)],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?,+ Arc::clone(&child),+ ],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let expected_err = InvariantChecker(InvariantLevel::Executable).check(&plan).unwrap_err();diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex 43f4be05b2..7abbcd6e95 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -40,10 +40,12 @@ use datafusion_common::project_schema;use datafusion_common::stats::Precision;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_physical_expr::EquivalenceProperties;-use datafusion_physical_plan::PlanProperties;use datafusion_physical_plan::StatisticsArgs;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;+use datafusion_physical_plan::{+ ChildrenPropertiesMode, PlanProperties, ReplaceChildrenOptions,+};use async_trait::async_trait;use futures::stream::Stream;@@ -165,13 +167,24 @@ impl ExecutionPlan for CustomExecutionPlan {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex 7437bbc543..a8f7f09ad0 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -41,6 +41,7 @@ use datafusion_expr::expr::{BinaryExpr, Cast};use datafusion_functions_aggregate::expr_fn::count;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};+use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use async_trait::async_trait;@@ -117,9 +118,10 @@ impl ExecutionPlan for CustomPlan {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {// CustomPlan has no childrenif children.is_empty() {@@ -129,6 +131,16 @@ impl ExecutionPlan for CustomPlan {}}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rsindex c14ca685b2..6213be8e2d 100644--- a/datafusion/core/tests/custom_sources_cases/statistics.rs+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs@@ -37,7 +37,9 @@ use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{project_schema, stats::Precision};use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};-use datafusion_physical_plan::{StatisticsArgs, StatisticsContext};+use datafusion_physical_plan::{+ ChildrenPropertiesMode, ReplaceChildrenOptions, StatisticsArgs, StatisticsContext,+};use async_trait::async_trait;@@ -160,13 +162,24 @@ impl ExecutionPlan for StatisticsValidation {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/fuzz_cases/once_exec.rs b/datafusion/core/tests/fuzz_cases/once_exec.rsindex 638cbe4c9d..c1db9a110d 100644--- a/datafusion/core/tests/fuzz_cases/once_exec.rs+++ b/datafusion/core/tests/fuzz_cases/once_exec.rs@@ -16,13 +16,14 @@// under the License.use arrow_schema::SchemaRef;-use datafusion_common::internal_datafusion_err;use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::{Result, internal_datafusion_err};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions,};use std::fmt::{Debug, Formatter};use std::sync::{Arc, Mutex};@@ -87,19 +88,30 @@ impl ExecutionPlan for OnceExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,- ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Returns a stream which yields datafn execute(&self,partition: usize,_context: Arc<TaskContext>,- ) -> datafusion_common::Result<SendableRecordBatchStream> {+ ) -> Result<SendableRecordBatchStream> {assert_eq!(partition, 0);let stream = self.stream.lock().unwrap().take();@@ -111,8 +123,8 @@ impl ExecutionPlan for OnceExec {&self,_f: &mut dyn FnMut(&Arc<dyn datafusion_physical_plan::PhysicalExpr>,- ) -> datafusion_common::Result<TreeNodeRecursion>,- ) -> datafusion_common::Result<TreeNodeRecursion> {+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {Ok(TreeNodeRecursion::Continue)}}diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex e0b152d1f0..2dbacf1d89 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -74,7 +74,8 @@ use datafusion_physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlanProperties, PlanProperties, displayable,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlanProperties,+ PlanProperties, ReplaceChildrenOptions, displayable,};use insta::Settings;@@ -193,9 +194,10 @@ impl ExecutionPlan for SortRequiredExec {vec![Some(OrderingRequirements::from(self.expr.clone()))]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert_eq!(children.len(), 1);let child = children.pop().unwrap();@@ -205,6 +207,16 @@ impl ExecutionPlan for SortRequiredExec {)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,@@ -290,15 +302,26 @@ impl ExecutionPlan for SinglePartitionMaintainsOrderExec {vec![false]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert_eq!(children.len(), 1);let child = children.pop().unwrap();Ok(Arc::new(Self::new(child)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rsindex 83fabcdff8..86b60519da 100644--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs@@ -45,8 +45,9 @@ use datafusion_physical_plan::limit::GlobalLimitExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,- PlanProperties, SendableRecordBatchStream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ ExecutionPlanProperties, Partitioning, PlanProperties, ReplaceChildrenOptions,+ SendableRecordBatchStream,};use datafusion_physical_optimizer::output_requirements::OutputRequirementExec;@@ -130,18 +131,32 @@ impl ExecutionPlan for MockMultiPartitionExec {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}++ fn replace_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(self)+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,) -> Result<TreeNodeRecursion> {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(- self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,- ) -> Result<Arc<dyn ExecutionPlan>> {- Ok(self)- }+fn execute(&self,_partition: usize,@@ -1022,17 +1037,27 @@ impl ExecutionPlan for MockReqExec {fn maintains_input_order(&self) -> Vec<bool> {vec![true]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- mut c: Vec<Arc<dyn ExecutionPlan>>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- assert_eq!(c.len(), 1);+ assert_eq!(children.len(), 1);Ok(Arc::new(MockReqExec::new(- c.pop().expect("1 child"),+ children.pop().expect("1 child"),self.dist.clone(),self.ord.clone(),)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn execute(&self,_p: usize,diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex c7e3799842..265279a8ca 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -39,13 +39,15 @@ use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpruse datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::join_selection::JoinSelection;-use datafusion_physical_plan::ExecutionPlanProperties;use datafusion_physical_plan::displayable;use datafusion_physical_plan::joins::utils::ColumnIndex;use datafusion_physical_plan::joins::utils::JoinFilter;use datafusion_physical_plan::joins::{HashJoinExec, NestedLoopJoinExec, PartitionMode};use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;+use datafusion_physical_plan::{+ ChildrenPropertiesMode, ExecutionPlanProperties, ReplaceChildrenOptions,+};use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs,StatisticsContext,@@ -1108,13 +1110,24 @@ impl ExecutionPlan for UnboundedExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -1212,13 +1225,24 @@ impl ExecutionPlan for StatisticsExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs b/datafusion/core/tests/physical_optimizer/pushdown_utils.rsindex 4f8b9ad42b..0c2286527d 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs@@ -30,6 +30,7 @@ use datafusion_physical_expr_common::physical_expr::fmt_sql;use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_plan::filter::batch_filter;use datafusion_physical_plan::filter_pushdown::{FilterPushdownPhase, PushedDown};+use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, displayable,filter::FilterExec,@@ -489,9 +490,10 @@ impl ExecutionPlan for TestNode {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert!(children.len() == 1);Ok(Arc::new(TestNode::new(@@ -501,6 +503,16 @@ impl ExecutionPlan for TestNode {)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex 833077fe49..0835497f34 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -70,9 +70,9 @@ use datafusion_physical_plan::tree_node::PlanContext;use datafusion_physical_plan::union::UnionExec;use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, InputDistributionRequirements,- InputOrderMode, Partitioning, PlanProperties, SortOrderPushdownResult,- StatisticsArgs, displayable,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ InputDistributionRequirements, InputOrderMode, Partitioning, PlanProperties,+ ReplaceChildrenOptions, SortOrderPushdownResult, StatisticsArgs, displayable,};/// Create a non sorted parquet exec@@ -526,9 +526,10 @@ impl ExecutionPlan for RequirementsTestExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert_eq!(children.len(), 1);Ok(RequirementsTestExec::new(Arc::clone(&children[0]))@@ -538,6 +539,16 @@ impl ExecutionPlan for RequirementsTestExec {.into_arc())}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -1025,9 +1036,10 @@ impl ExecutionPlan for TestScan {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {if children.is_empty() {Ok(self)@@ -1036,6 +1048,16 @@ impl ExecutionPlan for TestScan {}}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rsindex c61fe018aa..0eefcdb551 100644--- a/datafusion/core/tests/user_defined/insert_operation.rs+++ b/datafusion/core/tests/user_defined/insert_operation.rs@@ -28,7 +28,9 @@ use datafusion_common::config::Dialect;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_expr::{Expr, TableType, dml::InsertOp};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};-use datafusion_physical_plan::execution_plan::SchedulingType;+use datafusion_physical_plan::{+ ChildrenPropertiesMode, ReplaceChildrenOptions, execution_plan::SchedulingType,+};use datafusion_physical_plan::{DisplayAs, ExecutionPlan, PlanProperties,execution_plan::{Boundedness, EmissionType},@@ -162,14 +164,25 @@ impl ExecutionPlan for TestInsertExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert!(children.is_empty());Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex 2b042b613d..da7fdd8879 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -99,6 +99,7 @@ use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};use datafusion_optimizer::AnalyzerRule;use datafusion_optimizer::optimizer::ApplyOrder;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};+use datafusion_physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use async_trait::async_trait;use datafusion_common::cast::as_string_view_array;@@ -725,13 +726,24 @@ impl ExecutionPlan for TopKExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(TopKExec::new(children[0].clone(), self.k)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Execute one partition and return an iterator over RecordBatchfn execute(&self,diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rsindex b89cf5d356..4bf04133b7 100644--- a/datafusion/datasource/src/sink.rs+++ b/datafusion/datasource/src/sink.rs@@ -32,9 +32,9 @@ use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequireuse datafusion_physical_plan::metrics::MetricsSet;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties,- InputDistributionRequirements, Partitioning, PlanProperties,- SendableRecordBatchStream, execute_input_stream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ ExecutionPlanProperties, InputDistributionRequirements, Partitioning, PlanProperties,+ ReplaceChildrenOptions, SendableRecordBatchStream, execute_input_stream,};use async_trait::async_trait;@@ -305,9 +305,10 @@ impl ExecutionPlan for DataSinkExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(Self::new(Arc::clone(&children[0]),@@ -316,6 +317,16 @@ impl ExecutionPlan for DataSinkExec {)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex 929fed02b3..741010c595 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -33,7 +33,8 @@ use datafusion_physical_plan::metrics::{use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::stream::BatchSplitStream;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions,};use itertools::Itertools;@@ -400,6 +401,24 @@ impl ExecutionPlan for DataSourceExec {Vec::new()}+ fn replace_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(self)+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,@@ -408,13 +427,6 @@ impl ExecutionPlan for DataSourceExec {self.data_source.apply_expressions(f)}- fn with_new_children(- self: Arc<Self>,- _: Vec<Arc<dyn ExecutionPlan>>,- ) -> Result<Arc<dyn ExecutionPlan>> {- Ok(self)- }-/// Implementation of [`ExecutionPlan::repartitioned`] which relies upon the inner [`DataSource::repartitioned`].////// If the data source does not support changing its partitioning, returns `Ok(None)` (the default). Referdiff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex a0dd5e6cb6..d7ee5dace3 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -25,8 +25,8 @@ use datafusion_common::{DataFusionError, Result, Statistics};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr_common::metrics::MetricsSet;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, StatisticsArgs,- StatisticsContext,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions, StatisticsArgs, StatisticsContext,};use stabby::string::String as SString;use stabby::vec::Vec as SVec;@@ -185,7 +185,10 @@ unsafe extern "C" fn with_new_children_fn_wrapper(.collect();let children = sresult_return!(children);- let new_plan = sresult_return!(inner_plan.with_new_children(children));+ let new_plan = sresult_return!(inner_plan.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute)+ ));FFI_Result::Ok(FFI_ExecutionPlan::new(new_plan, runtime))}@@ -302,7 +305,7 @@ fn pass_runtime_to_children(// If the parent is foreign and the child is local to this library, then when// we called `children()` above we will get something other than a// `ForeignExecutionPlan`. In this case wrap the plan in a `ForeignExecutionPlan`- // because when we call `with_new_children` below it will extract the+ // because when we call `replace_children` below it will extract the// FFI plan that does contain the runtime.if plan_is_foreign && !child.is::<ForeignExecutionPlan>() {updated_children = true;@@ -315,7 +318,12 @@ fn pass_runtime_to_children(}).collect::<Result<Vec<_>>>()?;if updated_children {- Arc::clone(plan).with_new_children(children).map(Some)+ Arc::clone(plan)+ .replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ .map(Some)} else {Ok(None)}@@ -453,9 +461,10 @@ impl ExecutionPlan for ForeignExecutionPlan {self.children.iter().collect()}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {let children = children.into_iter()@@ -467,6 +476,16 @@ impl ExecutionPlan for ForeignExecutionPlan {(&new_plan).try_into()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,@@ -621,9 +640,10 @@ pub mod tests {self.children.iter().collect()}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(EmptyExec {props: Arc::clone(&self.props),@@ -635,6 +655,16 @@ pub mod tests {}))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -777,7 +807,10 @@ pub mod tests {assert_eq!(parent_foreign.children().len(), 0);assert_eq!(child_foreign.children().len(), 0);- let parent_foreign = parent_foreign.with_new_children(vec![child_foreign])?;+ let parent_foreign = parent_foreign.replace_children(+ vec![child_foreign],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;assert_eq!(parent_foreign.children().len(), 1);// Version 2: Adding child to the local plan@@ -787,7 +820,10 @@ pub mod tests {let child_foreign = <Arc<dyn ExecutionPlan>>::try_from(&child_local)?;let parent_plan = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let parent_plan = parent_plan.with_new_children(vec![child_foreign])?;+ let parent_plan = parent_plan.replace_children(+ vec![child_foreign],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let mut parent_local = FFI_ExecutionPlan::new(parent_plan, None);parent_local.library_marker_id = crate::mock_foreign_marker_id;let parent_foreign = <Arc<dyn ExecutionPlan>>::try_from(&parent_local)?;diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex b9f353e89b..83057d8c45 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -37,7 +37,9 @@ use datafusion_common::{Result, exec_err};use datafusion_execution::RecordBatchStream;use datafusion_expr::Expr;use datafusion_physical_expr::{EquivalenceProperties, Partitioning};-use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::{+ ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions,+};use datafusion_session::Session;use futures::Stream;use tokio::runtime::Handle;@@ -211,13 +213,24 @@ impl ExecutionPlan for AsyncTestExecutionPlan {Vec::default()}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/ffi/tests/ffi_execution_plan.rs b/datafusion/ffi/tests/ffi_execution_plan.rsindex 946dca61e1..4067d7eb49 100644--- a/datafusion/ffi/tests/ffi_execution_plan.rs+++ b/datafusion/ffi/tests/ffi_execution_plan.rs@@ -22,12 +22,15 @@ mod tests {use arrow_schema::DataType;use datafusion_common::DataFusionError;use datafusion_common::tree_node::TreeNodeRecursion;- use datafusion_ffi::execution_plan::FFI_ExecutionPlan;- use datafusion_ffi::execution_plan::ForeignExecutionPlan;- use datafusion_ffi::execution_plan::{ExecutionPlanPrivateData, tests::EmptyExec};+ use datafusion_ffi::execution_plan::{+ ExecutionPlanPrivateData, FFI_ExecutionPlan, ForeignExecutionPlan,+ tests::EmptyExec,+ };use datafusion_ffi::tests::utils::get_module;- use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::execution_plan::InvariantLevel;+ use datafusion_physical_plan::{+ ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions,+ };use std::sync::Arc;#[test]@@ -135,7 +138,10 @@ mod tests {let grandchild_plan = generate_local_plan();- let child_plan = child_plan.with_new_children(vec![grandchild_plan])?;+ let child_plan = child_plan.replace_children(+ vec![grandchild_plan],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;unsafe {// Originally the runtime is not set. We go through the unsafe casting@@ -150,7 +156,10 @@ mod tests {assert!((*grandchild_private_data).runtime.is_none());}- let parent_plan = generate_local_plan().with_new_children(vec![child_plan])?;+ let parent_plan = generate_local_plan().replace_children(+ vec![child_plan],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;// Adding the grandchild beneath this FFI plan should get the runtime passed down.let runtime = tokio::runtime::Builder::new_current_thread()diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rsindex 10da9e4e75..93862df3b4 100644--- a/datafusion/physical-optimizer/src/ensure_coop.rs+++ b/datafusion/physical-optimizer/src/ensure_coop.rs@@ -130,7 +130,10 @@ impl PhysicalOptimizerRule for EnsureCooperative {#[cfg(test)]mod tests {use super::*;- use datafusion_physical_plan::{displayable, test::scan_partitioned};+ use datafusion_physical_plan::{+ ChildrenPropertiesMode, ReplaceChildrenOptions, displayable,+ test::scan_partitioned,+ };use insta::assert_snapshot;#[tokio::test]@@ -328,9 +331,10 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(DummyExec::new(&self.name,@@ -339,6 +343,15 @@ mod tests {self.evaluation_type,)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn execute(&self,_: usize,diff --git a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rsindex adbc3dde7a..07bc98b2db 100644--- a/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs+++ b/datafusion/physical-optimizer/src/ensure_requirements/enforce_distribution.rs@@ -55,7 +55,9 @@ use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy,};use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;-use datafusion_physical_plan::execution_plan::EmissionType;+use datafusion_physical_plan::execution_plan::{+ EmissionType, replace_children_if_necessary,+};use datafusion_physical_plan::joins::{CrossJoinExec, HashJoinExec, PartitionMode, SortMergeJoinExec,};@@ -69,7 +71,7 @@ use datafusion_physical_plan::windows::WindowAggExec;use datafusion_physical_plan::windows::{BoundedWindowAggExec, get_best_fitting_window};use datafusion_physical_plan::{ChildSatisfactionOptions, Distribution, ExecutionPlan, InputDistributionRequirements,- Partitioning, with_new_children_if_necessary,+ Partitioning,};use itertools::izip;@@ -760,8 +762,10 @@ fn preserving_order_enables_streaming(return Ok(false);}// Build parent with the ordered child- let with_ordered =- Arc::clone(parent).with_new_children(vec![Arc::clone(ordered_child)])?;+ let with_ordered = replace_children_if_necessary(+ Arc::clone(parent),+ vec![Arc::clone(ordered_child)],+ )?;if with_ordered.pipeline_behavior() == EmissionType::Final {// Parent is blocking even with ordering — no benefitreturn Ok(false);@@ -769,7 +773,8 @@ fn preserving_order_enables_streaming(// Build parent with an unordered child via CoalescePartitionsExec.let unordered_child: Arc<dyn ExecutionPlan> =Arc::new(CoalescePartitionsExec::new(Arc::clone(ordered_child)));- let without_ordered = Arc::clone(parent).with_new_children(vec![unordered_child])?;+ let without_ordered =+ replace_children_if_necessary(Arc::clone(parent), vec![unordered_child])?;Ok(without_ordered.pipeline_behavior() == EmissionType::Final)}@@ -1519,16 +1524,16 @@ pub fn ensure_distribution(// DataArc::new(InterleaveExec::try_new(children_plans)?)} else {- // Route through `with_new_children_if_necessary` so the common+ // Route through `replace_children_if_necessary` so the common// case where no child was replaced above skips the expensive- // `with_new_children` rebuild. For nodes like `ProjectionExec`,- // `with_new_children` recomputes schema / equivalence properties /+ // `replace_children` rebuild. For nodes like `ProjectionExec`,+ // `replace_children` recomputes schema / equivalence properties /// output ordering via `try_new` even when the input Arcs are// identical, which dominates `ensure_distribution` time on deep// projection stacks over plans where no distribution change// applies (point queries with no join / aggregate / unmet// ordering).- with_new_children_if_necessary(plan, children_plans)?+ replace_children_if_necessary(plan, children_plans)?};Ok(Transformed::yes(DistributionContext::new(diff --git a/datafusion/physical-optimizer/src/filter_pushdown.rs b/datafusion/physical-optimizer/src/filter_pushdown.rsindex 06aa632a9d..18fe151000 100644--- a/datafusion/physical-optimizer/src/filter_pushdown.rs+++ b/datafusion/physical-optimizer/src/filter_pushdown.rs@@ -39,11 +39,12 @@ use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};use datafusion_common::{Result, assert_eq_or_internal_err, config::ConfigOptions};use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::physical_expr::is_volatile;+use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::execution_plan::replace_children_if_necessary;use datafusion_physical_plan::filter_pushdown::{ChildFilterPushdownResult, ChildPushdownResult, FilterPushdownPhase,FilterPushdownPropagation, PushedDown,};-use datafusion_physical_plan::{ExecutionPlan, with_new_children_if_necessary};use itertools::{Itertools, izip};@@ -573,7 +574,7 @@ fn push_down_filters(}// Re-create this node with new children- let updated_node = with_new_children_if_necessary(Arc::clone(node), new_children)?;+ let updated_node = replace_children_if_necessary(Arc::clone(node), new_children)?;// TODO: by calling `handle_child_pushdown_result` we are assuming that the// `ExecutionPlan` implementation will not change the plan itself.diff --git a/datafusion/physical-optimizer/src/hash_join_buffering.rs b/datafusion/physical-optimizer/src/hash_join_buffering.rsindex 7a198cac13..dbdfd34a9a 100644--- a/datafusion/physical-optimizer/src/hash_join_buffering.rs+++ b/datafusion/physical-optimizer/src/hash_join_buffering.rs@@ -21,6 +21,7 @@ use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_physical_plan::ExecutionPlan;use datafusion_physical_plan::buffer::BufferExec;+use datafusion_physical_plan::execution_plan::replace_children_if_necessary;use datafusion_physical_plan::joins::HashJoinExec;use std::sync::Arc;@@ -74,19 +75,25 @@ impl PhysicalOptimizerRule for HashJoinBuffering {if node.left.is::<BufferExec>() {return Ok(Transformed::no(plan));}- plan.with_new_children(vec![- Arc::new(BufferExec::new(Arc::clone(&node.left), capacity)),- Arc::clone(&node.right),- ])?+ replace_children_if_necessary(+ plan,+ vec![+ Arc::new(BufferExec::new(Arc::clone(&node.left), capacity)),+ Arc::clone(&node.right),+ ],+ )?} else {// Do not stack BufferExec nodes together.if node.right.is::<BufferExec>() {return Ok(Transformed::no(plan));}- plan.with_new_children(vec![- Arc::clone(&node.left),- Arc::new(BufferExec::new(Arc::clone(&node.right), capacity)),- ])?+ replace_children_if_necessary(+ plan,+ vec![+ Arc::clone(&node.left),+ Arc::new(BufferExec::new(Arc::clone(&node.right), capacity)),+ ],+ )?},))})diff --git a/datafusion/physical-optimizer/src/limit_pushdown.rs b/datafusion/physical-optimizer/src/limit_pushdown.rsindex 01a288f7f1..f88a2be14e 100644--- a/datafusion/physical-optimizer/src/limit_pushdown.rs+++ b/datafusion/physical-optimizer/src/limit_pushdown.rs@@ -72,6 +72,7 @@ use datafusion_common::tree_node::{Transformed, TreeNodeRecursion};use datafusion_common::utils::combine_limit;use datafusion_physical_plan::coalesce_partitions::CoalescePartitionsExec;use datafusion_physical_plan::empty::EmptyExec;+use datafusion_physical_plan::execution_plan::replace_children_if_necessary;use datafusion_physical_plan::limit::{GlobalLimitExec, LocalLimitExec};use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;use datafusion_physical_plan::projection::ProjectionExec;@@ -403,7 +404,7 @@ pub(crate) fn pushdown_limits(.collect::<Result<_>>()?;if changed {- new_node.data.with_new_children(new_children)+ replace_children_if_necessary(new_node.data, new_children)} else {Ok(new_node.data)}diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rsindex fc8bf490b9..5419812701 100644--- a/datafusion/physical-optimizer/src/output_requirements.rs+++ b/datafusion/physical-optimizer/src/output_requirements.rs@@ -34,7 +34,9 @@ use datafusion_common::{Result, Statistics, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::Distribution;use datafusion_physical_expr_common::sort_expr::OrderingRequirements;-use datafusion_physical_plan::execution_plan::Boundedness;+use datafusion_physical_plan::execution_plan::{+ Boundedness, replace_children_if_necessary,+};use datafusion_physical_plan::projection::{ProjectionExec, make_with_child, update_expr, update_ordering_requirement,};@@ -42,8 +44,9 @@ use datafusion_physical_plan::scalar_subquery::ScalarSubqueryExec;use datafusion_physical_plan::sorts::sort::SortExec;use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec;use datafusion_physical_plan::{- ChildStats, DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties,- PlanProperties, SendableRecordBatchStream, StatisticsArgs,+ ChildStats, ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ ExecutionPlanProperties, PlanProperties, ReplaceChildrenOptions,+ SendableRecordBatchStream, StatisticsArgs,};/// This rule either adds or removes [`OutputRequirements`]s to/from the physical@@ -233,9 +236,10 @@ impl ExecutionPlan for OutputRequirementExec {vec![self.order_requirement.clone()]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(Self::new(children.remove(0), // has a single child@@ -245,6 +249,16 @@ impl ExecutionPlan for OutputRequirementExec {)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -471,7 +485,7 @@ fn require_top_ordering_helper(require_top_ordering_helper(Arc::clone(&children[idx]))?;if is_changed {children[idx] = new_child;- return Ok((plan.with_new_children(children)?, true));+ return Ok((replace_children_if_necessary(plan, children)?, true));}}Ok((plan, false))diff --git a/datafusion/physical-optimizer/src/topk_repartition.rs b/datafusion/physical-optimizer/src/topk_repartition.rsindex 115bdc3cb5..d8fa1ac986 100644--- a/datafusion/physical-optimizer/src/topk_repartition.rs+++ b/datafusion/physical-optimizer/src/topk_repartition.rs@@ -48,6 +48,7 @@ use crate::PhysicalOptimizerRule;use datafusion_common::Result;use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_physical_plan::execution_plan::replace_children_if_necessary;use std::sync::Arc;// CoalesceBatchesExec is deprecated on main (replaced by arrow-rs BatchCoalescer),// but older DataFusion versions may still insert it between SortExec and RepartitionExec.@@ -151,7 +152,7 @@ impl PhysicalOptimizerRule for TopKRepartition {// Rebuild the tree above the repartitionlet new_sort_input = if let Some(parent) = repart_parent {- parent.with_new_children(vec![new_repartition])?+ replace_children_if_necessary(parent, vec![new_repartition])?} else {new_repartition};diff --git a/datafusion/physical-optimizer/src/window_topn.rs b/datafusion/physical-optimizer/src/window_topn.rsindex 29b8f4a460..20bd8b0d38 100644--- a/datafusion/physical-optimizer/src/window_topn.rs+++ b/datafusion/physical-optimizer/src/window_topn.rs@@ -60,6 +60,7 @@ use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal};use datafusion_physical_expr::window::StandardWindowExpr;use datafusion_physical_expr::{LexOrdering, PhysicalSortExpr};use datafusion_physical_plan::ExecutionPlan;+use datafusion_physical_plan::execution_plan::replace_children_if_necessary;use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::projection::ProjectionExec;use datafusion_physical_plan::repartition::RepartitionExec;@@ -192,13 +193,13 @@ impl WindowTopN {.ok()?;// Step 7: Rebuild window with PartitionedTopKExec as its child- let mut result = window_exec- .with_new_children(vec![Arc::new(partitioned_topk)])- .ok()?;+ let mut result =+ replace_children_if_necessary(window_exec, vec![Arc::new(partitioned_topk)])+ .ok()?;// Step 8: Rebuild intermediate nodes (ProjectionExec/RepartitionExec)for node in intermediates.into_iter().rev() {- result = node.with_new_children(vec![result]).ok()?;+ result = replace_children_if_necessary(node, vec![result]).ok()?;}Some(result)diff --git a/datafusion/physical-plan/benches/compute_statistics.rs b/datafusion/physical-plan/benches/compute_statistics.rsindex 93c95ea4ba..cddf4c2396 100644--- a/datafusion/physical-plan/benches/compute_statistics.rs+++ b/datafusion/physical-plan/benches/compute_statistics.rs@@ -47,8 +47,8 @@ use datafusion_physical_plan::filter::FilterExec;use datafusion_physical_plan::joins::CrossJoinExec;use datafusion_physical_plan::statistics::StatisticsArgs;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, Partitioning, SendableRecordBatchStream,- StatisticsContext,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Partitioning,+ ReplaceChildrenOptions, SendableRecordBatchStream, StatisticsContext,};/// Minimal leaf node for benchmarking@@ -98,18 +98,29 @@ impl ExecutionPlan for BenchLeaf {vec![]}- fn apply_expressions(- &self,- _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,- ) -> Result<TreeNodeRecursion> {- Ok(TreeNodeRecursion::Continue)+ fn replace_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ Ok(self)}fn with_new_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(self)+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)}fn execute(diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 1c5b56ffad..a39c6f3486 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -165,9 +165,10 @@ use crate::filter_pushdown::{};use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};use crate::statistics::{ChildStats, StatisticsArgs};+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};use crate::{DisplayFormatType, Distribution, ExecutionPlan, InputDistributionRequirements,- InputOrderMode, SendableRecordBatchStream, Statistics, check_if_same_properties,+ InputOrderMode, SendableRecordBatchStream, Statistics,};use datafusion_common::config::ConfigOptions;use parking_lot::Mutex;@@ -2031,6 +2032,45 @@ impl ExecutionPlan for AggregateExec {vec![&self.input]}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let mut me = AggregateExec::try_new_with_schema(+ self.mode,+ Arc::clone(&self.group_by),+ self.aggr_expr.to_vec(),+ Arc::clone(&self.filter_expr),+ Arc::clone(&children[0]),+ Arc::clone(&self.input_schema),+ Arc::clone(&self.schema),+ )?;+ me.limit_options = self.limit_options;+ me.dynamic_filter.clone_from(&self.dynamic_filter);+ Ok(Arc::new(me))+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn apply_expressions(&self,f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,@@ -2068,36 +2108,14 @@ impl ExecutionPlan for AggregateExec {.collect()}- fn with_new_children(- self: Arc<Self>,- children: Vec<Arc<dyn ExecutionPlan>>,- ) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);-- let mut me = AggregateExec::try_new_with_schema(- self.mode,- Arc::clone(&self.group_by),- self.aggr_expr.to_vec(),- Arc::clone(&self.filter_expr),- Arc::clone(&children[0]),- Arc::clone(&self.input_schema),- Arc::clone(&self.schema),- )?;- me.limit_options = self.limit_options;- me.dynamic_filter.clone_from(&self.dynamic_filter);-- Ok(Arc::new(me))- }-fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -3681,18 +3699,29 @@ mod tests {vec![]}- fn apply_expressions(- &self,- _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,- ) -> Result<TreeNodeRecursion> {- Ok(TreeNodeRecursion::Continue)+ fn replace_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ internal_err!("Children cannot be replaced in {self:?}")}fn with_new_children(self: Arc<Self>,- _: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- internal_err!("Children cannot be replaced in {self:?}")+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)}fn execute(@@ -5029,8 +5058,10 @@ mod tests {Arc::clone(&blocking_exec) as Arc<dyn ExecutionPlan>,schema,)?);- let new_agg =- Arc::clone(&aggregate_exec).with_new_children(vec![blocking_exec])?;+ let new_agg = Arc::clone(&aggregate_exec).replace_children(+ vec![blocking_exec],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;assert_eq!(new_agg.schema(), aggregate_exec.schema());Ok(())}diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rsindex 9a69518953..d1519828c2 100644--- a/datafusion/physical-plan/src/analyze.rs+++ b/datafusion/physical-plan/src/analyze.rs@@ -27,7 +27,10 @@ use super::{use crate::display::DisplayableExecutionPlan;use crate::execution_plan::EvaluationType;use crate::metrics::{MetricCategory, MetricType};-use crate::{DisplayFormatType, ExecutionPlan, Partitioning};+use crate::{+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, Partitioning,+ ReplaceChildrenOptions,+};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::format::ExplainFormat;@@ -228,9 +231,10 @@ impl ExecutionPlan for AnalyzeExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(AnalyzeExec::builder(@@ -246,6 +250,16 @@ impl ExecutionPlan for AnalyzeExec {))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rsindex 91531ec35c..f3ef13d4fd 100644--- a/datafusion/physical-plan/src/async_func.rs+++ b/datafusion/physical-plan/src/async_func.rs@@ -19,13 +19,14 @@ use crate::coalesce::LimitedBatchCoalescer;use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ ExecutionPlanProperties, PlanProperties, ReplaceChildrenOptions,+ validate_child_count,};use arrow::array::RecordBatch;use arrow_schema::{FieldRef, Fields, Schema, SchemaRef};+use datafusion_common::Result;use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};-use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;@@ -170,31 +171,43 @@ impl ExecutionPlan for AsyncFuncExec {)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- assert_eq_or_internal_err!(- children.len(),- 1,- "AsyncFuncExec wrong number of children"- );- check_if_same_properties!(self, children);- Ok(Arc::new(AsyncFuncExec::try_new(- self.async_exprs.clone(),- children.swap_remove(0),- )?))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(AsyncFuncExec::try_new(+ self.async_exprs.clone(),+ children.swap_remove(0),+ )?)),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -316,6 +329,7 @@ impl AsyncFuncExec {node: &datafusion_proto_models::protobuf::PhysicalPlanNode,ctx: &crate::proto::ExecutionPlanDecodeCtx<'_>,) -> Result<Arc<dyn ExecutionPlan>> {+ use datafusion_common::assert_eq_or_internal_err;use datafusion_proto_models::protobuf;let async_func = crate::expect_plan_variant!(node,diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rsindex a1c3c7ea01..24cca6b0b1 100644--- a/datafusion/physical-plan/src/buffer.rs+++ b/datafusion/physical-plan/src/buffer.rs@@ -18,7 +18,9 @@//! [`BufferExec`] decouples production and consumption on messages by buffering the input in the//! background up to a certain capacity.-use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};+use crate::execution_plan::{+ CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,+};use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,@@ -27,13 +29,13 @@ use crate::projection::ProjectionExec;use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::RecordBatchStreamAdapter;use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SortOrderPushdownResult,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions, SortOrderPushdownResult, validate_child_count,};use arrow::array::RecordBatch;use datafusion_common::config::ConfigOptions;use datafusion_common::tree_node::TreeNodeRecursion;-use datafusion_common::{Result, Statistics, internal_err, plan_err};+use datafusion_common::{Result, Statistics, internal_err};use datafusion_common_runtime::SpawnedTask;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};use datafusion_execution::{SendableRecordBatchStream, TaskContext};@@ -167,26 +169,42 @@ impl ExecutionPlan for BufferExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- if children.len() != 1 {- return plan_err!("BufferExec can only have one child");+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ Ok(Arc::new(Self::new(children.swap_remove(0), self.capacity)))+ }}- Ok(Arc::new(Self::new(children.swap_remove(0), self.capacity)))+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -271,9 +289,10 @@ impl ExecutionPlan for BufferExec {projection: &ProjectionExec,) -> Result<Option<Arc<dyn ExecutionPlan>>> {match self.input.try_swapping_with_projection(projection)? {- Some(new_input) => Ok(Some(- Arc::new(self.clone()).with_new_children(vec![new_input])?,- )),+ Some(new_input) => Ok(Some(replace_children_if_necessary(+ Arc::new(self.clone()),+ vec![new_input],+ )?)),None => Ok(None),}}diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex 511e5d793b..cb0f9b2ce4 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -27,8 +27,8 @@ use crate::projection::ProjectionExec;use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::EmptyRecordBatchStream;use crate::{- DisplayFormatType, ExecutionPlan, RecordBatchStream, SendableRecordBatchStream,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, RecordBatchStream,+ ReplaceChildrenOptions, SendableRecordBatchStream, validate_child_count,};use arrow::datatypes::SchemaRef;@@ -39,7 +39,7 @@ use datafusion_execution::TaskContext;use datafusion_physical_expr::PhysicalExpr;use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};-use crate::execution_plan::CardinalityEffect;+use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,@@ -181,26 +181,43 @@ impl ExecutionPlan for CoalesceBatchesExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(- CoalesceBatchesExec::new(children.swap_remove(0), self.target_batch_size)- .with_fetch(self.fetch),- ))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(+ CoalesceBatchesExec::new(children.swap_remove(0), self.target_batch_size)+ .with_fetch(self.fetch),+ )),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -260,9 +277,10 @@ impl ExecutionPlan for CoalesceBatchesExec {projection: &ProjectionExec,) -> Result<Option<Arc<dyn ExecutionPlan>>> {match self.input.try_swapping_with_projection(projection)? {- Some(new_input) => Ok(Some(- Arc::new(self.clone()).with_new_children(vec![new_input])?,- )),+ Some(new_input) => Ok(Some(replace_children_if_necessary(+ Arc::new(self.clone()),+ vec![new_input],+ )?)),None => Ok(None),}}diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 5cd7a707c2..6f58eb2f1e 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -26,12 +26,17 @@ use super::{DisplayAs, ExecutionPlanProperties, PlanProperties, SendableRecordBatchStream,Statistics,};-use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};+use crate::execution_plan::{+ CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,+};use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase};use crate::projection::{ProjectionExec, make_with_child};use crate::sort_pushdown::SortOrderPushdownResult;use crate::statistics::{ChildStats, StatisticsArgs};-use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_properties};+use crate::{+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, Partitioning,+ ReplaceChildrenOptions, validate_child_count,+};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_common::config::ConfigOptions;@@ -151,25 +156,44 @@ impl ExecutionPlan for CoalescePartitionsExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let mut plan = CoalescePartitionsExec::new(children.swap_remove(0));+ plan.fetch = self.fetch;+ Ok(Arc::new(plan))+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let mut plan = CoalescePartitionsExec::new(children.swap_remove(0));- plan.fetch = self.fetch;- Ok(Arc::new(plan))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -304,8 +328,7 @@ impl ExecutionPlan for CoalescePartitionsExec {self.input.with_preserve_order(preserve_order).and_then(|new_input| {- Arc::new(self.clone())- .with_new_children(vec![new_input])+ replace_children_if_necessary(Arc::new(self.clone()), vec![new_input]).ok()})}diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex dc7d98891e..9e27b26d6e 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -87,15 +87,16 @@ use crate::filter_pushdown::{use crate::projection::ProjectionExec;use crate::statistics::{ChildStats, StatisticsArgs};use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,- SendableRecordBatchStream, SortOrderPushdownResult, check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,+ SortOrderPushdownResult, validate_child_count,};use arrow::record_batch::RecordBatch;use arrow_schema::Schema;-use datafusion_common::{Result, Statistics, assert_eq_or_internal_err};+use datafusion_common::{Result, Statistics};use datafusion_execution::TaskContext;-use crate::execution_plan::SchedulingType;+use crate::execution_plan::{SchedulingType, replace_children_if_necessary};use crate::stream::RecordBatchStreamAdapter;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use futures::{Stream, StreamExt};@@ -275,27 +276,41 @@ impl ExecutionPlan for CooperativeExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- assert_eq_or_internal_err!(- children.len(),- 1,- "CooperativeExec requires exactly one child"- );- check_if_same_properties!(self, children);- Ok(Arc::new(CooperativeExec::new(children.swap_remove(0))))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ Ok(Arc::new(CooperativeExec::new(children.swap_remove(0))))+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -332,9 +347,10 @@ impl ExecutionPlan for CooperativeExec {projection: &ProjectionExec,) -> Result<Option<Arc<dyn ExecutionPlan>>> {match self.input.try_swapping_with_projection(projection)? {- Some(new_input) => Ok(Some(- Arc::new(self.clone()).with_new_children(vec![new_input])?,- )),+ Some(new_input) => Ok(Some(replace_children_if_necessary(+ Arc::new(self.clone()),+ vec![new_input],+ )?)),None => Ok(None),}}@@ -365,11 +381,13 @@ impl ExecutionPlan for CooperativeExec {match child.try_pushdown_sort(order)? {SortOrderPushdownResult::Exact { inner } => {- let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ let new_exec =+ replace_children_if_necessary(Arc::new(self.clone()), vec![inner])?;Ok(SortOrderPushdownResult::Exact { inner: new_exec })}SortOrderPushdownResult::Inexact { inner } => {- let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ let new_exec =+ replace_children_if_necessary(Arc::new(self.clone()), vec![inner])?;Ok(SortOrderPushdownResult::Inexact { inner: new_exec })}SortOrderPushdownResult::Unsupported => {diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rsindex 2370e3e6ce..d2bdcef2e9 100644--- a/datafusion/physical-plan/src/display.rs+++ b/datafusion/physical-plan/src/display.rs@@ -1511,7 +1511,10 @@ mod tests {use datafusion_physical_expr::PhysicalExpr;use crate::statistics::StatisticsArgs;- use crate::{DisplayAs, ExecutionPlan, PlanProperties};+ use crate::{+ ChildrenPropertiesMode, DisplayAs, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions,+ };use super::DisplayableExecutionPlan;@@ -1545,9 +1548,10 @@ mod tests {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}@@ -1559,6 +1563,16 @@ mod tests {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_: usize,@@ -1636,10 +1650,11 @@ mod tests {use crate::empty::EmptyExec;use crate::filter::FilterExec;use crate::projection::ProjectionExec;+ use crate::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions};use datafusion_physical_expr::expressions::{binary, col, lit};use datafusion_physical_expr::{Partitioning, PhysicalExpr};- fn sample_plan() -> Arc<dyn crate::ExecutionPlan> {+ fn sample_plan() -> Arc<dyn ExecutionPlan> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false),Field::new("b", DataType::Int32, false),@@ -1727,12 +1742,23 @@ mod tests {{Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)}- fn with_new_children(++ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn execute(&self,_: usize,diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rsindex bd91ec742d..dd08ff36a9 100644--- a/datafusion/physical-plan/src/empty.rs+++ b/datafusion/physical-plan/src/empty.rs@@ -20,7 +20,10 @@use std::sync::Arc;use crate::memory::MemoryStream;-use crate::{DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics};+use crate::{+ ChildrenPropertiesMode, DisplayAs, PlanProperties, ReplaceChildrenOptions,+ SendableRecordBatchStream, Statistics,+};use crate::{DisplayFormatType, ExecutionPlan, Partitioning,execution_plan::{Boundedness, EmissionType},@@ -127,13 +130,24 @@ impl ExecutionPlan for EmptyExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,@@ -247,8 +261,8 @@ impl EmptyExec {mod tests {use super::*;use crate::common;+ use crate::execution_plan::replace_children_if_necessary;use crate::test;- use crate::with_new_children_if_necessary;#[tokio::test]async fn empty() -> Result<()> {@@ -271,7 +285,7 @@ mod tests {let schema = test::aggr_test_schema();let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));- let empty2 = with_new_children_if_necessary(+ let empty2 = replace_children_if_necessary(Arc::clone(&empty) as Arc<dyn ExecutionPlan>,vec![],)?;@@ -279,7 +293,7 @@ mod tests {let too_many_kids = vec![empty2];assert!(- with_new_children_if_necessary(empty, too_many_kids).is_err(),+ replace_children_if_necessary(empty, too_many_kids).is_err(),"expected error when providing list of kids");Ok(())diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex ff803746aa..a4d081b3d9 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -267,6 +267,26 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// joins).fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>>;+ /// Returns a clone of the existing plan with the children replaced,+ /// skipping recomputation of plan properties when the options indicate+ /// the new children's properties are unchanged.+ ///+ /// Callers should typically call [`replace_children_if_necessary`] and+ /// not invoke this method directly.+ fn replace_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ #[expect(deprecated)]+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ self.with_new_children_and_same_properties(children)+ }+ ChildrenPropertiesMode::Recompute => self.with_new_children(children),+ }+ }+/// Apply a closure `f` to each root expression that this node owns and uses/// during execution, either by evaluating it or updating it dynamically.///@@ -331,27 +351,98 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,) -> Result<TreeNodeRecursion>;- /// Returns a new `ExecutionPlan` where all existing children were replaced- /// by the `children`, in order+ /// Deprecated.+ ///+ /// DataFusion will remove this method in the future in favor of+ /// [`ExecutionPlan::replace_children`].+ ///+ /// Note that this method is still required by the trait; implementations+ /// should delegate to [`ExecutionPlan::replace_children`] with+ /// [`ChildrenPropertiesMode::Recompute`].+ ///+ /// # Example Implementation+ /// ```+ /// # #![allow(deprecated)]+ /// # use std::fmt;+ /// # use std::sync::Arc;+ /// # use datafusion_common::Result;+ /// # use datafusion_common::tree_node::TreeNodeRecursion;+ /// # use datafusion_execution::{SendableRecordBatchStream, TaskContext};+ /// # use datafusion_physical_expr::PhysicalExpr;+ /// # use datafusion_physical_plan::{+ /// # ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan,+ /// # PlanProperties, ReplaceChildrenOptions,+ /// # };+ /// # #[derive(Debug)]+ /// # struct MyExec {+ /// # input: Arc<dyn ExecutionPlan>,+ /// # }+ /// # impl DisplayAs for MyExec {+ /// # fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {+ /// # write!(f, "MyExec")+ /// # }+ /// # }+ /// impl ExecutionPlan for MyExec {+ /// // ...+ /// # fn name(&self) -> &'static str {+ /// # "MyExec"+ /// # }+ /// # fn properties(&self) -> &Arc<PlanProperties> {+ /// # self.input.properties()+ /// # }+ /// # fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ /// # vec![&self.input]+ /// # }+ /// # fn apply_expressions(+ /// # &self,+ /// # _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ /// # ) -> Result<TreeNodeRecursion> {+ /// # Ok(TreeNodeRecursion::Continue)+ /// # }+ /// # fn execute(+ /// # &self,+ /// # _partition: usize,+ /// # _context: Arc<TaskContext>,+ /// # ) -> Result<SendableRecordBatchStream> {+ /// # unimplemented!()+ /// # }+ /// fn replace_children(+ /// self: Arc<Self>,+ /// mut children: Vec<Arc<dyn ExecutionPlan>>,+ /// _options: ReplaceChildrenOptions,+ /// ) -> Result<Arc<dyn ExecutionPlan>> {+ /// Ok(Arc::new(MyExec {+ /// input: children.swap_remove(0),+ /// }))+ /// }+ ///+ /// fn with_new_children(+ /// self: Arc<Self>,+ /// children: Vec<Arc<dyn ExecutionPlan>>,+ /// ) -> Result<Arc<dyn ExecutionPlan>> {+ /// // call into `replace_children` with `ReplaceChildrenOptions`+ /// self.replace_children(+ /// children,+ /// ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ /// )+ /// }+ /// }+ /// ```+ #[deprecated(+ since = "55.0.0",+ note = "Use `ExecutionPlan::replace_children` with `ReplaceChildrenOptions`"+ )]fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>>;- /// Fast-path used by [`with_new_children_if_necessary`] when the new- /// `children` are known to have the same [`PlanProperties`] as the current- /// children. Implementations should swap the children in without- /// recomputing this plan's `PlanProperties` (typically by cloning `self`- /// and replacing the child pointers).- ///- /// The default implementation falls back to- /// [`ExecutionPlan::with_new_children`] which is always correct but- /// forfeits the fast-path: implementations that own an expensive- /// `PlanProperties` (e.g. projection mapping, complex equivalence- /// classes) should override this method.- ///- /// Callers should route through [`with_new_children_if_necessary`] and- /// not invoke this method directly.+ /// Deprecated. Implement [`ExecutionPlan::replace_children`] instead.+ #[deprecated(+ since = "55.0.0",+ note = "Use `ExecutionPlan::replace_children` with `ReplaceChildrenOptions`"+ )]+ #[expect(deprecated)]fn with_new_children_and_same_properties(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -362,11 +453,11 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// Reset any internal state within this [`ExecutionPlan`].////// This method is called when an [`ExecutionPlan`] needs to be re-executed,- /// such as in recursive queries. Unlike [`ExecutionPlan::with_new_children`], this method+ /// such as in recursive queries. Unlike [`ExecutionPlan::replace_children`], this method/// ensures that any stateful components (e.g., [`DynamicFilterPhysicalExpr`])/// are reset to their initial state.///- /// The default implementation simply calls [`ExecutionPlan::with_new_children`] with the existing children,+ /// The default implementation simply calls [`ExecutionPlan::replace_children`] with the existing children,/// effectively creating a new instance of the [`ExecutionPlan`] with the same children but without/// necessarily resetting any internal state. Implementations that require resetting of some/// internal state should override this method to provide the necessary logic.@@ -375,13 +466,16 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// it will be called from within a walk of the execution plan tree so that it will be called on each child later/// or was already called on each child.///- /// Note to implementers: unlike [`ExecutionPlan::with_new_children`] this method does not accept new children as an argument,+ /// Note to implementers: unlike [`ExecutionPlan::replace_children`] this method does not accept new children as an argument,/// thus it is expected that any cached plan properties will remain valid after the reset.////// [`DynamicFilterPhysicalExpr`]: datafusion_physical_expr::expressions::DynamicFilterPhysicalExprfn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {let children = self.children().into_iter().cloned().collect();- self.with_new_children(children)+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}/// If supported, attempt to increase the partitioning of this `ExecutionPlan` to@@ -390,7 +484,7 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// If the `ExecutionPlan` does not support changing its partitioning,/// returns `Ok(None)` (the default).///- /// It is the `ExecutionPlan` can increase its partitioning, but not to the+ /// If the `ExecutionPlan` can increase its partitioning, but not to/// `target_partitions`, it may return an ExecutionPlan with fewer/// partitions. This might happen, for example, if each new partition would/// be too small to be efficiently processed individually.@@ -511,7 +605,7 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// partition: usize,/// context: Arc<TaskContext>,/// ) -> Result<SendableRecordBatchStream> {- /// // use functions from futures crate convert the batch into a stream+ /// // use functions from futures crate to convert the batch into a stream/// let fut = futures::future::ready(Ok(self.batch.clone()));/// let stream = futures::stream::once(fut);/// Ok(Box::pin(RecordBatchStreamAdapter::new(@@ -743,7 +837,7 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// up the plan that `DataSourceExec` can actually bind the filters.////// The default implementation bars all parent filters from being pushed down and adds no new filters.- /// This is the safest option, making filter pushdown opt-in on a per-node pasis.+ /// This is the safest option, making filter pushdown opt-in on a per-node basis.////// There are two different phases in filter pushdown, which some operators may handle the same and some differently./// Depending on the phase the operator may or may not be allowed to modify the plan.@@ -936,6 +1030,36 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {}}+/// Options for [`ExecutionPlan::replace_children`]+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub struct ReplaceChildrenOptions {+ /// Describes how plan properties should be handled for the replacement+ /// children.+ pub children_properties: ChildrenPropertiesMode,+}++impl ReplaceChildrenOptions {+ /// Create new options for [`ExecutionPlan::replace_children`].+ pub const fn new(children_properties: ChildrenPropertiesMode) -> Self {+ Self {+ children_properties,+ }+ }+}++/// Indicates whether the plan properties of the new children must be recomputed.+///+/// Part of [`ReplaceChildrenOptions`].+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum ChildrenPropertiesMode {+ /// The plan properties of the new children are identical to the properties+ /// of the existing children, so we can skip recomputation.+ Keep,+ /// The plan properties of the new children are different from the properties+ /// of the existing children, so we must recompute the properties from scratch.+ Recompute,+}+/// Allows a type to be treated as a reference to an/// [`Arc<dyn PhysicalExpr>`].///@@ -1354,12 +1478,10 @@ pub(crate) fn emission_type_from_children<'a>(}}-/// Stores certain, often expensive to compute, plan properties used in query-/// optimization.+/// Stores plan properties used in query optimization.///-/// These properties are stored a single structure to permit this information to-/// be computed once and then those cached results used multiple times without-/// recomputation (aka a cache)+/// Serves as a cache for these properties, which are often+/// expensive to compute.#[derive(Debug, Clone)]pub struct PlanProperties {/// See [ExecutionPlanProperties::equivalence_properties]@@ -1563,19 +1685,19 @@ pub fn need_data_exchange(plan: Arc<dyn ExecutionPlan>) -> bool {////// 1. **Same child pointers** — if every `children[i]` is `Arc::ptr_eq` to the/// corresponding existing child, the original `plan` is returned-/// unchanged (no allocation, no [`ExecutionPlan::with_new_children`]+/// unchanged (no allocation, no [`ExecutionPlan::replace_children`]/// call)./// 2. **Same child properties** — if the children's `PlanProperties` Arcs/// match (via [`has_same_children_properties`]), the plan's own/// `PlanProperties` cache can be reused. This calls-/// [`ExecutionPlan::with_new_children_and_same_properties`], which-/// swaps the child pointers without recomputing `PlanProperties`.+/// [`ExecutionPlan::replace_children`] with [`ChildrenPropertiesMode::Keep`],+/// which swaps the child pointers without recomputing `PlanProperties`./// 3. **Full recompute** — otherwise, delegate to-/// [`ExecutionPlan::with_new_children`], which recomputes-/// `PlanProperties` from scratch.+/// [`ExecutionPlan::replace_children`] with [`ChildrenPropertiesMode::Recompute`],+/// which recomputes `PlanProperties` from scratch.////// The size of `children` must be equal to the size of `ExecutionPlan::children()`.-pub fn with_new_children_if_necessary(+pub fn replace_children_if_necessary(plan: Arc<dyn ExecutionPlan>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {@@ -1596,11 +1718,25 @@ pub fn with_new_children_if_necessary(}// Layer 2: same child properties → reuse `PlanProperties` cache.if has_same_children_properties(plan.as_ref(), &children)? {- return plan.with_new_children_and_same_properties(children);+ return plan.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ );}}// Layer 3: full recompute.- plan.with_new_children(children)+ plan.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+}++#[deprecated(since = "55.0.0", note = "Use `replace_children_if_necessary`")]+pub fn with_new_children_if_necessary(+ plan: Arc<dyn ExecutionPlan>,+ children: Vec<Arc<dyn ExecutionPlan>>,+) -> Result<Arc<dyn ExecutionPlan>> {+ replace_children_if_necessary(plan, children)}/// Return a [`DisplayableExecutionPlan`] wrapper around an@@ -1871,9 +2007,9 @@ pub fn has_same_children_properties(/// the same as plan already has. Could be used to implement fast-path for method/// [`ExecutionPlan::with_new_children`].///-/// New call sites should route through [`with_new_children_if_necessary`],+/// New call sites should route through [`replace_children_if_necessary`],/// which applies this check together with the child-pointer short-circuit-/// (see [`with_new_children_if_necessary`] for the layered policy). This+/// (see [`replace_children_if_necessary`] for the layered policy). This/// macro remains for direct-caller sites that have not been migrated yet.#[macro_export]macro_rules! check_if_same_properties {@@ -1888,6 +2024,22 @@ macro_rules! check_if_same_properties {};}+/// Helper macro to validate that replacement children match a plan's existing+/// child count.+///+/// This is useful for [`ExecutionPlan::replace_children`] implementations that+/// need to preserve the same child-count validation behavior.+#[macro_export]+macro_rules! validate_child_count {+ ($plan: expr, $children: expr) => {+ datafusion_common::assert_eq_or_internal_err!(+ $children.len(),+ $plan.children().len(),+ "Wrong number of children"+ );+ };+}+/// Utility function yielding a string representation of the given [`ExecutionPlan`].pub fn get_plan_string(plan: &Arc<dyn ExecutionPlan>) -> Vec<String> {let formatted = displayable(plan.as_ref()).indent(true).to_string();@@ -1979,9 +2131,10 @@ mod tests {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}@@ -1993,6 +2146,16 @@ mod tests {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {self.dynamic_expressions.iter().map(Arc::clone).collect()}@@ -2086,13 +2249,24 @@ mod tests {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -2143,13 +2317,24 @@ mod tests {self.0.apply_expressions(f)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {Some(self.0.as_ref())}@@ -2208,12 +2393,23 @@ mod tests {) -> Result<TreeNodeRecursion> {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(++ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn execute(&self,_partition: usize,@@ -2277,38 +2473,59 @@ mod tests {) -> Result<TreeNodeRecursion> {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(++ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- self.recompute_calls- .fetch_add(1, std::sync::atomic::Ordering::SeqCst);- // Full recompute: allocate a fresh `PlanProperties` Arc so this- // path is observable via `Arc::ptr_eq` on properties.- let new_input = children.swap_remove(0);- let cache = Arc::new(PlanProperties::new(- EquivalenceProperties::new(Arc::new(Schema::empty())),- Partitioning::UnknownPartitioning(1),- EmissionType::Final,- Boundedness::Bounded,- ));- Ok(Arc::new(Self {- input: new_input,- cache,- recompute_calls: Arc::clone(&self.recompute_calls),- fast_path_calls: Arc::clone(&self.fast_path_calls),- }))+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ self.fast_path_calls+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ Ok(Arc::new(Self {+ input: children.swap_remove(0),+ ..Self::clone(&*self)+ }))+ }+ ChildrenPropertiesMode::Recompute => {+ self.recompute_calls+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ // Full recompute: allocate a fresh `PlanProperties` Arc so this+ // path is observable via `Arc::ptr_eq` on properties.+ let new_input = children.swap_remove(0);+ let cache = Arc::new(PlanProperties::new(+ EquivalenceProperties::new(Arc::new(Schema::empty())),+ Partitioning::UnknownPartitioning(1),+ EmissionType::Final,+ Boundedness::Bounded,+ ));+ Ok(Arc::new(Self {+ input: new_input,+ cache,+ recompute_calls: Arc::clone(&self.recompute_calls),+ fast_path_calls: Arc::clone(&self.fast_path_calls),+ }))+ }+ }+ }+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- self.fast_path_calls- .fetch_add(1, std::sync::atomic::Ordering::SeqCst);- Ok(Arc::new(Self {- input: children.swap_remove(0),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(&self,@@ -2406,9 +2623,9 @@ mod tests {}/// Cover the three short-circuit layers of- /// [`with_new_children_if_necessary`].+ /// [`replace_children_if_necessary`].#[test]- fn test_with_new_children_if_necessary_layers() -> Result<()> {+ fn test_replace_children_if_necessary_layers() -> Result<()> {use std::sync::atomic::Ordering;// Two leaves that share the same `PlanProperties` Arc but sit behind@@ -2438,7 +2655,7 @@ mod tests {let orig_props = Arc::clone(parent.properties());// Layer 1: same child pointer → returns the original plan Arc verbatim.- let out = with_new_children_if_necessary(+ let out = replace_children_if_necessary(Arc::clone(&parent_dyn),vec![Arc::clone(&leaf_a)],)?;@@ -2451,7 +2668,7 @@ mod tests {// Arc is reused (not reallocated).assert!(!Arc::ptr_eq(&leaf_a, &leaf_b));assert!(Arc::ptr_eq(leaf_a.properties(), leaf_b.properties()));- let out = with_new_children_if_necessary(+ let out = replace_children_if_necessary(Arc::clone(&parent_dyn),vec![Arc::clone(&leaf_b)],)?;@@ -2461,7 +2678,7 @@ mod tests {// Layer 3: child's `PlanProperties` Arc differs → full recompute.assert!(!Arc::ptr_eq(leaf_a.properties(), leaf_c.properties()));- let out = with_new_children_if_necessary(+ let out = replace_children_if_necessary(Arc::clone(&parent_dyn),vec![Arc::clone(&leaf_c)],)?;@@ -2479,7 +2696,7 @@ mod tests {/// `with_new_children`, so downstream / external `ExecutionPlan`/// implementations keep the semantics-preserving path.#[test]- fn test_with_new_children_if_necessary_default_fallback() -> Result<()> {+ fn test_replace_children_if_necessary_default_fallback() -> Result<()> {use std::sync::atomic::Ordering;let leaf_props = Arc::new(PlanProperties::new(@@ -2498,10 +2715,20 @@ mod tests {let parent = Arc::new(WithChildrenTestParentDefault::new(Arc::clone(&leaf_a)));let parent_dyn: Arc<dyn ExecutionPlan> = Arc::clone(&parent) as _;- // Distinct child Arc but same `PlanProperties` Arc — the helper- // enters the "same properties" branch and calls the trait method,- // whose default forwards to `with_new_children`.- let out = with_new_children_if_necessary(+ // Using the same child means we return the original plan Arc verbatim, so even when+ // the `replace_children` `ChildrenPropertiesMode::Keep` path is not defined,+ // we do not recompute.+ let out = replace_children_if_necessary(+ Arc::clone(&parent_dyn),+ vec![Arc::clone(&leaf_a)],+ )?;+ assert!(Arc::ptr_eq(&out, &parent_dyn));+ assert_eq!(parent.recompute_calls.load(Ordering::SeqCst), 0);++ // Using a distinct child but the same `PlanProperties` Arc means the helper+ // attempts to enter the Keep branch. If it does not exist, we fall back+ // to recomputation.+ let out = replace_children_if_necessary(Arc::clone(&parent_dyn),vec![Arc::clone(&leaf_b)],)?;diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rsindex 72ceae81f3..3b31ee748b 100644--- a/datafusion/physical-plan/src/explain.rs+++ b/datafusion/physical-plan/src/explain.rs@@ -22,7 +22,10 @@ use std::sync::Arc;use super::{DisplayAs, PlanProperties, SendableRecordBatchStream};use crate::execution_plan::{Boundedness, EmissionType};use crate::stream::RecordBatchStreamAdapter;-use crate::{DisplayFormatType, ExecutionPlan, Partitioning};+use crate::{+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, Partitioning,+ ReplaceChildrenOptions,+};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::display::StringifiedPlan;@@ -124,13 +127,24 @@ impl ExecutionPlan for ExplainExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 65abfa259d..5df5482fb7 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -28,10 +28,9 @@ use super::{ColumnStatistics, DisplayAs, ExecutionPlanProperties, PlanProperties,RecordBatchStream, SendableRecordBatchStream, Statistics,};-use crate::check_if_same_properties;use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus};use crate::common::can_project;-use crate::execution_plan::CardinalityEffect;+use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation, PushedDown,@@ -44,6 +43,7 @@ use crate::projection::{};use crate::statistics::{ChildStats, StatisticsArgs, StatisticsContext};use crate::stream::EmptyRecordBatchStream;+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};use crate::{DisplayFormatType, ExecutionPlan,metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RatioMetrics},@@ -552,27 +552,46 @@ impl ExecutionPlan for FilterExec {vec![true]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let new_input = children.swap_remove(0);- FilterExecBuilder::from(&*self)- .with_input(new_input)- .build()- .map(|e| Arc::new(e) as _)+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let new_input = children.swap_remove(0);+ FilterExecBuilder::from(&*self)+ .with_input(new_input)+ .build()+ .map(|e| Arc::new(e) as _)+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -820,8 +839,7 @@ impl ExecutionPlan for FilterExec {self.input.with_preserve_order(preserve_order).and_then(|new_input| {- Arc::new(self.clone())- .with_new_children(vec![new_input])+ replace_children_if_necessary(Arc::new(self.clone()), vec![new_input]).ok()})}diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 524887bf03..8a477c1021 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -34,9 +34,10 @@ use crate::projection::{use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::EmptyRecordBatchStream;use crate::{- ColumnStatistics, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,- ExecutionPlanProperties, PlanProperties, RecordBatchStream,- SendableRecordBatchStream, Statistics, check_if_same_properties, handle_state,+ ChildrenPropertiesMode, ColumnStatistics, DisplayAs, DisplayFormatType, Distribution,+ ExecutionPlan, ExecutionPlanProperties, PlanProperties, RecordBatchStream,+ ReplaceChildrenOptions, SendableRecordBatchStream, Statistics, handle_state,+ validate_child_count,};use arrow::array::{RecordBatch, RecordBatchOptions};@@ -276,32 +277,51 @@ impl ExecutionPlan for CrossJoinExec {Ok(TreeNodeRecursion::Continue)}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ let left = children.swap_remove(0);+ let right = children.swap_remove(0);++ Ok(Arc::new(Self {+ left,+ right,+ metrics: ExecutionPlanMetricsSet::new(),+ left_fut: Default::default(),+ cache: Arc::clone(&self.cache),+ schema: Arc::clone(&self.schema),+ }))+ }+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(CrossJoinExec::new(+ Arc::clone(&children[0]),+ Arc::clone(&children[1]),+ ))),+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(CrossJoinExec::new(- Arc::clone(&children[0]),- Arc::clone(&children[1]),- )))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- let left = children.swap_remove(0);- let right = children.swap_remove(0);-- Ok(Arc::new(Self {- left,- right,- metrics: ExecutionPlanMetricsSet::new(),- left_fut: Default::default(),- cache: Arc::clone(&self.cache),- schema: Arc::clone(&self.schema),- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex cd9048b58c..08d209003a 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -22,7 +22,6 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};use std::sync::{Arc, OnceLock};use std::vec;-use crate::ExecutionPlanProperties;use crate::execution_plan::{EmissionType, boundedness_from_children, has_same_children_properties,plan_contains_expression_id, stub_properties,@@ -53,6 +52,10 @@ use crate::projection::{};use crate::repartition::REPARTITION_RANDOM_STATE;use crate::statistics::{ChildStats, StatisticsArgs};+use crate::{+ ChildrenPropertiesMode, ExecutionPlanProperties, ReplaceChildrenOptions,+ validate_child_count,+};use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,InputDistributionRequirements, Partitioning, PlanProperties,@@ -1375,11 +1378,32 @@ impl ExecutionPlan for HashJoinExec {/// This method is called during query optimization when the optimizer creates new/// plan nodes. Importantly, it creates a fresh bounds_accumulator via `try_new`/// rather than cloning the existing one because partitioning may have changed.+ fn replace_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ self.builder().with_new_children(children)?.build_exec()+ }+ ChildrenPropertiesMode::Recompute => self+ .builder()+ .recompute_properties()+ .with_new_children(children)?+ .build_exec(),+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- self.builder().with_new_children(children)?.build_exec()+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {@@ -2451,6 +2475,7 @@ mod tests {use crate::filter::FilterExecBuilder;use crate::joins::hash_join::stream::lookup_join_hashmap;use crate::test::{TestMemoryExec, assert_join_metrics};+ use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions};use crate::{common, expressions::Column, repartition::RepartitionExec, test::build_table_i32,test::exec::MockExec,@@ -2523,13 +2548,24 @@ mod tests {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 7069a8b448..eb1df638c7 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -44,9 +44,9 @@ use crate::projection::{};use crate::statistics::{ChildStats, StatisticsArgs};use crate::{- DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- PlanProperties, RecordBatchStream, SendableRecordBatchStream,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,+ ExecutionPlanProperties, PlanProperties, RecordBatchStream, ReplaceChildrenOptions,+ SendableRecordBatchStream, validate_child_count,};use arrow::array::{@@ -574,43 +574,61 @@ impl ExecutionPlan for NestedLoopJoinExec {)}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ let left = children.swap_remove(0);+ let right = children.swap_remove(0);+ Ok(Arc::new(Self {+ left,+ right,+ metrics: ExecutionPlanMetricsSet::new(),+ build_side_data: Default::default(),+ left_spill_data: Arc::new(OnceAsync::default()),+ cache: Arc::clone(&self.cache),+ filter: self.filter.clone(),+ join_type: self.join_type,+ join_schema: Arc::clone(&self.join_schema),+ column_indices: self.column_indices.clone(),+ projection: self.projection.clone(),+ }))+ }+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(+ NestedLoopJoinExecBuilder::new(+ Arc::clone(&children[0]),+ Arc::clone(&children[1]),+ self.join_type,+ )+ .with_filter(self.filter.clone())+ .with_projection_ref(self.projection.clone())+ .build()?,+ )),+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(- NestedLoopJoinExecBuilder::new(- Arc::clone(&children[0]),- Arc::clone(&children[1]),- self.join_type,- )- .with_filter(self.filter.clone())- .with_projection_ref(self.projection.clone())- .build()?,- ))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- let left = children.swap_remove(0);- let right = children.swap_remove(0);-- Ok(Arc::new(Self {- left,- right,- metrics: ExecutionPlanMetricsSet::new(),- build_side_data: Default::default(),- left_spill_data: Arc::new(OnceAsync::default()),- cache: Arc::clone(&self.cache),- filter: self.filter.clone(),- join_type: self.join_type,- join_schema: Arc::clone(&self.join_schema),- column_indices: self.column_indices.clone(),- projection: self.projection.clone(),- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rsindex b60ec1c784..c42ec67ef8 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs@@ -53,7 +53,8 @@ use crate::joins::piecewise_merge_join::utils::{use crate::joins::utils::asymmetric_join_output_partitioning;use crate::metrics::MetricsSet;use crate::{- DisplayAs, DisplayFormatType, ExecutionPlanProperties, check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlanProperties,+ ReplaceChildrenOptions, validate_child_count,};use crate::{ExecutionPlan, PlanProperties,@@ -517,56 +518,80 @@ impl ExecutionPlan for PiecewiseMergeJoinExec {}}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ let buffered = children.swap_remove(0);+ let streamed = children.swap_remove(0);+ Ok(Arc::new(Self {+ buffered,+ streamed,+ on: self.on.clone(),+ operator: self.operator,+ join_type: self.join_type,+ schema: Arc::clone(&self.schema),+ left_child_plan_required_order: self+ .left_child_plan_required_order+ .clone(),+ right_batch_required_orders: self.right_batch_required_orders.clone(),+ sort_options: self.sort_options,+ cache: Arc::clone(&self.cache),+ num_partitions: self.num_partitions,++ // Re-set state.+ metrics: ExecutionPlanMetricsSet::new(),+ buffered_fut: Default::default(),+ }))+ }+ ChildrenPropertiesMode::Recompute => match &children[..] {+ [left, right] => Ok(Arc::new(PiecewiseMergeJoinExec::try_new(+ Arc::clone(left),+ Arc::clone(right),+ self.on.clone(),+ self.operator,+ self.join_type,+ self.num_partitions,+ )?)),+ _ => internal_err!(+ "PiecewiseMergeJoin should have 2 children, found {}",+ children.len()+ ),+ },+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- match &children[..] {- [left, right] => Ok(Arc::new(PiecewiseMergeJoinExec::try_new(- Arc::clone(left),- Arc::clone(right),- self.on.clone(),- self.operator,- self.join_type,- self.num_partitions,- )?)),- _ => internal_err!(- "PiecewiseMergeJoin should have 2 children, found {}",- children.len()- ),- }+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- let buffered = children.swap_remove(0);- let streamed = children.swap_remove(0);- Ok(Arc::new(Self {- buffered,- streamed,- on: self.on.clone(),- operator: self.operator,- join_type: self.join_type,- schema: Arc::clone(&self.schema),- left_child_plan_required_order: self.left_child_plan_required_order.clone(),- right_batch_required_orders: self.right_batch_required_orders.clone(),- sort_options: self.sort_options,- cache: Arc::clone(&self.cache),- num_partitions: self.num_partitions,-- // Re-set state.- metrics: ExecutionPlanMetricsSet::new(),- buffered_fut: Default::default(),- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {let buffered = Arc::clone(&self.buffered);let streamed = Arc::clone(&self.streamed);- self.with_new_children_and_same_properties(vec![buffered, streamed])+ self.replace_children(+ vec![buffered, streamed],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex 55a4b2136c..b48905500d 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -40,9 +40,9 @@ use crate::projection::{use crate::spill::spill_manager::SpillManager;use crate::statistics::{ChildStats, StatisticsArgs};use crate::{- DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- InputDistributionRequirements, PlanProperties, SendableRecordBatchStream, Statistics,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,+ ExecutionPlanProperties, InputDistributionRequirements, PlanProperties,+ ReplaceChildrenOptions, SendableRecordBatchStream, Statistics, validate_child_count,};use arrow::compute::SortOptions;@@ -449,37 +449,56 @@ impl ExecutionPlan for SortMergeJoinExec {crate::apply_expression_roots(join_keys.chain(filter), f)}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ let left = children.swap_remove(0);+ let right = children.swap_remove(0);+ Ok(Arc::new(Self {+ left,+ right,+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ }))+ }+ ChildrenPropertiesMode::Recompute => match &children[..] {+ [left, right] => Ok(Arc::new(SortMergeJoinExec::try_new(+ Arc::clone(left),+ Arc::clone(right),+ self.on.clone(),+ self.filter.clone(),+ self.join_type,+ self.sort_options.clone(),+ self.null_equality,+ )?)),+ _ => internal_err!("SortMergeJoin wrong number of children"),+ },+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- match &children[..] {- [left, right] => Ok(Arc::new(SortMergeJoinExec::try_new(- Arc::clone(left),- Arc::clone(right),- self.on.clone(),- self.filter.clone(),- self.join_type,- self.sort_options.clone(),- self.null_equality,- )?)),- _ => internal_err!("SortMergeJoin wrong number of children"),- }+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- let left = children.swap_remove(0);- let right = children.swap_remove(0);- Ok(Arc::new(Self {- left,- right,- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex 589042a828..0c6e84b36c 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -31,7 +31,6 @@ use std::sync::Arc;use std::task::{Context, Poll};use std::vec;-use crate::check_if_same_properties;use crate::common::SharedMemoryReservation;use crate::execution_plan::{boundedness_from_children, emission_type_from_children};use crate::joins::stream_join_utils::{@@ -50,6 +49,7 @@ use crate::projection::{JoinData, ProjectionExec, try_pushdown_through_join_with_column_indices,};use crate::stream::EmptyRecordBatchStream;+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,InputDistributionRequirements, PlanProperties, RecordBatchStream,@@ -462,36 +462,57 @@ impl ExecutionPlan for SymmetricHashJoinExec {crate::apply_expression_roots(join_keys.chain(filter), f)}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => {+ let left = children.swap_remove(0);+ let right = children.swap_remove(0);+ Ok(Arc::new(Self {+ left,+ right,+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ }))+ }+ ChildrenPropertiesMode::Recompute => {+ Ok(Arc::new(SymmetricHashJoinExec::try_new(+ Arc::clone(&children[0]),+ Arc::clone(&children[1]),+ self.on.clone(),+ self.filter.clone(),+ &self.join_type,+ self.null_equality,+ self.left_sort_exprs.clone(),+ self.right_sort_exprs.clone(),+ self.mode,+ )?))+ }+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(SymmetricHashJoinExec::try_new(- Arc::clone(&children[0]),- Arc::clone(&children[1]),- self.on.clone(),- self.filter.clone(),- &self.join_type,- self.null_equality,- self.left_sort_exprs.clone(),- self.right_sort_exprs.clone(),- self.mode,- )?))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- let left = children.swap_remove(0);- let right = children.swap_remove(0);- Ok(Arc::new(Self {- left,- right,- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn metrics(&self) -> Option<MetricsSet> {diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 941b4e561b..9e50a93b21 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -44,10 +44,12 @@ pub use crate::display::{DefaultDisplay, DisplayAs, DisplayFormatType, VerboseDipub use crate::distribution_requirements::{ChildSatisfactionOptions, InputDistributionRequirements,};+#[expect(deprecated)]pub use crate::execution_plan::{- AsPhysicalExprRef, ExecutionPlan, ExecutionPlanProperties, PlanProperties,- apply_expression_roots, collect, collect_partitioned, displayable,- execute_input_stream, execute_stream, execute_stream_partitioned, get_plan_string,+ AsPhysicalExprRef, ChildrenPropertiesMode, ExecutionPlan, ExecutionPlanProperties,+ PlanProperties, ReplaceChildrenOptions, apply_expression_roots, collect,+ collect_partitioned, displayable, execute_input_stream, execute_stream,+ execute_stream_partitioned, get_plan_string, replace_children_if_necessary,with_new_children_if_necessary,};pub use crate::metrics::Metric;diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex 9dbdf17dbc..dd62c93d1c 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -29,8 +29,8 @@ use super::{use crate::execution_plan::{Boundedness, CardinalityEffect};use crate::statistics::{ChildStats, StatisticsArgs};use crate::{- DisplayFormatType, Distribution, ExecutionPlan, Partitioning,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayFormatType, Distribution, ExecutionPlan, Partitioning,+ ReplaceChildrenOptions, validate_child_count,};use arrow::datatypes::SchemaRef;@@ -175,26 +175,45 @@ impl ExecutionPlan for GlobalLimitExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let mut new_limit =+ GlobalLimitExec::new(children.swap_remove(0), self.skip, self.fetch);+ new_limit.set_required_ordering(self.required_ordering.clone());+ Ok(Arc::new(new_limit))+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let mut new_limit =- GlobalLimitExec::new(children.swap_remove(0), self.skip, self.fetch);- new_limit.set_required_ordering(self.required_ordering.clone());- Ok(Arc::new(new_limit))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -421,25 +440,45 @@ impl ExecutionPlan for LocalLimitExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let mut new_limit = LocalLimitExec::new(children.swap_remove(0), self.fetch);- new_limit.set_required_ordering(self.required_ordering.clone());- Ok(Arc::new(new_limit))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let mut new_limit =+ LocalLimitExec::new(children.swap_remove(0), self.fetch);+ new_limit.set_required_ordering(self.required_ordering.clone());+ Ok(Arc::new(new_limit))+ }+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -873,7 +912,7 @@ mod tests {}#[test]- fn with_new_children_preserves_required_ordering() -> Result<()> {+ fn replace_children_preserves_required_ordering() -> Result<()> {let source = test::scan_partitioned(1);let schema = source.schema();let ordering = LexOrdering::new(vec![PhysicalSortExpr {@@ -886,15 +925,19 @@ mod tests {let mut global = GlobalLimitExec::new(Arc::clone(&source), 0, Some(10));global.set_required_ordering(ordering.clone());- let rebuilt =- Arc::new(global).with_new_children(vec![test::scan_partitioned(1)])?;+ let rebuilt = Arc::new(global).replace_children(+ vec![test::scan_partitioned(1)],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let rebuilt = rebuilt.downcast_ref::<GlobalLimitExec>().unwrap();assert_eq!(rebuilt.required_ordering(), &ordering);let mut local = LocalLimitExec::new(source, 10);local.set_required_ordering(ordering.clone());- let rebuilt =- Arc::new(local).with_new_children(vec![test::scan_partitioned(1)])?;+ let rebuilt = Arc::new(local).replace_children(+ vec![test::scan_partitioned(1)],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let rebuilt = rebuilt.downcast_ref::<LocalLimitExec>().unwrap();assert_eq!(rebuilt.required_ordering(), &ordering);diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex eb141b8c70..efe42c7ebc 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -26,8 +26,8 @@ use crate::coop::cooperative;use crate::execution_plan::{Boundedness, EmissionType, SchedulingType};use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,- RecordBatchStream, SendableRecordBatchStream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,+ PlanProperties, RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,};use arrow::array::RecordBatch;@@ -319,9 +319,10 @@ impl ExecutionPlan for LazyMemoryExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert_or_internal_err!(children.is_empty(),@@ -330,6 +331,16 @@ impl ExecutionPlan for LazyMemoryExec {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rsindex ec54201e7b..16b89e9eca 100644--- a/datafusion/physical-plan/src/operator_statistics/mod.rs+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs@@ -1027,7 +1027,10 @@ mod tests {use crate::filter::FilterExec;use crate::projection::ProjectionExec;use crate::statistics::StatisticsArgs;- use crate::{DisplayAs, DisplayFormatType, PlanProperties};+ use crate::{+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, PlanProperties,+ ReplaceChildrenOptions,+ };use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::stats::Precision;use datafusion_common::{ColumnStatistics, ScalarValue};@@ -1107,13 +1110,24 @@ mod tests {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn properties(&self) -> &Arc<PlanProperties> {&self.cache}@@ -1214,15 +1228,26 @@ mod tests {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(CustomExec {input: Arc::clone(&children[0]),}))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn properties(&self) -> &Arc<PlanProperties> {self.input.properties()}diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rsindex de07529bab..67c063b65c 100644--- a/datafusion/physical-plan/src/placeholder_row.rs+++ b/datafusion/physical-plan/src/placeholder_row.rs@@ -23,8 +23,9 @@ use crate::coop::cooperative;use crate::execution_plan::{Boundedness, EmissionType, SchedulingType};use crate::memory::MemoryStream;use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,- SendableRecordBatchStream, Statistics, common,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,+ PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream, Statistics,+ common,};use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};@@ -145,13 +146,24 @@ impl ExecutionPlan for PlaceholderRowExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,@@ -250,16 +262,15 @@ impl PlaceholderRowExec {#[cfg(test)]mod tests {use super::*;- use crate::test;- use crate::with_new_children_if_necessary;+ use crate::{execution_plan::replace_children_if_necessary, test};#[test]- fn with_new_children() -> Result<()> {+ fn replace_children() -> Result<()> {let schema = test::aggr_test_schema();let placeholder = Arc::new(PlaceholderRowExec::new(schema));- let placeholder_2 = with_new_children_if_necessary(+ let placeholder_2 = replace_children_if_necessary(Arc::clone(&placeholder) as Arc<dyn ExecutionPlan>,vec![],)?;@@ -267,7 +278,7 @@ mod tests {let too_many_kids = vec![placeholder_2];assert!(- with_new_children_if_necessary(placeholder, too_many_kids).is_err(),+ replace_children_if_necessary(placeholder, too_many_kids).is_err(),"expected error when providing list of kids");Ok(())diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex ecdd78cc2a..cf362cdee5 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -27,14 +27,17 @@ use super::{SendableRecordBatchStream, SortOrderPushdownResult, Statistics,};use crate::column_rewriter::PhysicalColumnRewriter;-use crate::execution_plan::CardinalityEffect;+use crate::execution_plan::{CardinalityEffect, replace_children_if_necessary};use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation, FilterRemapper, PushedDownPredicate,};use crate::joins::utils::{ColumnIndex, JoinFilter, JoinOn, JoinOnRef};use crate::statistics::{ChildStats, StatisticsArgs};-use crate::{DisplayFormatType, ExecutionPlan, PhysicalExpr, check_if_same_properties};+use crate::{+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, PhysicalExpr,+ ReplaceChildrenOptions, validate_child_count,+};use std::collections::HashMap;use std::pin::Pin;use std::sync::Arc;@@ -337,27 +340,44 @@ impl ExecutionPlan for ProjectionExec {crate::apply_expression_roots(self.projector.projection().as_ref().iter(), f)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- ProjectionExec::try_from_projector(- self.projector.clone(),- children.swap_remove(0),+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => ProjectionExec::try_from_projector(+ self.projector.clone(),+ children.swap_remove(0),+ )+ .map(|p| Arc::new(p) as _),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),)- .map(|p| Arc::new(p) as _)}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -510,11 +530,13 @@ impl ExecutionPlan for ProjectionExec {// Recursively push down to child nodematch child.try_pushdown_sort(&child_order)? {SortOrderPushdownResult::Exact { inner } => {- let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ let new_exec =+ replace_children_if_necessary(Arc::new(self.clone()), vec![inner])?;Ok(SortOrderPushdownResult::Exact { inner: new_exec })}SortOrderPushdownResult::Inexact { inner } => {- let new_exec = Arc::new(self.clone()).with_new_children(vec![inner])?;+ let new_exec =+ replace_children_if_necessary(Arc::new(self.clone()), vec![inner])?;Ok(SortOrderPushdownResult::Inexact { inner: new_exec })}SortOrderPushdownResult::Unsupported => {@@ -530,8 +552,7 @@ impl ExecutionPlan for ProjectionExec {self.input.with_preserve_order(preserve_order).and_then(|new_input| {- Arc::new(self.clone())- .with_new_children(vec![new_input])+ replace_children_if_necessary(Arc::new(self.clone()), vec![new_input]).ok()})}diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex 4c6f0493ad..0a56488de8 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -30,8 +30,8 @@ use crate::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet, RecordOutput,};use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream,- SendableRecordBatchStream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,};use arrow::array::{BooleanArray, BooleanBuilder};use arrow::compute::filter_record_batch;@@ -183,9 +183,10 @@ impl ExecutionPlan for RecursiveQueryExec {])}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {RecursiveQueryExec::try_new(self.name.clone(),@@ -197,6 +198,16 @@ impl ExecutionPlan for RecursiveQueryExec {.map(|e| Arc::new(e) as _)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 8e7bec5dee..063954a72a 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -15,7 +15,7 @@// specific language governing permissions and limitations// under the License.-//! This file implements the [`RepartitionExec`] operator, which maps N input+//! This file implements the [`RepartitionExec`] operator, which maps N input//! partitions to M output partitions based on a partitioning scheme, optionally//! maintaining the order of the input rows in the output.@@ -43,8 +43,8 @@ use crate::spill::spill_pool::{self, SpillPoolSink, SpillPoolWriter};use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::{EmptyRecordBatchStream, RecordBatchStreamAdapter};use crate::{- DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, Statistics,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayFormatType, ExecutionPlan, Partitioning,+ PlanProperties, ReplaceChildrenOptions, Statistics, validate_child_count,};use arrow::array::{Array, PrimitiveArray, RecordBatch, RecordBatchOptions, UInt64Array};@@ -1564,31 +1564,50 @@ impl ExecutionPlan for RepartitionExec {}}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let mut repartition = RepartitionExec::try_new(- children.swap_remove(0),- self.partitioning().clone(),- )?;- if self.preserve_order {- repartition = repartition.with_preserve_order();+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ state: Default::default(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let mut repartition = RepartitionExec::try_new(+ children.swap_remove(0),+ self.partitioning().clone(),+ )?;+ if self.preserve_order {+ repartition = repartition.with_preserve_order();+ }+ Ok(Arc::new(repartition))+ }}- Ok(Arc::new(repartition))+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- state: Default::default(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn benefits_from_input_partitioning(&self) -> Vec<bool> {@@ -3227,13 +3246,24 @@ mod tests {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsindex ee3c2e5d07..f2b7c5e0b5 100644--- a/datafusion/physical-plan/src/scalar_subquery.rs+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -37,7 +37,10 @@ use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};use crate::joins::utils::{OnceAsync, OnceFut};use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::RecordBatchStreamAdapter;-use crate::{DisplayAs, DisplayFormatType, SendableRecordBatchStream};+use crate::{+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ReplaceChildrenOptions,+ SendableRecordBatchStream,+};use futures::StreamExt;use futures::TryStreamExt;@@ -164,9 +167,10 @@ impl ExecutionPlan for ScalarSubqueryExec {children}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {// First child is the main input, the rest are subquery plans.let input = children.remove(0);@@ -186,6 +190,16 @@ impl ExecutionPlan for ScalarSubqueryExec {)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {self.results.clear();Ok(Arc::new(ScalarSubqueryExec {@@ -452,9 +466,10 @@ mod tests {vec![&self.inner]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::new(Self::new(children.remove(0),@@ -469,6 +484,16 @@ mod tests {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rsindex 5f15f8b6cb..478ac14e11 100644--- a/datafusion/physical-plan/src/sorts/partial_sort.rs+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs@@ -61,9 +61,9 @@ use crate::sorts::sort::sort_batch;use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::EmptyRecordBatchStream;use crate::{- DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- Partitioning, PlanProperties, SendableRecordBatchStream, Statistics,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,+ ExecutionPlanProperties, Partitioning, PlanProperties, ReplaceChildrenOptions,+ SendableRecordBatchStream, Statistics, validate_child_count,};use arrow::compute::concat_batches;@@ -385,31 +385,50 @@ impl ExecutionPlan for PartialSortExec {)}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics_set: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let new_partial_sort = PartialSortExec::new(+ self.expr.clone(),+ Arc::clone(&children[0]),+ self.common_prefix_length,+ )+ .with_fetch(self.fetch)+ .with_preserve_partitioning(self.preserve_partitioning);++ Ok(Arc::new(new_partial_sort))+ }+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let new_partial_sort = PartialSortExec::new(- self.expr.clone(),- Arc::clone(&children[0]),- self.common_prefix_length,+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),)- .with_fetch(self.fetch)- .with_preserve_partitioning(self.preserve_partitioning);-- Ok(Arc::new(new_partial_sort))}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics_set: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex 78dd9b9696..41ccfab683 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -48,6 +48,7 @@ use futures::TryStreamExt;use crate::execution_plan::{Boundedness, EmissionType};use crate::metrics::ExecutionPlanMetricsSet;use crate::topk::{PartitionedTopK, PartitionedTopKRank, build_sort_fields};+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions};use crate::{DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,PlanProperties, SendableRecordBatchStream, stream::RecordBatchStreamAdapter,@@ -366,9 +367,10 @@ impl ExecutionPlan for PartitionedTopKExec {vec![&self.input]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert_eq!(children.len(), 1);Ok(Arc::new(PartitionedTopKExec::try_new(@@ -390,6 +392,16 @@ impl ExecutionPlan for PartitionedTopKExec {)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 5c6b86acc5..6c782f5134 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -28,6 +28,7 @@ use parking_lot::RwLock;use crate::common::spawn_buffered;use crate::execution_plan::{Boundedness, CardinalityEffect, EmissionType, has_same_children_properties,+ replace_children_if_necessary,};use crate::expressions::PhysicalSortExpr;use crate::filter::FilterExec;@@ -51,9 +52,9 @@ use crate::stream::{ObservedStream, RecordBatchStreamAdapter};use crate::topk::TopK;use crate::topk::TopKDynamicFilters;use crate::{- DisplayAs, DisplayFormatType, Distribution, EmptyRecordBatchStream, ExecutionPlan,- ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream,- Statistics,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Distribution,+ EmptyRecordBatchStream, ExecutionPlan, ExecutionPlanProperties, Partitioning,+ PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream, Statistics,};use arrow::array::{RecordBatch, RecordBatchOptions};@@ -1307,16 +1308,17 @@ impl ExecutionPlan for SortExec {vec![false]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {let mut new_sort = self.cloned();assert_eq!(children.len(), 1, "SortExec should have exactly one child");new_sort.input = Arc::clone(&children[0]);- if !has_same_children_properties(self.as_ref(), &children)? {- // Recompute the properties based on the new input since they may have changed+ if options.children_properties == ChildrenPropertiesMode::Recompute {+ // Recompute the properties based on the new input since they may have changed.let (cache, sort_prefix) = Self::compute_properties(&new_sort.input,new_sort.expr.clone(),@@ -1332,12 +1334,28 @@ impl ExecutionPlan for SortExec {Ok(Arc::new(new_sort))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ match has_same_children_properties(self.as_ref(), &children)? {+ true => self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ ),+ false => self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ ),+ }+ }+fn reset_state(self: Arc<Self>) -> Result<Arc<dyn ExecutionPlan>> {let children = self.children().into_iter().cloned().collect();- let new_sort = self.with_new_children(children)?;+ let new_sort = replace_children_if_necessary(self, children)?;let mut new_sort = new_sort.downcast_ref::<SortExec>()- .expect("cloned 1 lines above this line, we know the type")+ .expect("rebuilt SortExec with new children").clone();// Our dynamic filter and execution metrics are the state we need to reset.new_sort.filter = Some(new_sort.create_filter());@@ -1781,9 +1799,10 @@ mod tests {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}@@ -1795,6 +1814,16 @@ mod tests {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex ac6f5d18cd..ad17f2c213 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -26,9 +26,9 @@ use crate::projection::{ProjectionExec, make_with_child, update_ordering};use crate::sorts::streaming_merge::StreamingMergeBuilder;use crate::statistics::{ChildStats, StatisticsArgs};use crate::{- DisplayAs, DisplayFormatType, Distribution, ExecutionPlan, ExecutionPlanProperties,- Partitioning, PlanProperties, SendableRecordBatchStream, Statistics,- check_if_same_properties,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,+ ExecutionPlanProperties, Partitioning, PlanProperties, ReplaceChildrenOptions,+ SendableRecordBatchStream, Statistics, validate_child_count,};use datafusion_common::tree_node::TreeNodeRecursion;@@ -38,7 +38,9 @@ use datafusion_execution::memory_pool::MemoryConsumer;use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};-use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};+use crate::execution_plan::{+ CardinalityEffect, EvaluationType, SchedulingType, replace_children_if_necessary,+};use log::{debug, trace};/// Sort preserving merge execution plan@@ -251,8 +253,7 @@ impl ExecutionPlan for SortPreservingMergeExec {self.input.with_preserve_order(preserve_order).and_then(|new_input| {- Arc::new(self.clone())- .with_new_children(vec![new_input])+ replace_children_if_necessary(Arc::new(self.clone()), vec![new_input]).ok()})}@@ -293,26 +294,43 @@ impl ExecutionPlan for SortPreservingMergeExec {)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(- SortPreservingMergeExec::new(self.expr.clone(), children.swap_remove(0))- .with_fetch(self.fetch),- ))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(+ SortPreservingMergeExec::new(self.expr.clone(), children.swap_remove(0))+ .with_fetch(self.fetch),+ )),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -1591,12 +1609,23 @@ mod tests {) -> Result<TreeNodeRecursion> {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(++ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rsindex a82f8d9441..7b0058e798 100644--- a/datafusion/physical-plan/src/streaming.rs+++ b/datafusion/physical-plan/src/streaming.rs@@ -30,7 +30,10 @@ use crate::projection::{ProjectionExec, all_alias_free_columns, new_projections_for_columns, update_ordering,};use crate::stream::RecordBatchStreamAdapter;-use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};+use crate::{+ ChildrenPropertiesMode, ExecutionPlan, Partitioning, ReplaceChildrenOptions,+ SendableRecordBatchStream,+};use arrow::datatypes::{Schema, SchemaRef};use datafusion_common::tree_node::TreeNodeRecursion;@@ -280,9 +283,10 @@ impl ExecutionPlan for StreamingTableExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {if children.is_empty() {Ok(self)@@ -291,6 +295,16 @@ impl ExecutionPlan for StreamingTableExec {}}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex 68e6ff7eca..b38a46d160 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -24,7 +24,6 @@ use std::pin::Pin;use std::sync::Arc;use std::task::Context;-use crate::ExecutionPlan;use crate::common;use crate::execution_plan::{Boundedness, EmissionType};use crate::memory::MemoryStream;@@ -32,6 +31,7 @@ use crate::metrics::MetricsSet;use crate::statistics::StatisticsArgs;use crate::stream::RecordBatchStreamAdapter;use crate::streaming::PartitionStream;+use crate::{ChildrenPropertiesMode, ExecutionPlan, ReplaceChildrenOptions};use crate::{DisplayAs, DisplayFormatType, PlanProperties};use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};@@ -148,13 +148,24 @@ impl ExecutionPlan for TestMemoryExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn repartitioned(&self,_target_partitions: usize,diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rsindex 9a3f05a6e0..1e2005e908 100644--- a/datafusion/physical-plan/src/test/exec.rs+++ b/datafusion/physical-plan/src/test/exec.rs@@ -17,6 +17,7 @@//! Simple iterator over batches for use in testing+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions};use crate::{DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,RecordBatchStream, SendableRecordBatchStream, Statistics, common,@@ -218,13 +219,24 @@ impl ExecutionPlan for MockExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Returns a stream which yields datafn execute(&self,@@ -449,9 +461,10 @@ impl ExecutionPlan for BarrierExec {unimplemented!()}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}@@ -463,6 +476,16 @@ impl ExecutionPlan for BarrierExec {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Returns a stream which yields datafn execute(&self,@@ -592,9 +615,10 @@ impl ExecutionPlan for ErrorExec {unimplemented!()}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unimplemented!()}@@ -606,6 +630,16 @@ impl ExecutionPlan for ErrorExec {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Returns a stream which yields datafn execute(&self,@@ -692,13 +726,24 @@ impl ExecutionPlan for StatisticsExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -796,9 +841,10 @@ impl ExecutionPlan for BlockingExec {vec![]}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {internal_err!("Children cannot be replaced in {self:?}")}@@ -810,6 +856,16 @@ impl ExecutionPlan for BlockingExec {Ok(TreeNodeRecursion::Continue)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,@@ -952,13 +1008,24 @@ impl ExecutionPlan for PanicExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {internal_err!("Children cannot be replaced in {:?}", self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/tree_node.rs b/datafusion/physical-plan/src/tree_node.rsindex aa4f144f91..dcdceff869 100644--- a/datafusion/physical-plan/src/tree_node.rs+++ b/datafusion/physical-plan/src/tree_node.rs@@ -20,7 +20,8 @@use std::fmt::{self, Display, Formatter};use std::sync::Arc;-use crate::{ExecutionPlan, displayable, with_new_children_if_necessary};+use crate::execution_plan::replace_children_if_necessary;+use crate::{ExecutionPlan, displayable};use datafusion_common::Result;use datafusion_common::tree_node::{ConcreteTreeNode, DynTreeNode};@@ -35,7 +36,7 @@ impl DynTreeNode for dyn ExecutionPlan {arc_self: Arc<Self>,new_children: Vec<Arc<Self>>,) -> Result<Arc<Self>> {- with_new_children_if_necessary(arc_self, new_children)+ replace_children_if_necessary(arc_self, new_children)}}@@ -73,7 +74,7 @@ impl<T> PlanContext<T> {/// if the `PlanContext.children` have been changed.pub fn update_plan_from_children(mut self) -> Result<Self> {let children_plans = self.children.iter().map(|c| Arc::clone(&c.plan)).collect();- self.plan = with_new_children_if_necessary(self.plan, children_plans)?;+ self.plan = replace_children_if_necessary(self.plan, children_plans)?;Ok(self)}diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex fb62deecc3..c1cc5da31a 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -31,7 +31,6 @@ use super::{PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics,metrics::{ExecutionPlanMetricsSet, MetricsSet},};-use crate::check_if_same_properties;use crate::execution_plan::{CardinalityEffect, InvariantLevel, boundedness_from_children,check_default_invariants, emission_type_from_children,@@ -45,6 +44,7 @@ use crate::metrics::BaselineMetrics;use crate::projection::{ProjectionExec, ProjectionExpr, make_with_child};use crate::statistics::{ChildStats, StatisticsArgs};use crate::stream::ObservedStream;+use crate::{ChildrenPropertiesMode, ReplaceChildrenOptions, validate_child_count};use arrow::datatypes::{Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;@@ -330,23 +330,40 @@ impl ExecutionPlan for UnionExec {Ok(TreeNodeRecursion::Continue)}+ fn replace_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ inputs: children,+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => UnionExec::try_new(children),+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- UnionExec::try_new(children)+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- inputs: children,- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(@@ -716,28 +733,47 @@ impl ExecutionPlan for InterleaveExec {Ok(TreeNodeRecursion::Continue)}+ fn replace_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ inputs: children,+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ // New children are no longer interleavable, which might be a bug of optimization rewrite.+ assert_or_internal_err!(+ can_interleave(children.iter()),+ "Can not create InterleaveExec: new children can not be interleaved"+ );+ Ok(Arc::new(InterleaveExec::try_new(children)?))+ }+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- // New children are no longer interleavable, which might be a bug of optimization rewrite.- assert_or_internal_err!(- can_interleave(children.iter()),- "Can not create InterleaveExec: new children can not be interleaved"- );- check_if_same_properties!(self, children);- Ok(Arc::new(InterleaveExec::try_new(children)?))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- inputs: children,- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex 1877f668c5..3fa274b27a 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -28,8 +28,9 @@ use super::metrics::{use super::{DisplayAs, ExecutionPlanProperties, PlanProperties};use crate::stream::EmptyRecordBatchStream;use crate::{- DisplayFormatType, Distribution, ExecutionPlan, RecordBatchStream,- SendableRecordBatchStream, check_if_same_properties,+ ChildrenPropertiesMode, DisplayFormatType, Distribution, ExecutionPlan,+ RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,+ validate_child_count,};use arrow::array::{@@ -235,29 +236,46 @@ impl ExecutionPlan for UnnestExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(UnnestExec::new(- children.swap_remove(0),- self.list_column_indices.clone(),- self.struct_column_indices.clone(),- Arc::clone(&self.schema),- self.options.clone(),- )?))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(UnnestExec::new(+ children.swap_remove(0),+ self.list_column_indices.clone(),+ self.struct_column_indices.clone(),+ Arc::clone(&self.schema),+ self.options.clone(),+ )?)),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn required_input_distribution(&self) -> Vec<Distribution> {diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex c6a417cd44..d4c98009ba 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -35,10 +35,10 @@ use crate::windows::{window_equivalence_properties,};use crate::{- ColumnStatistics, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,- ExecutionPlanProperties, InputDistributionRequirements, InputOrderMode,- PlanProperties, RecordBatchStream, SendableRecordBatchStream, Statistics, WindowExpr,- check_if_same_properties,+ ChildrenPropertiesMode, ColumnStatistics, DisplayAs, DisplayFormatType, Distribution,+ ExecutionPlan, ExecutionPlanProperties, InputDistributionRequirements,+ InputOrderMode, PlanProperties, RecordBatchStream, ReplaceChildrenOptions,+ SendableRecordBatchStream, Statistics, WindowExpr, validate_child_count,};use arrow::compute::take_record_batch;@@ -463,30 +463,49 @@ impl ExecutionPlan for BoundedWindowAggExec {vec![true]}+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let new = BoundedWindowAggExec::try_new(+ self.window_expr.clone(),+ Arc::clone(&children[0]),+ self.input_order_mode.clone(),+ self.can_repartition,+ )?+ .with_state_observer(self.state_observer.clone())?;+ Ok(Arc::new(new))+ }+ }+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- let new = BoundedWindowAggExec::try_new(- self.window_expr.clone(),- Arc::clone(&children[0]),- self.input_order_mode.clone(),- self.can_repartition,- )?- .with_state_observer(self.state_observer.clone())?;- Ok(Arc::new(new))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex bbf9a14fd5..d794e7df9d 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -33,10 +33,10 @@ use crate::windows::{window_equivalence_properties,};use crate::{- ColumnStatistics, DisplayAs, DisplayFormatType, Distribution, ExecutionPlan,- ExecutionPlanProperties, InputDistributionRequirements, PhysicalExpr, PlanProperties,- RecordBatchStream, SendableRecordBatchStream, Statistics, WindowExpr,- check_if_same_properties,+ ChildrenPropertiesMode, ColumnStatistics, DisplayAs, DisplayFormatType, Distribution,+ ExecutionPlan, ExecutionPlanProperties, InputDistributionRequirements, PhysicalExpr,+ PlanProperties, RecordBatchStream, ReplaceChildrenOptions, SendableRecordBatchStream,+ Statistics, WindowExpr, validate_child_count,};use arrow::array::ArrayRef;@@ -262,27 +262,44 @@ impl ExecutionPlan for WindowAggExec {}}- fn with_new_children(+ fn replace_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- check_if_same_properties!(self, children);- Ok(Arc::new(WindowAggExec::try_new(- self.window_expr.clone(),- children.swap_remove(0),- true,- )?))+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => Ok(Arc::new(WindowAggExec::try_new(+ self.window_expr.clone(),+ children.swap_remove(0),+ true,+ )?)),+ }+ }++ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )}fn with_new_children_and_same_properties(self: Arc<Self>,- mut children: Vec<Arc<dyn ExecutionPlan>>,+ children: Vec<Arc<dyn ExecutionPlan>>,) -> Result<Arc<dyn ExecutionPlan>> {- Ok(Arc::new(Self {- input: children.swap_remove(0),- metrics: ExecutionPlanMetricsSet::new(),- ..Self::clone(&*self)- }))+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Keep),+ )}fn execute(diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rsindex 83cd0a15a6..b5d6fd47bc 100644--- a/datafusion/physical-plan/src/work_table.rs+++ b/datafusion/physical-plan/src/work_table.rs@@ -25,8 +25,8 @@ use crate::execution_plan::{Boundedness, EmissionType, SchedulingType};use crate::memory::MemoryStream;use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};use crate::{- DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,- SendableRecordBatchStream, Statistics,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,+ ReplaceChildrenOptions, SendableRecordBatchStream, Statistics,};use crate::statistics::StatisticsArgs;@@ -194,13 +194,24 @@ impl ExecutionPlan for WorkTableExec {Ok(TreeNodeRecursion::Continue)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(Arc::clone(&self) as Arc<dyn ExecutionPlan>)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+/// Stream the batches that were written to the work table.fn execute(&self,diff --git a/datafusion/proto/tests/cases/plans/dispatch.rs b/datafusion/proto/tests/cases/plans/dispatch.rsindex af9d8a62d3..75f299107e 100644--- a/datafusion/proto/tests/cases/plans/dispatch.rs+++ b/datafusion/proto/tests/cases/plans/dispatch.rs@@ -27,8 +27,8 @@ use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion::physical_plan::sorts::sort::SortExec;use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,- SendableRecordBatchStream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr,+ PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream,};use datafusion::prelude::SessionContext;use datafusion_common::tree_node::TreeNodeRecursion;@@ -81,14 +81,28 @@ impl ExecutionPlan for DowncastDelegatingExec {self.inner.apply_expressions(f)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {- let inner = Arc::clone(&self.inner).with_new_children(children)?;+ let inner = Arc::clone(&self.inner).replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;Ok(Arc::new(Self::new(inner)))}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn downcast_delegate(&self) -> Option<&dyn ExecutionPlan> {Some(self.inner.as_ref())}diff --git a/datafusion/proto/tests/cases/plans/dynamic_filters.rs b/datafusion/proto/tests/cases/plans/dynamic_filters.rsindex cda649b4c5..ee0ff9d8b1 100644--- a/datafusion/proto/tests/cases/plans/dynamic_filters.rs+++ b/datafusion/proto/tests/cases/plans/dynamic_filters.rs@@ -47,8 +47,8 @@ use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode};use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr};use datafusion::physical_plan::sorts::sort::SortExec;use datafusion::physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,- SendableRecordBatchStream,+ ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr,+ PlanProperties, ReplaceChildrenOptions, SendableRecordBatchStream,};use datafusion::prelude::SessionContext;use datafusion_common::config::{ConfigOptions, TableParquetOptions};@@ -789,13 +789,24 @@ impl ExecutionPlan for CustomExecWithExprs {datafusion_physical_plan::apply_expression_roots(&self.exprs, f)}- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {unreachable!()}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(+ children,+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/proto/tests/cases/plans/limits.rs b/datafusion/proto/tests/cases/plans/limits.rsindex e1c33ff949..a832d46d53 100644--- a/datafusion/proto/tests/cases/plans/limits.rs+++ b/datafusion/proto/tests/cases/plans/limits.rs@@ -38,6 +38,7 @@ use datafusion::physical_plan::coop::CooperativeExec;use datafusion::physical_plan::empty::EmptyExec;use datafusion::physical_plan::expressions::{PhysicalSortExpr, col};use datafusion::physical_plan::limit::{GlobalLimitExec, LocalLimitExec};+use datafusion::physical_plan::{ChildrenPropertiesMode, ReplaceChildrenOptions};use datafusion::prelude::SessionContext;use datafusion_common::Result;use datafusion_common::config::ConfigOptions;@@ -148,7 +149,10 @@ fn roundtrip_limit_required_ordering_reaches_data_source() -> Result<()> {roundtrip_test_and_return(Arc::new(limit), &ctx, &codec, &proto_converter)?;// Child replacement must not erase the decoded ordering before pushdown.- let rebuilt = decoded.with_new_children(vec![make_scan()])?;+ let rebuilt = decoded.replace_children(+ vec![make_scan()],+ ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),+ )?;let optimized =LimitPushdown::new().optimize(rebuilt, &ConfigOptions::default())?;diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.mdindex c6a316aa74..c094f8bf7e 100644--- a/docs/source/library-user-guide/custom-table-providers.md+++ b/docs/source/library-user-guide/custom-table-providers.md@@ -247,14 +247,22 @@ impl ExecutionPlan for MyExecPlan {vec![] // Leaf node -- no children}- fn with_new_children(+ fn replace_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {assert!(children.is_empty());Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(children, ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute))+ }+fn execute(&self,partition: usize,@@ -655,7 +663,7 @@ and reading files that cannot possibly match the query.# use datafusion::execution::context::TaskContext;# use datafusion::logical_expr::{Expr, TableProviderFilterPushDown};# use datafusion::physical_expr::EquivalenceProperties;-# use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr, PlanProperties};+# use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr, PlanProperties, ChildrenPropertiesMode, ReplaceChildrenOptions};# use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};#/// A table provider backed by date-partitioned directories.@@ -764,7 +772,15 @@ impl DatePartitionedTable {# fn name(&self) -> &str { "DatePartitionedExec" }# fn properties(&self) -> &Arc<PlanProperties> { &self.properties }# fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { vec![] }-# fn with_new_children(self: Arc<Self>, _: Vec<Arc<dyn ExecutionPlan>>) -> Result<Arc<dyn ExecutionPlan>> { Ok(self) }+# fn replace_children(self: Arc<Self>, _: Vec<Arc<dyn ExecutionPlan>>, _: ReplaceChildrenOptions) -> Result<Arc<dyn ExecutionPlan>> { Ok(self) }+#+# fn with_new_children(+# self: Arc<Self>,+# children: Vec<Arc<dyn ExecutionPlan>>,+# ) -> Result<Arc<dyn ExecutionPlan>> {+# self.replace_children(children, ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute))+# }+## fn execute(&self, _: usize, _: Arc<TaskContext>) -> Result<SendableRecordBatchStream> { todo!() }# fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }# }@@ -801,10 +817,8 @@ use datafusion::physical_expr::EquivalenceProperties;use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};use datafusion::physical_plan::stream::RecordBatchStreamAdapter;use datafusion::physical_plan::{-# DisplayAs, DisplayFormatType,- ExecutionPlan, Partitioning,-# PhysicalExpr,- PlanProperties,+# DisplayAs, DisplayFormatType, PhysicalExpr,+ ChildrenPropertiesMode, ReplaceChildrenOptions, ExecutionPlan, Partitioning, PlanProperties,};use futures::stream;@@ -874,13 +888,21 @@ impl ExecutionPlan for CountingExec {fn properties(&self) -> &Arc<PlanProperties> { &self.properties }fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { vec![] }- fn with_new_children(+ fn replace_children(self: Arc<Self>,- _children: Vec<Arc<dyn ExecutionPlan>>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ _: ReplaceChildrenOptions,) -> Result<Arc<dyn ExecutionPlan>> {Ok(self)}+ fn with_new_children(+ self: Arc<Self>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ self.replace_children(children, ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute))+ }+fn execute(&self,partition: usize,diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex 7d0c19c846..d64f287ea0 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -576,6 +576,78 @@ See [PR #22733](https://github.com/apache/datafusion/pull/22733) fordetails, including the per-variant size breakdown and benchmarkresults.+### `ExecutionPlan::with_new_children` and `ExecutionPlan::with_new_children_and_same_properties` deprecated++`with_new_children` and `with_new_children_and_same_properties` have been+deprecated. These methods are used to replace the child plans of an+`ExecutionPlan` while leaving the plan otherwise identical.++`with_new_children_if_necessary` has also been deprecated in favor of+`replace_children_if_necessary` for consistency in naming.++As noted [here](https://github.com/apache/datafusion/pull/23332#discussion_r3554897693),+while the addition of `with_new_children_and_same_properties` has the benefit+of skipping potentially expensive computation in the case that replacement children+have the same properties as the original children, it widens the API surface area+of `ExecutionPlan` in a way that could be confusing for users.++Thus, to rectify this, we unify these methods by introducing `replace_children`.+`replace_children` solves this problem by taking `ReplaceChildrenOptions`,+which includes a `ChildrenPropertiesMode`. The mode has two variants,+`Keep` and `Recompute`, which tell `replace_children` whether plan+properties can be reused or need to be recomputed.++This method is called from `replace_children_if_necessary`, which is the+standard entry point that should be used for replacing the children of a node.++**Migration guide:**++To migrate from `with_new_children` and `with_new_children_and_same_properties`+to `replace_children`, it is recommended to implement `replace_children` with+a `match` statement matching on the `ChildrenPropertiesMode`. In the case that+the properties match the children, `ChildrenPropertiesMode::Keep`,+follow the body of `with_new_children_and_same_properties`. In the case that+the properties do not match the children, `ChildrenPropertiesMode::Recompute`,+follow the body of `with_new_children`.++For example, take a look at the implementation for `FilterExec`:++```+ fn replace_children(+ self: Arc<Self>,+ mut children: Vec<Arc<dyn ExecutionPlan>>,+ options: ReplaceChildrenOptions,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ validate_child_count!(self, children);+ match options.children_properties {+ ChildrenPropertiesMode::Keep => Ok(Arc::new(Self {+ input: children.swap_remove(0),+ metrics: ExecutionPlanMetricsSet::new(),+ ..Self::clone(&*self)+ })),+ ChildrenPropertiesMode::Recompute => {+ let new_input = children.swap_remove(0);+ FilterExecBuilder::from(&*self)+ .with_input(new_input)+ .build()+ .map(|e| Arc::new(e) as _)+ }+ }+ }+```++In the case that the options indicate the properties are the same, we can simply+swap the children without having to recompute the properties. In the other case,+we create a new node from scratch.++To ensure that this works correctly, it is recommended that users also look+through their codebase and ensure that they use `replace_children_if_necessary`+for these changes — `replace_children_if_necessary` should be preferred over+manual use of `replace_children`, since `replace_children_if_necessary` will+call `replace_children` with the correct options filled in.++See [PR #23903](https://github.com/apache/datafusion/pull/23903) for details.+### `ListingOptions::target_partitions` and `collect_stat` removedThe `target_partitions` and `collect_stat` fields on