Apache DataFusion SQL Query Engine
Reapply "Add ExecutionPlan::apply_expressions() (apache#20337)" (apache#22437) (#24018)
Jayant Shrivastava2bfdd4aea181
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex a2d7d76999..a5a38edf0b 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -27,6 +27,7 @@ use datafusion::arrow::array::{UInt8Builder, UInt64Builder};use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::arrow::record_batch::RecordBatch;use datafusion::common::assert_batches_eq;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::datasource::{TableProvider, TableType, provider_as_source};use datafusion::error::Result;use datafusion::execution::context::TaskContext;@@ -314,4 +315,13 @@ impl ExecutionPlan for CustomExec {None,)?))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rsindex ca765774d1..6decb84b55 100644--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs@@ -29,6 +29,7 @@use arrow::array::record_batch;use arrow::record_batch::RecordBatch;use arrow_schema::SchemaRef;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::common::{exec_datafusion_err, internal_err};use datafusion::datasource::{DefaultTableSource, memory::MemTable};use datafusion::error::Result;@@ -291,4 +292,13 @@ impl ExecutionPlan for BufferingExecutionPlan {}),)))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rsindex 6077a982c3..d5197fe61b 100644--- a/datafusion-examples/examples/proto/composed_extension_codec.rs+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs@@ -37,6 +37,7 @@ use std::sync::Arc;use datafusion::common::Result;use datafusion::common::internal_err;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::execution::TaskContext;use datafusion::physical_plan::{DisplayAs, ExecutionPlan};use datafusion::prelude::SessionContext;@@ -124,6 +125,15 @@ impl ExecutionPlan for ParentExec {) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {unreachable!()}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// A PhysicalExtensionCodec that can serialize and deserialize ParentExec@@ -202,6 +212,15 @@ impl ExecutionPlan for ChildExec {) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {unreachable!()}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// A PhysicalExtensionCodec that can serialize and deserialize ChildExecdiff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex c019e136cc..388175ee3a 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -118,7 +118,7 @@ use datafusion::{};use datafusion_common::{DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,- plan_datafusion_err, plan_err,+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,};use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{@@ -750,6 +750,15 @@ impl ExecutionPlan for SampleExec {Ok(Arc::new(stats))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// Bernoulli sampler: includes each row with probability `(upper - lower)`.diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 5d07133799..ef5669a3a1 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -31,6 +31,7 @@ use arrow::compute::{and, filter_record_batch};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::error::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};use datafusion_datasource::memory::{MemSink, MemorySourceConfig};use datafusion_datasource::sink::DataSinkExec;@@ -39,13 +40,13 @@ use datafusion_expr::dml::InsertOp;use datafusion_expr::physical_planning_context::PhysicalPlanningContext;use datafusion_expr::{Expr, SortExpr, TableType};use datafusion_physical_expr::{- LexOrdering, PhysicalExpr, create_physical_expr, create_physical_sort_exprs,+ LexOrdering, create_physical_expr, create_physical_sort_exprs,};use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,- collect_partitioned,+ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr,+ PlanProperties, collect_partitioned,};use datafusion_session::Session;@@ -597,4 +598,11 @@ impl ExecutionPlan for DmlResultExec {stream,)))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 58f00551ac..1202f08a56 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -4950,6 +4950,13 @@ mod tests {) -> Result<SendableRecordBatchStream> {unimplemented!("NoOpExecutionPlan::execute");}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}struct ExpressionExtensionPlanner;@@ -5117,6 +5124,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for OkExtensionNode {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {@@ -5163,6 +5176,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for InvariantFailsExtensionNode {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {@@ -5287,6 +5306,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for ExecutableInvariantFails {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex c70722cb2f..43f4be05b2 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -38,6 +38,7 @@ use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;use datafusion_common::project_schema;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::PlanProperties;use datafusion_physical_plan::StatisticsArgs;@@ -209,6 +210,15 @@ impl ExecutionPlan for CustomExecutionPlan {.collect(),}))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[async_trait]diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex 18695accd0..7437bbc543 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -35,6 +35,7 @@ use datafusion::prelude::*;use datafusion::scalar::ScalarValue;use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, internal_err, not_impl_err};use datafusion_expr::expr::{BinaryExpr, Cast};use datafusion_functions_aggregate::expr_fn::count;@@ -148,6 +149,15 @@ impl ExecutionPlan for CustomPlan {})),)))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[derive(Clone, Debug)]diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rsindex d289b5c348..c14ca685b2 100644--- a/datafusion/core/tests/custom_sources_cases/statistics.rs+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs@@ -33,6 +33,7 @@ use datafusion::{scalar::ScalarValue,};use datafusion_catalog::Session;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{project_schema, stats::Precision};use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};@@ -185,6 +186,15 @@ impl ExecutionPlan for StatisticsValidation {Ok(Arc::new(self.stats.clone()))}}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}fn init_ctx(stats: Statistics, schema: Schema) -> Result<SessionContext> {diff --git a/datafusion/core/tests/fuzz_cases/once_exec.rs b/datafusion/core/tests/fuzz_cases/once_exec.rsindex 9b57141061..638cbe4c9d 100644--- a/datafusion/core/tests/fuzz_cases/once_exec.rs+++ b/datafusion/core/tests/fuzz_cases/once_exec.rs@@ -17,6 +17,7 @@use arrow_schema::SchemaRef;use datafusion_common::internal_datafusion_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};@@ -105,4 +106,13 @@ impl ExecutionPlan for OnceExec {stream.ok_or_else(|| internal_datafusion_err!("Stream already consumed"))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> datafusion_common::Result<TreeNodeRecursion>,+ ) -> datafusion_common::Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex ac7e7a75a2..e0b152d1f0 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -40,7 +40,9 @@ use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::ScalarValue;use datafusion_common::config::CsvOptions;use datafusion_common::error::Result;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_expr::{JoinType, Operator};@@ -203,6 +205,13 @@ impl ExecutionPlan for SortRequiredExec {)))}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,@@ -290,6 +299,13 @@ impl ExecutionPlan for SinglePartitionMaintainsOrderExec {Ok(Arc::new(Self::new(child)))}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rsindex c04ccd2f3c..83fabcdff8 100644--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs@@ -24,7 +24,7 @@use insta::assert_snapshot;use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::{TransformedResult, TreeNode};+use datafusion_common::tree_node::{TransformedResult, TreeNode, TreeNodeRecursion};use datafusion_physical_optimizer::PhysicalOptimizerRule;use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;use datafusion_physical_optimizer::ensure_requirements::enforce_sorting::{@@ -130,6 +130,12 @@ impl ExecutionPlan for MockMultiPartitionExec {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,_children: Vec<Arc<dyn ExecutionPlan>>,@@ -993,6 +999,12 @@ impl ExecutionPlan for MockReqExec {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn input_distribution_requirements(&self,) -> datafusion_physical_plan::InputDistributionRequirements {diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex f2b1f66c28..a26761107a 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -50,7 +50,9 @@ use datafusion_functions_aggregate::{min_max::{max_udaf, min_udaf},};use datafusion_physical_expr::{- LexOrdering, PhysicalSortExpr, expressions::col, utils::conjunction,+ LexOrdering, PhysicalSortExpr,+ expressions::{DynamicFilterPhysicalExpr, col},+ utils::conjunction,};use datafusion_physical_expr::{Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder,@@ -2919,23 +2921,30 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {);}-// Not portable to sqllogictest: asserts on the dynamic filter's Arc ownership.-// The observable behavior (probe-side scan-// receiving the dynamic filter when the data source supports it) is-// already covered by the simpler CollectLeft port in push_down_filter_parquet.slt;-// the with_support(false) branch has no SQL analog (parquet always supports-// pushdown).-#[expect(- deprecated,- reason = "the borrowed getter avoids adding a producer Arc that is_used would count"-)]-#[tokio::test]-async fn test_hashjoin_dynamic_filter_pushdown_is_used() {- use datafusion_common::JoinType;- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};+// Not portable to sqllogictest: verifies whether the optimized probe-side plan+// retains the HashJoinExec's dynamic filter expression. The with_support(false)+// branch has no SQL analog because parquet supports filter pushdown.+#[test]+fn test_hashjoin_dynamic_filter_pushdown_is_used() {+ fn contains_expression_id(plan: &Arc<dyn ExecutionPlan>, expression_id: u64) -> bool {+ let mut found = false;+ plan.apply(|node| {+ node.apply_expressions(&mut |root| {+ root.apply(|expr| {+ if expr.expression_id() == Some(expression_id) {+ found = true;+ Ok(TreeNodeRecursion::Stop)+ } else {+ Ok(TreeNodeRecursion::Continue)+ }+ })+ })+ })+ .unwrap();+ found+ }- // Test both cases: probe side with and without filter pushdown support- for (probe_supports_pushdown, expected_is_used) in [(false, false), (true, true)] {+ for (probe_supports_pushdown, expected_consumer) in [(false, false), (true, true)] {let build_side_schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, false),Field::new("b", DataType::Utf8, false),@@ -2988,31 +2997,26 @@ async fn test_hashjoin_dynamic_filter_pushdown_is_used() {.unwrap(),) as Arc<dyn ExecutionPlan>;- // Apply filter pushdown optimizationlet mut config = ConfigOptions::default();config.execution.parquet.pushdown_filters = true;config.optimizer.enable_dynamic_filter_pushdown = true;let plan = FilterPushdown::new_post_optimization().optimize(plan, &config).unwrap();-- // Get the HashJoinExec to check the dynamic filterlet hash_join = plan.downcast_ref::<HashJoinExec>().expect("Plan should be HashJoinExec");+ let dynamic_filters = hash_join.dynamic_expressions_produced();+ let expression_id = dynamic_filters+ .first()+ .expect("Dynamic filter should be created")+ .expression_id()+ .expect("Dynamic filters always have an expression ID");- // Verify that a dynamic filter was created- let dynamic_filter = hash_join- .dynamic_filter_expr()- .expect("Dynamic filter should be created");-- // Verify that is_used() returns the expected value based on probe side support.- // When probe_supports_pushdown=false: no consumer holds a reference (is_used=false)- // When probe_supports_pushdown=true: probe side holds a reference (is_used=true)assert_eq!(- dynamic_filter.is_used(),- expected_is_used,- "is_used() should return {expected_is_used} when probe side support is {probe_supports_pushdown}"+ contains_expression_id(hash_join.right(), expression_id),+ expected_consumer,+ "probe consumer should be {expected_consumer} when pushdown support is {probe_supports_pushdown}");}}@@ -3110,6 +3114,106 @@ async fn test_filter_with_projection_pushdown() {assert_batches_eq!(expected, &result);}+/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree.+///+/// Not portable to sqllogictest: asserts by walking the plan tree with+/// `apply_expressions` + `downcast_ref::<DynamicFilterPhysicalExpr>` and+/// counting nodes. Neither API is observable from SQL.+#[tokio::test]+async fn test_discover_dynamic_filters_via_expressions_api() {+ fn count_dynamic_filters(plan: &Arc<dyn ExecutionPlan>) -> usize {+ let mut count = 0;++ // Check expressions from this node using apply_expressions+ let _ = plan.apply_expressions(&mut |expr| {+ if let Some(_df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {+ count += 1;+ }+ Ok(TreeNodeRecursion::Continue)+ });++ // Recursively visit children+ for child in plan.children() {+ count += count_dynamic_filters(child);+ }++ count+ }++ // Create build side (left)+ let build_batches =+ vec![record_batch!(("a", Utf8, ["foo", "bar"]), ("b", Int32, [1, 2])).unwrap()];+ let build_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Int32, false),+ ]));+ let build_scan = TestScanBuilder::new(build_schema.clone())+ .with_support(true)+ .with_batches(build_batches)+ .build();++ // Create probe side (right)+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["foo", "bar", "baz", "qux"]),+ ("c", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ];+ let probe_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("c", DataType::Float64, false),+ ]));+ let probe_scan = TestScanBuilder::new(probe_schema.clone())+ .with_support(true)+ .with_batches(probe_batches)+ .build();++ // Create HashJoinExec+ let plan = Arc::new(+ HashJoinExec::try_new(+ build_scan,+ probe_scan,+ vec![(+ col("a", &build_schema).unwrap(),+ col("a", &probe_schema).unwrap(),+ )],+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ ) as Arc<dyn ExecutionPlan>;++ // Before optimization: no dynamic filters+ let count_before = count_dynamic_filters(&plan);+ assert_eq!(+ count_before, 0,+ "Before optimization, should have no dynamic filters"+ );++ // Apply filter pushdown optimization (this creates dynamic filters)+ let mut config = ConfigOptions::default();+ config.optimizer.enable_dynamic_filter_pushdown = true;+ config.execution.parquet.pushdown_filters = true;+ let optimized_plan = FilterPushdown::new_post_optimization()+ .optimize(plan, &config)+ .unwrap();++ // After optimization: should discover dynamic filters+ // We expect 2 dynamic filters:+ // 1. In the HashJoinExec (producer)+ // 2. In the DataSourceExec (consumer, pushed down to the probe side)+ let count_after = count_dynamic_filters(&optimized_plan);+ assert_eq!(+ count_after, 2,+ "After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}"+ );+}+#[test]fn test_discover_dynamic_expression_producers() {fn producer_count(plan: &Arc<dyn ExecutionPlan>) -> usize {diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex 3827e6e98b..c7e3799842 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -25,6 +25,7 @@ use std::{use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{ColumnStatistics, JoinType, ScalarValue, stats::Precision};use datafusion_common::{JoinSide, NullEquality};use datafusion_common::{Result, Statistics};@@ -1125,6 +1126,13 @@ impl ExecutionPlan for UnboundedExec {batch: self.batch.clone(),}))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[derive(Eq, PartialEq, Debug)]@@ -1230,6 +1238,13 @@ impl ExecutionPlan for StatisticsExec {self.stats.clone()}))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[test]diff --git a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs b/datafusion/core/tests/physical_optimizer/pushdown_utils.rsindex 2ffd1899b3..4f8b9ad42b 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs@@ -18,6 +18,7 @@use arrow::datatypes::SchemaRef;use arrow::{array::RecordBatch, compute::concat_batches};use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, config::ConfigOptions, internal_err};use datafusion_datasource::{PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,@@ -234,6 +235,21 @@ impl FileSource for TestSource {fn table_schema(&self) -> &datafusion_datasource::TableSchema {&self.table_schema}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.predicate.iter().chain(+ self.projection+ .iter()+ .flatten()+ .map(|proj_expr| &proj_expr.expr),+ ),+ f,+ )+ }}#[derive(Debug, Clone)]@@ -549,4 +565,11 @@ impl ExecutionPlan for TestNode {Ok(res)}}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots([&self.predicate], f)+ }}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex 3235ea25fd..833077fe49 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -30,7 +30,9 @@ use datafusion::datasource::physical_plan::ParquetSource;use datafusion::datasource::source::{DataSource, DataSourceExec};use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;use datafusion_common::{ColumnStatistics, JoinType, NullEquality, Result, Statistics, internal_err,@@ -543,6 +545,13 @@ impl ExecutionPlan for RequirementsTestExec {) -> Result<SendableRecordBatchStream> {unimplemented!("Test exec does not support execution")}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// A [`PlanContext`] object is susceptible to being left in an inconsistent state after@@ -1091,6 +1100,13 @@ impl ExecutionPlan for TestScan {})}}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// Helper function to create a TestScan with ordering@@ -1107,6 +1123,13 @@ struct InexactMemorySource {}impl DataSource for InexactMemorySource {+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ self.inner.apply_expressions(f)+ }+fn open(&self,partition: usize,diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rsindex f3d3f70bdf..c61fe018aa 100644--- a/datafusion/core/tests/user_defined/insert_operation.rs+++ b/datafusion/core/tests/user_defined/insert_operation.rs@@ -25,6 +25,7 @@ use datafusion::{};use datafusion_catalog::{Session, TableProvider};use datafusion_common::config::Dialect;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_expr::{Expr, TableType, dml::InsertOp};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::SchedulingType;@@ -176,6 +177,15 @@ impl ExecutionPlan for TestInsertExec {) -> Result<datafusion_execution::SendableRecordBatchStream> {unimplemented!("TestInsertExec is a stub for testing.")}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}fn make_count_schema() -> SchemaRef {diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex 99363ba500..2b042b613d 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -91,7 +91,9 @@ use datafusion::{prelude::{SessionConfig, SessionContext},};use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::{ScalarValue, assert_eq_or_internal_err, assert_or_internal_err};use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};use datafusion_optimizer::AnalyzerRule;@@ -749,6 +751,15 @@ impl ExecutionPlan for TopKExec {state: BTreeMap::new(),}))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}// A very specialized TopK implementationdiff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex dbdc1b1cf0..bf92faa9a1 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -40,6 +40,7 @@ use arrow::buffer::Buffer;use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};use datafusion_common::error::Result;use datafusion_common::exec_datafusion_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::PartitionedFile;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -393,6 +394,21 @@ impl FileSource for ArrowSource {Some(&self.projection.source)}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.projection+ .source+ .iter()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+/// Emit an `ArrowScan` node wrapping the shared base config.////// Decoding defaults to the IPC file format because protobuf does notdiff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex cef85c58df..3956a7318d 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -22,6 +22,7 @@ use std::sync::Arc;use arrow::datatypes::{Schema, SchemaRef};use arrow_avro::reader::{Reader, ReaderBuilder};use datafusion_common::error::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -169,6 +170,21 @@ impl FileSource for AvroSource {false}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.projection+ .source+ .iter()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+/// Emit an `AvroScan` node wrapping the shared base config.#[cfg(feature = "proto")]fn try_to_proto(diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex f498a3c5b7..d5fc6288ea 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -33,6 +33,7 @@ use datafusion_datasource::{use arrow::csv;use datafusion_common::config::CsvOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, exec_datafusion_err};use datafusion_common_runtime::JoinSet;use datafusion_datasource::file::FileSource;@@ -309,6 +310,21 @@ impl FileSource for CsvSource {}}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.projection+ .source+ .iter()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+/// Emit a `CsvScan` node wrapping the shared base config and CSV options.#[cfg(feature = "proto")]fn try_to_proto(diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex b7c2e5a45c..c6d420bafb 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -27,6 +27,7 @@ use crate::utils::{ChannelReader, JsonArrayToNdjsonReader};use datafusion_common::error::{DataFusionError, Result};use datafusion_common::exec_datafusion_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common_runtime::{JoinSet, SpawnedTask};use datafusion_datasource::boundary_stream::AlignedBoundaryStream;use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};@@ -232,6 +233,21 @@ impl FileSource for JsonSource {"json"}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.projection+ .source+ .iter()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+/// Emit a `JsonScan` node wrapping the shared base config.#[cfg(feature = "proto")]fn try_to_proto(diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex b11436812c..097b4563af 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -40,6 +40,7 @@ use arrow::array::timezone::Tz;use arrow::datatypes::TimeUnit;use datafusion_common::DataFusionError;use datafusion_common::config::TableParquetOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -1058,6 +1059,20 @@ impl FileSource for ParquetSource {})}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn PhysicalExpr>,+ ) -> datafusion_common::Result<TreeNodeRecursion>,+ ) -> datafusion_common::Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.predicate+ .iter()+ .chain(self.projection.iter().map(|proj_expr| &proj_expr.expr)),+ f,+ )+ }+/// Emit a `ParquetScan` node wrapping the shared base config plus the/// Parquet-specific pushdown predicate and `TableParquetOptions`.#[cfg(feature = "proto")]diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex 691bb314b7..f1a94f2e12 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -29,6 +29,7 @@ use crate::morsel::{FileOpenerMorselizer, Morselizer};#[expect(deprecated)]use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, not_impl_err};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};@@ -352,6 +353,23 @@ pub trait FileSource: Any + Send + Sync {None}+ /// Apply a function to all physical expressions used by this file source.+ ///+ /// This includes:+ /// - Filter predicates (which may contain dynamic filters)+ /// - Projection expressions+ ///+ /// The function `f` should be called once per expression unless the function returns+ /// [`TreeNodeRecursion::Stop`] to stop iteration.+ ///+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.+ ///+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;+/// Serialize this file source into a full [`PhysicalPlanNode`] (a/// `DataSourceExec` wrapping the `FileScanConfig`), if it knows how.///diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rsindex 766da2f8f7..91dcd5b76f 100644--- a/datafusion/datasource/src/file_scan_config/mod.rs+++ b/datafusion/datasource/src/file_scan_config/mod.rs@@ -37,6 +37,7 @@ use crate::{use arrow::datatypes::Fields;use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,};@@ -89,7 +90,9 @@ use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};/// # use arrow::datatypes::{Field, Fields, DataType, Schema, SchemaRef};/// # use object_store::ObjectStore;/// # use datafusion_common::Result;+/// # use datafusion_common::tree_node::TreeNodeRecursion;/// # use datafusion_datasource::file::FileSource;+/// # use datafusion_physical_plan::PhysicalExpr;/// # use datafusion_datasource::file_groups::FileGroup;/// # use datafusion_datasource::PartitionedFile;/// # use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};@@ -119,6 +122,7 @@ use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};/// # fn file_type(&self) -> &str { "parquet" }/// # // Note that this implementation drops the projection on the floor, it is not complete!/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }+/// # fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }/// # }/// # impl ParquetSource {/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }@@ -1162,6 +1166,14 @@ impl DataSource for FileScanConfig {Some(Arc::new(new_config))}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Delegate to the file source+ self.file_source.apply_expressions(f)+ }+/// Create any shared state that should be passed between sibling streams/// during one execution.///@@ -1585,9 +1597,12 @@ mod tests {use arrow::datatypes::Field;use datafusion_common::ColumnStatistics;use datafusion_common::stats::Precision;+ use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_batches_eq, internal_err};use datafusion_execution::TaskContext;use datafusion_expr::SortExpr;+ use datafusion_physical_expr::PhysicalExpr;+#[cfg(feature = "proto")]use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};use datafusion_physical_expr::create_physical_sort_expr;@@ -1656,6 +1671,13 @@ mod tests {inner: Arc::new(self.clone()) as Arc<dyn FileSource>,})}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[cfg(feature = "proto")]@@ -1702,6 +1724,13 @@ mod tests {"proto-hook-test"}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn try_to_proto(&self,_base: &FileScanConfig,@@ -3023,6 +3052,13 @@ mod tests {inner: Arc::new(self.clone()) as Arc<dyn FileSource>,})}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[test]diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rsindex 15ea2600f1..2370ed87a2 100644--- a/datafusion/datasource/src/memory.rs+++ b/datafusion/datasource/src/memory.rs@@ -28,6 +28,7 @@ use crate::source::{DataSource, DataSourceExec};use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::datatypes::{Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, ScalarValue, assert_or_internal_err, plan_err, project_schema,};@@ -257,6 +258,13 @@ impl DataSource for MemorySourceConfig {.transpose()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Serialize this `MemorySourceConfig` as a `MemoryScanExecNode` wrapped/// in a [`PhysicalPlanNode`]. Byte-compatible with the former central/// `MemoryScan` arm in `datafusion-proto`.diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rsindex 25b559f780..b89cf5d356 100644--- a/datafusion/datasource/src/sink.rs+++ b/datafusion/datasource/src/sink.rs@@ -24,9 +24,10 @@ use std::sync::Arc;use arrow::array::{ArrayRef, RecordBatch, UInt64Array};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::{Distribution, EquivalenceProperties};+use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr};use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequirements};use datafusion_physical_plan::metrics::MetricsSet;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;@@ -315,6 +316,13 @@ impl ExecutionPlan for DataSinkExec {)))}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Execute the plan and return a stream of `RecordBatch`es for/// the specified partition.fn execute(diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex 1fd5f865c4..929fed02b3 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -40,6 +40,7 @@ use itertools::Itertools;use crate::file::FileSource;use crate::file_scan_config::FileScanConfig;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, Result, Statistics};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};@@ -225,6 +226,22 @@ pub trait DataSource: Any + Send + Sync + Debug {None}+ /// Apply a closure to each expression used by this data source.+ ///+ /// This includes filter predicates (which may contain dynamic filters) and any+ /// other expressions used during data scanning.+ ///+ /// The function `f` should be called once per expression unless the function returns+ /// [`TreeNodeRecursion::Stop`] to stop iteration.+ ///+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.+ ///+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;+/// Injects arbitrary run-time state into this DataSource, returning a new instance/// that incorporates that state *if* it is relevant to the concrete DataSource implementation.///@@ -383,6 +400,14 @@ impl ExecutionPlan for DataSourceExec {Vec::new()}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Delegate to the underlying data source+ self.data_source.apply_expressions(f)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rsindex 5bef6d44e1..20dfae5b3a 100644--- a/datafusion/datasource/src/test_util.rs+++ b/datafusion/datasource/src/test_util.rs@@ -22,7 +22,7 @@ use crate::{use std::sync::Arc;use arrow::datatypes::Schema;-use datafusion_common::Result;+use datafusion_common::{Result, tree_node::TreeNodeRecursion};use datafusion_physical_expr::{PhysicalExpr, expressions::Column};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use object_store::ObjectStore;@@ -125,6 +125,13 @@ impl FileSource for MockSource {) -> Option<&datafusion_physical_plan::projection::ProjectionExprs> {Some(&self.projection.source)}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// Create a column expressiondiff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex 5191b2fc03..a0dd5e6cb6 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -20,6 +20,7 @@ use std::pin::Pin;use std::sync::Arc;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, Statistics};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr_common::metrics::MetricsSet;@@ -51,6 +52,10 @@ pub struct FFI_ExecutionPlan {/// Return a vector of children planspub children: unsafe extern "C" fn(plan: &Self) -> SVec<FFI_ExecutionPlan>,+ /// Return the physical expression roots owned by this plan node.+ pub apply_expressions:+ unsafe extern "C" fn(plan: &Self) -> FFI_Result<SVec<FFI_PhysicalExpr>>,+/// Return the dynamic expressions produced by this plan node.pub dynamic_expressions_produced:unsafe extern "C" fn(plan: &Self) -> SVec<FFI_PhysicalExpr>,@@ -96,6 +101,9 @@ pub struct FFI_ExecutionPlan {/// Release the memory of the private data when it is no longer being used.pub release: unsafe extern "C" fn(arg: &mut Self),+ /// Return the major DataFusion version number of this provider.+ pub version: unsafe extern "C" fn() -> u64,+/// Internal data. This is only to be accessed by the provider of the plan./// A [`ForeignExecutionPlan`] should never attempt to access this data.pub private_data: *mut c_void,@@ -143,6 +151,17 @@ unsafe extern "C" fn children_fn_wrapper(.collect()}+unsafe extern "C" fn apply_expressions_fn_wrapper(+ plan: &FFI_ExecutionPlan,+) -> FFI_Result<SVec<FFI_PhysicalExpr>> {+ let mut expressions = SVec::new();+ let result = plan.inner().apply_expressions(&mut |expr| {+ expressions.push(FFI_PhysicalExpr::from(Arc::clone(expr)));+ Ok(TreeNodeRecursion::Continue)+ });+ sresult!(result.map(|_| expressions))+}+unsafe extern "C" fn dynamic_expressions_produced_fn_wrapper(plan: &FFI_ExecutionPlan,) -> SVec<FFI_PhysicalExpr> {@@ -321,6 +340,7 @@ impl FFI_ExecutionPlan {Self {properties: properties_fn_wrapper,children: children_fn_wrapper,+ apply_expressions: apply_expressions_fn_wrapper,dynamic_expressions_produced: dynamic_expressions_produced_fn_wrapper,with_new_children: with_new_children_fn_wrapper,name: name_fn_wrapper,@@ -330,6 +350,7 @@ impl FFI_ExecutionPlan {partition_statistics: partition_statistics_fn_wrapper,clone: clone_fn_wrapper,release: release_fn_wrapper,+ version: crate::version,private_data: Box::into_raw(private_data) as *mut c_void,library_marker_id: crate::get_library_marker_id,}@@ -458,6 +479,24 @@ impl ExecutionPlan for ForeignExecutionPlan {}}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let expressions =+ df_result!(unsafe { (self.plan.apply_expressions)(&self.plan) })?;+ datafusion_physical_plan::apply_expression_roots(+ expressions.iter().map(|expression| {+ let expression: Arc<dyn datafusion_physical_plan::PhysicalExpr> =+ expression.into();+ expression+ }),+ f,+ )+ }+fn dynamic_expressions_produced(&self,) -> Vec<Arc<dyn datafusion_physical_plan::PhysicalExpr>> {@@ -509,6 +548,7 @@ pub mod tests {pub struct EmptyExec {props: Arc<PlanProperties>,children: Vec<Arc<dyn ExecutionPlan>>,+ expressions: Vec<Arc<dyn PhysicalExpr>>,dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,metrics: Option<MetricsSet>,statistics: Option<Statistics>,@@ -524,6 +564,7 @@ pub mod tests {Boundedness::Bounded,)),children: Vec::default(),+ expressions: Vec::default(),dynamic_expressions: Vec::default(),metrics: None,statistics: None,@@ -540,6 +581,14 @@ pub mod tests {self}+ pub fn with_expressions(+ mut self,+ expressions: Vec<Arc<dyn PhysicalExpr>>,+ ) -> Self {+ self.expressions = expressions;+ self+ }+pub fn with_dynamic_expressions(mut self,dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,@@ -579,6 +628,7 @@ pub mod tests {Ok(Arc::new(EmptyExec {props: Arc::clone(&self.props),children,+ expressions: self.expressions.clone(),dynamic_expressions: self.dynamic_expressions.clone(),metrics: self.metrics.clone(),statistics: self.statistics.clone(),@@ -610,6 +660,13 @@ pub mod tests {Statistics::new_unknown(self.props.eq_properties.schema())})))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(&self.expressions, f)+ }}pub(crate) fn create_dynamic_filter() -> Arc<DynamicFilterPhysicalExpr> {@@ -645,6 +702,35 @@ pub mod tests {Ok(())}+ #[test]+ fn test_ffi_execution_plan_apply_expressions() -> Result<()> {+ let schema = Arc::new(arrow::datatypes::Schema::empty());+ let dynamic_filter = create_dynamic_filter();+ let expected_id = dynamic_filter+ .expression_id()+ .expect("dynamic filters always have an expression ID");+ let expression: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;+ let original_plan =+ Arc::new(EmptyExec::new(schema).with_expressions(vec![expression]));++ let mut ffi_plan = FFI_ExecutionPlan::new(original_plan, None);+ ffi_plan.library_marker_id = crate::mock_foreign_marker_id;+ let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into()?;++ let mut retained = None;+ foreign_plan.apply_expressions(&mut |expr| {+ retained = Some(Arc::clone(expr));+ Ok(TreeNodeRecursion::Continue)+ })?;+ drop(foreign_plan);++ assert_eq!(+ retained.and_then(|expr| expr.expression_id()),+ Some(expected_id)+ );+ Ok(())+ }+#[test]fn test_ffi_execution_plan_dynamic_expressions_produced() -> Result<()> {let schema = Arc::new(arrow::datatypes::Schema::empty());diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rsindex debe0bd7bf..8e6676b6e4 100644--- a/datafusion/ffi/src/physical_expr/mod.rs+++ b/datafusion/ffi/src/physical_expr/mod.rs@@ -726,7 +726,7 @@ impl PhysicalExpr for ForeignPhysicalExpr {}fn expression_id(&self) -> Option<u64> {- unsafe { (self.expr.expression_id)(&self.expr).into() }+ unsafe { (self.expr.expression_id)(&self.expr) }.into()}}@@ -762,7 +762,9 @@ mod tests {use datafusion_expr::interval_arithmetic::Interval;#[expect(deprecated)]use datafusion_expr::statistics::Distribution;- use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};+ use datafusion_physical_expr::expressions::{+ Column, DynamicFilterPhysicalExpr, NegativeExpr, NotExpr, lit,+ };use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};use crate::physical_expr::FFI_PhysicalExpr;@@ -777,6 +779,21 @@ mod tests {(original, foreign_expr)}+ #[test]+ fn ffi_physical_expr_expression_id() {+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));+ let expected_id = dynamic_filter+ .expression_id()+ .expect("dynamic filters always have an expression ID");+ let expression: Arc<dyn PhysicalExpr> =+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter);+ let mut ffi_expr = FFI_PhysicalExpr::from(expression);+ ffi_expr.library_marker_id = crate::mock_foreign_marker_id;++ let foreign_expr: Arc<dyn PhysicalExpr> = (&ffi_expr).into();+ assert_eq!(foreign_expr.expression_id(), Some(expected_id));+ }+fn test_record_batch() -> RecordBatch {record_batch!(("a", Int32, [1, 2, 3])).unwrap()}diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex 9821c3e501..b9f353e89b 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -32,6 +32,7 @@ use arrow::array::RecordBatch;use arrow::datatypes::Schema;use async_trait::async_trait;use datafusion_catalog::{MemoryCatalogProvider, TableProvider};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, exec_err};use datafusion_execution::RecordBatchStream;use datafusion_expr::Expr;@@ -227,6 +228,15 @@ impl ExecutionPlan for AsyncTestExecutionPlan {batch_receiver: self.batch_receiver.resubscribe(),}))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl datafusion_physical_plan::DisplayAs for AsyncTestExecutionPlan {diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rsindex ad7e069546..74310d9c28 100644--- a/datafusion/ffi/src/tests/mod.rs+++ b/datafusion/ffi/src/tests/mod.rs@@ -113,6 +113,8 @@ pub struct ForeignLibraryModule {pub create_empty_exec: extern "C" fn() -> FFI_ExecutionPlan,+ pub create_exec_with_expressions: extern "C" fn() -> FFI_ExecutionPlan,+pub create_exec_with_dynamic_expressions: extern "C" fn() -> FFI_ExecutionPlan,pub create_exec_with_statistics: extern "C" fn() -> FFI_ExecutionPlan,@@ -180,6 +182,13 @@ pub(crate) extern "C" fn create_empty_exec() -> FFI_ExecutionPlan {FFI_ExecutionPlan::new(plan, None)}+pub(crate) extern "C" fn create_exec_with_expressions() -> FFI_ExecutionPlan {+ let schema = Arc::new(Schema::empty());+ let expression: Arc<dyn PhysicalExpr> = create_dynamic_filter();+ let plan = Arc::new(EmptyExec::new(schema).with_expressions(vec![expression]));+ FFI_ExecutionPlan::new(plan, None)+}+pub(crate) extern "C" fn create_exec_with_dynamic_expressions() -> FFI_ExecutionPlan {let schema = Arc::new(Schema::empty());let expression: Arc<dyn PhysicalExpr> = create_dynamic_filter();@@ -286,6 +295,7 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {create_rank_udwf: create_ffi_rank_func,create_extension_options: config::create_extension_options,create_empty_exec,+ create_exec_with_expressions,create_exec_with_dynamic_expressions,create_exec_with_statistics,create_table_with_statistics,diff --git a/datafusion/ffi/tests/ffi_execution_plan.rs b/datafusion/ffi/tests/ffi_execution_plan.rsindex a26ba9a1a6..946dca61e1 100644--- a/datafusion/ffi/tests/ffi_execution_plan.rs+++ b/datafusion/ffi/tests/ffi_execution_plan.rs@@ -21,6 +21,7 @@ mod tests {use arrow::datatypes::Schema;use arrow_schema::DataType;use datafusion_common::DataFusionError;+ use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_ffi::execution_plan::FFI_ExecutionPlan;use datafusion_ffi::execution_plan::ForeignExecutionPlan;use datafusion_ffi::execution_plan::{ExecutionPlanPrivateData, tests::EmptyExec};@@ -64,6 +65,30 @@ mod tests {Ok(())}+ #[test]+ fn test_ffi_execution_plan_expressions_cross_library() -> Result<(), DataFusionError>+ {+ let module = get_module()?;+ let plan = (module.create_exec_with_expressions)();+ let plan: Arc<dyn ExecutionPlan> = (&plan).try_into()?;+ assert!(plan.is::<ForeignExecutionPlan>());++ let mut retained = None;+ plan.apply_expressions(&mut |expr| {+ retained = Some(Arc::clone(expr));+ Ok(TreeNodeRecursion::Continue)+ })?;+ drop(plan);++ assert!(+ retained+ .as_ref()+ .and_then(|expr| expr.expression_id())+ .is_some()+ );+ Ok(())+ }+#[test]fn test_ffi_execution_plan_dynamic_expressions_cross_library()-> Result<(), DataFusionError> {diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rsindex 0fd0ad93bf..eb3d457de8 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs@@ -434,6 +434,10 @@ impl DynamicFilterPhysicalExpr {/// We check both Arc counts to handle two cases:/// - Transformed filters (via `with_new_children`) share the inner Arc (inner count > 1)/// - Direct clones (via `Arc::clone`) increment the outer count (outer count > 1)+ #[deprecated(+ since = "55.0.0",+ note = "Traverse ExecutionPlan::apply_expressions and compare PhysicalExpr::expression_id instead"+ )]pub fn is_used(self: &Arc<Self>) -> bool {// Strong count > 1 means at least one consumer is holding a reference beyond the producer.Arc::strong_count(self) > 1 || Arc::strong_count(&self.inner) > 1@@ -1091,6 +1095,10 @@ mod test {}#[test]+ #[expect(+ deprecated,+ reason = "covers the deprecated API during its retention period"+ )]fn test_is_used() {let filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![],diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rsindex e7aacb2321..10da9e4e75 100644--- a/datafusion/physical-optimizer/src/ensure_coop.rs+++ b/datafusion/physical-optimizer/src/ensure_coop.rs@@ -264,10 +264,11 @@ mod tests {// Test that cooperative context is reset when encountering an eager evaluation boundary.use arrow::datatypes::Schema;use datafusion_common::internal_err;+ use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, Partitioning, PlanProperties,+ DisplayAs, DisplayFormatType, Partitioning, PhysicalExpr, PlanProperties,SendableRecordBatchStream,execution_plan::{Boundedness, EmissionType},};@@ -345,6 +346,13 @@ mod tests {) -> Result<SendableRecordBatchStream> {internal_err!("DummyExec does not support execution")}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}// Build a plan similar to the original test:diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rsindex b9d0d06da1..fc8bf490b9 100644--- a/datafusion/physical-optimizer/src/output_requirements.rs+++ b/datafusion/physical-optimizer/src/output_requirements.rs@@ -27,7 +27,9 @@ use std::sync::Arc;use crate::PhysicalOptimizerRule;use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::{Result, Statistics, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::Distribution;@@ -324,6 +326,15 @@ impl ExecutionPlan for OutputRequirementExec {fn fetch(&self) -> Option<usize> {self.fetch}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr>,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl PhysicalOptimizerRule for OutputRequirements {diff --git a/datafusion/physical-plan/benches/compute_statistics.rs b/datafusion/physical-plan/benches/compute_statistics.rsindex 56a518c952..93c95ea4ba 100644--- a/datafusion/physical-plan/benches/compute_statistics.rs+++ b/datafusion/physical-plan/benches/compute_statistics.rs@@ -33,6 +33,7 @@ use std::sync::Arc;use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};use datafusion_common::ScalarValue;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, Statistics};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;@@ -97,6 +98,13 @@ impl ExecutionPlan for BenchLeaf {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex bae95d368e..fdf6ce323b 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -156,7 +156,9 @@ use crate::aggregates::{partial_reduce_stream::PartialReduceHashAggregateStream,single_stream::SingleHashAggregateStream,};-use crate::execution_plan::{CardinalityEffect, EmissionType};+use crate::execution_plan::{+ CardinalityEffect, EmissionType, plan_contains_expression_id,+};use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,@@ -176,6 +178,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_schema::FieldRef;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{ColumnStatistics, Constraint, Constraints, Result, ScalarValue,assert_eq_or_internal_err, internal_err, not_impl_err,@@ -2028,6 +2031,33 @@ impl ExecutionPlan for AggregateExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let group_by = self.group_by.input_exprs();+ let aggregates = self.aggr_expr.iter().flat_map(|aggr| {+ let expressions = aggr.all_expressions();+ expressions+ .args+ .into_iter()+ .chain(expressions.order_by_exprs)+ });+ let filters = self.filter_expr.iter().flatten().cloned();+ let dynamic_filter = self.dynamic_filter.iter().map(|dynamic_filter| {+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter.filter)+ as Arc<dyn PhysicalExpr>+ });+ crate::apply_expression_roots(+ group_by+ .into_iter()+ .chain(aggregates)+ .chain(filters)+ .chain(dynamic_filter),+ f,+ )+ }+fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {self.dynamic_filter.iter()@@ -2175,27 +2205,12 @@ impl ExecutionPlan for AggregateExec {if phase == FilterPushdownPhase::Post&& let Some(dyn_filter) = &self.dynamic_filter{- // let child_accepts_dyn_filter = child_pushdown_result- // .self_filters- // .first()- // .map(|filters| {- // assert_eq_or_internal_err!(- // filters.len(),- // 1,- // "Aggregate only pushdown one self dynamic filter"- // );- // let filter = filters.get(0).unwrap(); // Asserted above- // Ok(matches!(filter.discriminant, PushedDown::Yes))- // })- // .unwrap_or_else(|| internal_err!("The length of self filters equals to the number of child of this ExecutionPlan, so it must be 1"))?;-- // HACK: The above snippet should be used, however, now the child reply- // `PushDown::No` can indicate they're not able to push down row-level- // filter, but still keep the filter for statistics pruning.- // So here, we try to use ref count to determine if the dynamic filter- // has actually be pushed down.- // Issue: <https://github.com/apache/datafusion/issues/18856>- let child_accepts_dyn_filter = Arc::strong_count(dyn_filter) > 1;+ let child_accepts_dyn_filter = dyn_filter+ .filter+ .expression_id()+ .map(|id| plan_contains_expression_id(&self.input, id))+ .transpose()?+ .unwrap_or(false);if !child_accepts_dyn_filter {// Child can't consume the self dynamic filter, so disable it by setting@@ -2589,6 +2604,8 @@ impl AggregateExec {})?;aggregate.with_dynamic_filter_expr(dynamic_filter)?} else {+ let mut aggregate = aggregate;+ aggregate.dynamic_filter = None;aggregate};@@ -3092,6 +3109,7 @@ mod tests {use crate::empty::EmptyExec;use crate::execution_plan::Boundedness;use crate::expressions::col;+ use crate::filter::FilterExecBuilder;use crate::metrics::MetricValue;use crate::statistics::{StatisticsArgs, StatisticsContext};use crate::test::TestMemoryExec;@@ -3127,7 +3145,7 @@ mod tests {use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::PhysicalSortExpr;use datafusion_physical_expr::aggregate::AggregateExprBuilder;- use datafusion_physical_expr::expressions::Literal;+ use datafusion_physical_expr::expressions::{Literal, NotExpr};use crate::projection::ProjectionExec;use crate::repartition::RepartitionExec;@@ -3644,6 +3662,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -7812,6 +7837,38 @@ mod tests {Ok(())}+ #[test]+ fn test_plan_contains_expression_id_recurses_plans_and_expressions() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let empty: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![col("a", &schema)?],+ lit(true),+ ));+ let expression_id = dynamic_filter+ .expression_id()+ .expect("dynamic filters always have an expression ID");++ assert!(!plan_contains_expression_id(&empty, expression_id)?);++ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter);+ let predicate: Arc<dyn PhysicalExpr> =+ Arc::new(NotExpr::new(dynamic_filter_expr));+ let filter: Arc<dyn ExecutionPlan> =+ Arc::new(FilterExecBuilder::new(predicate, empty).build()?);+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(+ [ProjectionExpr::new_from_expression(+ col("a", &schema)?,+ &schema,+ )?],+ filter,+ )?);++ assert!(plan_contains_expression_id(&projection, expression_id)?);+ Ok(())+ }+/// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the aggregate does not support dynamic filtering#[test]fn test_with_dynamic_filter_error_unsupported() -> Result<()> {diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rsindex f2c3489c73..9a69518953 100644--- a/datafusion/physical-plan/src/analyze.rs+++ b/datafusion/physical-plan/src/analyze.rs@@ -32,11 +32,13 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::format::ExplainFormat;use datafusion_common::instant::Instant;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, assert_eq_or_internal_err, internal_err,};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::PhysicalExpr;use futures::StreamExt;@@ -219,6 +221,13 @@ impl ExecutionPlan for AnalyzeExec {])}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rsindex d89c2cc6bf..91531ec35c 100644--- a/datafusion/physical-plan/src/async_func.rs+++ b/datafusion/physical-plan/src/async_func.rs@@ -157,6 +157,19 @@ impl ExecutionPlan for AsyncFuncExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots(+ self.async_exprs+ .iter()+ .cloned()+ .map(|expr| expr as Arc<dyn PhysicalExpr>),+ f,+ )+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rsindex 3be331a1ee..a1c3c7ea01 100644--- a/datafusion/physical-plan/src/buffer.rs+++ b/datafusion/physical-plan/src/buffer.rs@@ -32,6 +32,7 @@ use crate::{};use arrow::array::RecordBatch;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, Statistics, internal_err, plan_err};use datafusion_common_runtime::SpawnedTask;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};@@ -159,6 +160,13 @@ impl ExecutionPlan for BufferExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex c5b9176777..511e5d793b 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -34,6 +34,7 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::TaskContext;use datafusion_physical_expr::PhysicalExpr;@@ -173,6 +174,13 @@ impl ExecutionPlan for CoalesceBatchesExec {vec![false]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex f9694e0d16..5cd7a707c2 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -35,6 +35,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_properuse datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::PhysicalExpr;@@ -143,6 +144,13 @@ impl ExecutionPlan for CoalescePartitionsExec {vec![false]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex a5b57f546b..dc7d98891e 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -71,6 +71,7 @@//! that report [`SchedulingType::NonCooperative`] in their [plan properties](ExecutionPlan::properties).use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_physical_expr::PhysicalExpr;#[cfg(datafusion_coop = "tokio_fallback")]use futures::Future;@@ -267,6 +268,13 @@ impl ExecutionPlan for CooperativeExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rsindex 34493a5f51..2370e3e6ce 100644--- a/datafusion/physical-plan/src/display.rs+++ b/datafusion/physical-plan/src/display.rs@@ -1504,8 +1504,11 @@ mod tests {use std::fmt::Write;use std::sync::Arc;- use datafusion_common::{Result, Statistics, internal_datafusion_err};+ use datafusion_common::{+ Result, Statistics, internal_datafusion_err, tree_node::TreeNodeRecursion,+ };use datafusion_execution::{SendableRecordBatchStream, TaskContext};+ use datafusion_physical_expr::PhysicalExpr;use crate::statistics::StatisticsArgs;use crate::{DisplayAs, ExecutionPlan, PlanProperties};@@ -1549,6 +1552,13 @@ mod tests {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_: usize,@@ -1706,6 +1716,17 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![&self.inner]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &Arc<dyn PhysicalExpr>,+ ) -> Result<+ datafusion_common::tree_node::TreeNodeRecursion,+ >,+ ) -> Result<datafusion_common::tree_node::TreeNodeRecursion>+ {+ Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rsindex 3bd38bf238..bd91ec742d 100644--- a/datafusion/physical-plan/src/empty.rs+++ b/datafusion/physical-plan/src/empty.rs@@ -29,9 +29,10 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use crate::execution_plan::SchedulingType;use crate::statistics::StatisticsArgs;@@ -119,6 +120,13 @@ impl ExecutionPlan for EmptyExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex bc4ece5adf..763ec2f7df 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -28,7 +28,9 @@ pub use crate::stream::EmptyRecordBatchStream;use arrow_schema::Schema;pub use datafusion_common::hash_utils;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};pub use datafusion_common::utils::project_schema;pub use datafusion_common::{ColumnStatistics, Statistics, internal_err};pub use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};@@ -39,6 +41,7 @@ pub use datafusion_physical_expr::{};use std::any::Any;+use std::borrow::Borrow;use std::collections::HashSet;use std::fmt::Debug;use std::sync::{Arc, LazyLock};@@ -264,6 +267,70 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {/// joins).fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>>;+ /// Apply a closure `f` to each root expression that this node owns and uses+ /// during execution, either by evaluating it or updating it dynamically.+ ///+ /// An expression must not be visited solely because it describes an input or+ /// output property, such as cached ordering, partitioning, or equivalence+ /// metadata. However, these may be traversed indirectly. For example,+ /// `RepartitionExec` visits the partitioning expressions it evaluates and+ /// `SortExec` visits the sort expressions it evaluates to order rows.+ ///+ /// This method is shallow: it must not visit expression children or expressions+ /// owned by child execution plans.+ ///+ /// Similarly to other [`TreeNode`] APIs, the closure can return+ /// [`TreeNodeRecursion::Stop`] to stop iteration, otherwise iteration+ /// should continue. Note that [`TreeNodeRecursion::Continue`] and+ /// [`TreeNodeRecursion::Jump`] are equivalent because this method is not+ /// recursive.+ ///+ ///+ /// # Example Usage+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_physical_plan::ExecutionPlan;+ /// # use datafusion_common::tree_node::TreeNodeRecursion;+ /// # fn example(plan: Arc<dyn ExecutionPlan>) -> datafusion_common::Result<()> {+ /// // Count the number of expressions+ /// let mut count = 0;+ /// plan.apply_expressions(&mut |_expr| {+ /// count += 1;+ /// Ok(TreeNodeRecursion::Continue)+ /// })?;+ /// # Ok(())+ /// # }+ /// ```+ ///+ /// # Implementation Examples+ ///+ /// ## Node with expressions (e.g., FilterExec, ProjectionExec)+ ///+ /// Use [`apply_expression_roots`] to implement this method. It abstracts away the+ /// [`TreeNodeRecursion`] iteration from implementors.+ /// ```ignore+ /// fn apply_expressions(+ /// &self,+ /// f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ /// ) -> Result<TreeNodeRecursion> {+ /// apply_expression_roots([&self.predicate], f)+ /// }+ /// ```+ ///+ /// ## Node with no expressions (e.g., EmptyExec, MemoryExec)+ /// ```ignore+ /// fn apply_expressions(+ /// &self,+ /// _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ /// ) -> Result<TreeNodeRecursion> {+ /// Ok(TreeNodeRecursion::Continue)+ /// }+ /// ```+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;+/// Returns a new `ExecutionPlan` where all existing children were replaced/// by the `children`, in orderfn with_new_children(@@ -869,6 +936,58 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {}}+/// Applies `f` to a shallow sequence of physical expression roots.+///+/// [`TreeNodeRecursion::Stop`] stops iteration and is returned immediately.+/// [`TreeNodeRecursion::Jump`] is normalized to [`TreeNodeRecursion::Continue`]+/// because this function does not visit expression children.+pub fn apply_expression_roots<I>(+ roots: I,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+) -> Result<TreeNodeRecursion>+where+ I: IntoIterator,+ I::Item: Borrow<Arc<dyn PhysicalExpr>>,+{+ for root in roots {+ match f(root.borrow())? {+ TreeNodeRecursion::Stop => return Ok(TreeNodeRecursion::Stop),+ TreeNodeRecursion::Continue | TreeNodeRecursion::Jump => {}+ }+ }+ Ok(TreeNodeRecursion::Continue)+}++/// Returns whether `plan` contains a physical expression with `expression_id`.+///+/// This traverses both the execution plan and the children of each expression root+/// reported by [`ExecutionPlan::apply_expressions`].+pub(crate) fn plan_contains_expression_id(+ plan: &Arc<dyn ExecutionPlan>,+ expression_id: u64,+) -> Result<bool> {+ let mut found = false;+ plan.apply(|node| {+ node.apply_expressions(&mut |root| {+ root.apply(|expr| {+ if expr.expression_id() == Some(expression_id) {+ found = true;+ Ok(TreeNodeRecursion::Stop)+ } else {+ Ok(TreeNodeRecursion::Continue)+ }+ })+ })?;++ Ok(if found {+ TreeNodeRecursion::Stop+ } else {+ TreeNodeRecursion::Continue+ })+ })?;+ Ok(found)+}+impl dyn ExecutionPlan {/// Returns `true` if the plan is of type `T`.///@@ -1824,6 +1943,13 @@ mod tests {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {self.dynamic_expressions.iter().map(Arc::clone).collect()}@@ -1910,6 +2036,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -1960,6 +2093,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ self.0.apply_expressions(f)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -2019,6 +2159,12 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -2082,6 +2228,12 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -2173,6 +2325,12 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -2314,6 +2472,171 @@ mod tests {Ok(())}+ /// A test node that holds a fixed list of expressions, used to test+ /// `apply_expressions` behavior.+ #[derive(Debug)]+ struct MultiExprExec {+ exprs: Vec<Arc<dyn PhysicalExpr>>,+ children: Vec<Arc<dyn ExecutionPlan>>,+ }++ impl DisplayAs for MultiExprExec {+ fn fmt_as(+ &self,+ _t: DisplayFormatType,+ _f: &mut std::fmt::Formatter,+ ) -> std::fmt::Result {+ unimplemented!()+ }+ }++ impl ExecutionPlan for MultiExprExec {+ fn name(&self) -> &'static str {+ "MultiExprExec"+ }++ fn properties(&self) -> &Arc<PlanProperties> {+ unimplemented!()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ self.children.iter().collect()+ }++ fn with_new_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ unimplemented!()+ }++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ apply_expression_roots(&self.exprs, f)+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ unimplemented!()+ }++ fn partition_statistics(+ &self,+ _partition: Option<usize>,+ ) -> Result<Arc<Statistics>> {+ unimplemented!()+ }+ }++ /// Returns a simple literal `Arc<dyn PhysicalExpr>` for use in tests.+ fn lit_expr(val: i64) -> Arc<dyn PhysicalExpr> {+ use datafusion_physical_expr::expressions::Literal;+ Arc::new(Literal::new(datafusion_common::ScalarValue::Int64(Some(+ val,+ ))))+ }++ /// `apply_expressions` visits all expressions when `f` always returns `Continue`.+ #[test]+ fn test_apply_expressions_continue_visits_all() -> Result<()> {+ let plan = MultiExprExec {+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],+ children: vec![],+ };+ let mut visited = 0usize;+ plan.apply_expressions(&mut |_expr| {+ visited += 1;+ Ok(TreeNodeRecursion::Continue)+ })?;+ assert_eq!(visited, 3);+ Ok(())+ }++ #[test]+ fn test_apply_expressions_stop_halts_early() -> Result<()> {+ let plan = MultiExprExec {+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],+ children: vec![],+ };+ let mut visited = 0usize;+ let tnr = plan.apply_expressions(&mut |_expr| {+ visited += 1;+ Ok(TreeNodeRecursion::Stop)+ })?;+ // Only the first expression is visited; the rest are skipped.+ assert_eq!(visited, 1);+ assert_eq!(tnr, TreeNodeRecursion::Stop);+ Ok(())+ }++ #[test]+ fn test_apply_expressions_jump_visits_next_root() -> Result<()> {+ let plan = MultiExprExec {+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],+ children: vec![],+ };+ let mut visited = 0usize;+ let tnr = plan.apply_expressions(&mut |_expr| {+ visited += 1;+ Ok(TreeNodeRecursion::Jump)+ })?;+ assert_eq!(visited, 3);+ assert_eq!(tnr, TreeNodeRecursion::Continue);+ Ok(())+ }++ #[test]+ fn test_apply_expressions_does_not_recurse() -> Result<()> {+ use datafusion_physical_expr::expressions::NegativeExpr;++ let child: Arc<dyn ExecutionPlan> = Arc::new(MultiExprExec {+ exprs: vec![lit_expr(2)],+ children: vec![],+ });+ let nested: Arc<dyn PhysicalExpr> = Arc::new(NegativeExpr::new(lit_expr(1)));+ let plan = MultiExprExec {+ exprs: vec![nested],+ children: vec![child],+ };++ let mut visited = 0;+ plan.apply_expressions(&mut |expr| {+ visited += 1;+ assert!(expr.is::<NegativeExpr>());+ Ok(TreeNodeRecursion::Continue)+ })?;+ assert_eq!(visited, 1);+ Ok(())+ }++ #[test]+ fn test_apply_expressions_callback_can_retain_arc() -> Result<()> {+ let expected = lit_expr(1);+ let plan = MultiExprExec {+ exprs: vec![Arc::clone(&expected)],+ children: vec![],+ };+ let mut retained = None;+ plan.apply_expressions(&mut |expr| {+ retained = Some(Arc::clone(expr));+ Ok(TreeNodeRecursion::Continue)+ })?;+ drop(plan);++ assert!(Arc::ptr_eq(+ &expected,+ retained+ .as_ref()+ .expect("callback should retain expression")+ ));+ Ok(())+ }+#[test]fn test_execution_plan_name() {let schema1 = Arc::new(Schema::empty());diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rsindex a270a003eb..72ceae81f3 100644--- a/datafusion/physical-plan/src/explain.rs+++ b/datafusion/physical-plan/src/explain.rs@@ -26,9 +26,10 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::display::StringifiedPlan;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use log::trace;@@ -116,6 +117,13 @@ impl ExecutionPlan for ExplainExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex 50c8246b37..65abfa259d 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -55,6 +55,7 @@ use arrow::record_batch::RecordBatch;use datafusion_common::cast::as_boolean_array;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, ScalarValue, internal_err, plan_err, project_schema,};@@ -539,6 +540,13 @@ impl ExecutionPlan for FilterExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots([&self.predicate], f)+ }+fn maintains_input_order(&self) -> Vec<bool> {// Tell optimizer this operator doesn't reorder its inputvec![true]diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 16155aaafd..524887bf03 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -43,11 +43,13 @@ use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::compute::concat_batches;use arrow::datatypes::{Fields, Schema, SchemaRef};use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinType, Result, ScalarValue, assert_eq_or_internal_err, internal_err,};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::equivalence::join_equivalence_properties;use async_trait::async_trait;@@ -266,6 +268,14 @@ impl ExecutionPlan for CrossJoinExec {Some(self.metrics.clone_inner())}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // CrossJoin has no join conditions or expressions+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex e26df9a4ae..4e68d871b8 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -25,7 +25,7 @@ use std::vec;use crate::ExecutionPlanProperties;use crate::execution_plan::{EmissionType, boundedness_from_children, has_same_children_properties,- stub_properties,+ plan_contains_expression_id, stub_properties,};use crate::filter_pushdown::{ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,@@ -73,6 +73,7 @@ use arrow::record_batch::RecordBatch;use arrow::util::bit_util;use arrow_schema::{DataType, Schema};use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::memory::{RecordBatchMemoryCounter, estimate_memory_size};use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,@@ -1325,6 +1326,25 @@ impl ExecutionPlan for HashJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let join_keys = self+ .on+ .iter()+ .flat_map(|(left, right)| [Arc::clone(left), Arc::clone(right)]);+ let filter = self+ .filter+ .iter()+ .map(|filter| Arc::clone(filter.expression()));+ let dynamic_filter = self.dynamic_filter.iter().map(|dynamic_filter| {+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter.filter)+ as Arc<dyn PhysicalExpr>+ });+ crate::apply_expression_roots(join_keys.chain(filter).chain(dynamic_filter), f)+ }+fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {self.dynamic_filter.iter()@@ -1377,18 +1397,20 @@ impl ExecutionPlan for HashJoinExec {consider using CoalescePartitionsExec or the EnforceDistribution rule");- // Only enable dynamic filter pushdown if:- // - The session config enables dynamic filter pushdown- // - A dynamic filter exists- // - At least one consumer is holding a reference to it, this avoids expensive filter- // computation when disabled or when no consumer will use it.- let enable_dynamic_filter_pushdown = self+ // Only compute a dynamic filter when the probe subtree contains a consumer.+ // Searching from `self` would always find the producer expression owned by this join.+ let enable_dynamic_filter_pushdown = if self.allow_join_dynamic_filter_pushdown(context.session_config().options())- && self- .dynamic_filter+ {+ self.dynamic_filter.as_ref()- .map(|df| df.filter.is_used())- .unwrap_or(false);+ .and_then(|df| df.filter.expression_id())+ .map(|id| plan_contains_expression_id(&self.right, id))+ .transpose()?+ .unwrap_or(false)+ } else {+ false+ };let join_metrics = BuildProbeJoinMetrics::new(partition, &self.metrics);@@ -2411,6 +2433,7 @@ mod tests {use crate::coalesce_partitions::CoalescePartitionsExec;use crate::execution_plan::Boundedness;+ use crate::filter::FilterExecBuilder;use crate::joins::hash_join::stream::lookup_join_hashmap;use crate::test::{TestMemoryExec, assert_join_metrics};use crate::{@@ -2478,6 +2501,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -2626,7 +2656,12 @@ mod tests {Arc::new(Column::new_with_schema("b1", &right_schema).unwrap()) as _,)];let right: Arc<dyn ExecutionPlan> = Arc::new(- MockExec::new(vec![Ok(right_batch), err], right_schema).with_use_task(false),+ MockExec::new(vec![Ok(right_batch), err], right_schema)+ .with_use_task(false)+ // The planted error must only surface if the probe side is+ // polled, not when a parent node computes statistics during+ // planning.+ .with_unknown_statistics(),);(left, right, on)@@ -2706,6 +2741,8 @@ mod tests {mode: PartitionMode,) -> Result<(HashJoinExec, Arc<DynamicFilterPhysicalExpr>)> {let dynamic_filter = HashJoinExec::create_dynamic_filter(&on);+ let consumer: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;+ let right = Arc::new(FilterExecBuilder::new(consumer, right).build()?);let mut join = HashJoinExec::try_new(left,right,diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 515dcc2931..7069a8b448 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -61,6 +61,7 @@ use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_schema::DataType;use datafusion_common::cast::as_boolean_array;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, NullEquality, Result, ScalarValue, Statistics, arrow_err,assert_eq_or_internal_err, internal_datafusion_err, internal_err, project_schema,@@ -562,6 +563,17 @@ impl ExecutionPlan for NestedLoopJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to join filter expressions if present+ crate::apply_expression_roots(+ self.filter.iter().map(|filter| filter.expression()),+ f,+ )+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rsindex 5ec564295e..b60ec1c784 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs@@ -23,6 +23,7 @@ use arrow::{};use arrow_schema::{SchemaRef, SortOptions};use datafusion_common::not_impl_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, Result, internal_err};use datafusion_execution::{SendableRecordBatchStream,@@ -482,6 +483,14 @@ impl ExecutionPlan for PiecewiseMergeJoinExec {vec![&self.buffered, &self.streamed]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to the two expressions being compared in the range predicate+ crate::apply_expression_roots([&self.on.0, &self.on.1], f)+ }+fn required_input_distribution(&self) -> Vec<Distribution> {self.input_distribution_requirements().into_per_child()}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex 1abcd9d6c7..55a4b2136c 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -47,6 +47,7 @@ use crate::{use arrow::compute::SortOptions;use arrow::datatypes::SchemaRef;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err, internal_err,plan_err,@@ -439,6 +440,15 @@ impl ExecutionPlan for SortMergeJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let join_keys = self.on.iter().flat_map(|(left, right)| [left, right]);+ let filter = self.filter.iter().map(|filter| filter.expression());+ crate::apply_expression_roots(join_keys.chain(filter), f)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex eb358b10b4..86f721b711 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -66,6 +66,7 @@ use arrow::compute::concat_batches;use arrow::datatypes::{ArrowNativeType, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::hash_utils::create_hashes;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::bisect;use datafusion_common::{HashSet, JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err,@@ -452,6 +453,15 @@ impl ExecutionPlan for SymmetricHashJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let join_keys = self.on.iter().flat_map(|(left, right)| [left, right]);+ let filter = self.filter.iter().map(|filter| filter.expression());+ crate::apply_expression_roots(join_keys.chain(filter), f)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rsindex 8cba650b79..6e1df1f840 100644--- a/datafusion/physical-plan/src/lib.rs+++ b/datafusion/physical-plan/src/lib.rs@@ -45,9 +45,9 @@ pub use crate::distribution_requirements::{ChildSatisfactionOptions, InputDistributionRequirements,};pub use crate::execution_plan::{- ExecutionPlan, ExecutionPlanProperties, PlanProperties, collect, collect_partitioned,- displayable, execute_input_stream, execute_stream, execute_stream_partitioned,- get_plan_string, with_new_children_if_necessary,+ ExecutionPlan, ExecutionPlanProperties, PlanProperties, apply_expression_roots,+ collect, collect_partitioned, displayable, execute_input_stream, execute_stream,+ execute_stream_partitioned, get_plan_string, with_new_children_if_necessary,};pub use crate::metrics::Metric;pub use crate::ordering::InputOrderMode;diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex ddce680fc1..5bc5bc48f1 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -35,10 +35,11 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::LexOrdering;+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use futures::stream::{Stream, StreamExt};use log::trace;@@ -167,6 +168,13 @@ impl ExecutionPlan for GlobalLimitExec {vec![false]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -398,6 +406,13 @@ impl ExecutionPlan for LocalLimitExec {vec![true]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -648,7 +663,6 @@ mod tests {use arrow::array::RecordBatchOptions;use arrow::datatypes::Schema;use datafusion_common::stats::Precision;- use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::expressions::col;#[tokio::test]diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex ad54905f47..eb141b8c70 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -32,10 +32,11 @@ use crate::{use arrow::array::RecordBatch;use arrow::datatypes::SchemaRef;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, assert_or_internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryReservation;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use futures::Stream;@@ -311,6 +312,13 @@ impl ExecutionPlan for LazyMemoryExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rsindex 142768fcf4..ec54201e7b 100644--- a/datafusion/physical-plan/src/operator_statistics/mod.rs+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs@@ -1038,6 +1038,7 @@ mod tests {use std::fmt;use crate::execution_plan::{Boundedness, EmissionType};+ use datafusion_common::tree_node::TreeNodeRecursion;fn make_schema() -> Arc<Schema> {Arc::new(Schema::new(vec![@@ -1117,6 +1118,13 @@ mod tests {&self.cache}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,@@ -1219,6 +1227,13 @@ mod tests {self.input.properties()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rsindex 5d71058269..de07529bab 100644--- a/datafusion/physical-plan/src/placeholder_row.rs+++ b/datafusion/physical-plan/src/placeholder_row.rs@@ -29,9 +29,11 @@ use crate::{use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_or_internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::PhysicalExpr;use crate::statistics::StatisticsArgs;use log::trace;@@ -136,6 +138,13 @@ impl ExecutionPlan for PlaceholderRowExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex fac837b09f..32d444fba2 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -330,6 +330,20 @@ impl ExecutionPlan for ProjectionExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots(+ self.projector+ .projection()+ .as_ref()+ .iter()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex 00df227cb8..4c6f0493ad 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -37,12 +37,14 @@ use arrow::array::{BooleanArray, BooleanBuilder};use arrow::compute::filter_record_batch;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err, not_impl_err,};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use futures::{Stream, StreamExt, ready};@@ -153,6 +155,13 @@ impl ExecutionPlan for RecursiveQueryExec {vec![&self.static_term, &self.recursive_term]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+// TODO: control these hints and see whether we can// infer some from the child plans (static/recursive terms).fn maintains_input_order(&self) -> Vec<bool> {diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 873f35fd6a..2863524f16 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -53,6 +53,7 @@ use arrow::datatypes::{SchemaRef, UInt32Type};use arrow_schema::SortOptions;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, transpose};use datafusion_common::{ColumnStatistics, DataFusionError, HashMap, ScalarValue, SplitPoint,@@ -1337,6 +1338,20 @@ impl ExecutionPlan for RepartitionExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ match self.partitioning() {+ Partitioning::Hash(exprs, _) => crate::apply_expression_roots(exprs, f),+ Partitioning::Range(range) => crate::apply_expression_roots(+ range.ordering().iter().map(|sort_expr| &sort_expr.expr),+ f,+ ),+ _ => Ok(TreeNodeRecursion::Continue),+ }+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -2934,6 +2949,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -4168,6 +4190,13 @@ mod test {)?);let exec = RepartitionExec::try_new(source, partitioning)?;+ let mut expressions = vec![];+ exec.apply_expressions(&mut |expr| {+ expressions.push(expr.to_string());+ Ok(TreeNodeRecursion::Continue)+ })?;+ assert_eq!(expressions, ["c0@0"]);+// Range partition count is fixed by split points, so repartitioned()// cannot change it to an arbitrary target.let result = exec.repartitioned(10, &Default::default())?;diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rsindex 73acb2ab13..ee3c2e5d07 100644--- a/datafusion/physical-plan/src/scalar_subquery.rs+++ b/datafusion/physical-plan/src/scalar_subquery.rs@@ -27,9 +27,11 @@use std::fmt;use std::sync::Arc;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, ScalarValue, Statistics, exec_err, internal_err};use datafusion_execution::TaskContext;use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};+use datafusion_physical_expr::PhysicalExpr;use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};use crate::joins::utils::{OnceAsync, OnceFut};@@ -224,6 +226,13 @@ impl ExecutionPlan for ScalarSubqueryExec {)))}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn maintains_input_order(&self) -> Vec<bool> {// Only the main input (first child); subquery children don't contribute// to ordering.@@ -453,6 +462,13 @@ mod tests {)))}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rsindex 44e2b20adc..5f15f8b6cb 100644--- a/datafusion/physical-plan/src/sorts/partial_sort.rs+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs@@ -70,9 +70,10 @@ use arrow::compute::concat_batches;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::evaluate_partition_ranges;use datafusion_execution::{RecordBatchStream, TaskContext};-use datafusion_physical_expr::LexOrdering;+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use futures::{Stream, StreamExt, ready};use log::trace;@@ -374,6 +375,16 @@ impl ExecutionPlan for PartialSortExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots(+ self.expr.iter().map(|sort_expr| &sort_expr.expr),+ f,+ )+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rsindex 730440a429..78dd9b9696 100644--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs@@ -37,6 +37,7 @@ use std::sync::Arc;use arrow::datatypes::SchemaRef;use arrow::row::SortField;use datafusion_common::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::TaskContext;use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_physical_expr::PhysicalExpr;@@ -379,6 +380,16 @@ impl ExecutionPlan for PartitionedTopKExec {)?))}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots(+ self.expr.iter().map(|sort_expr| &sort_expr.expr),+ f,+ )+ }+fn execute(&self,partition: usize,diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex b1b6a84fd9..5c6b86acc5 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -60,6 +60,7 @@ use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};use arrow::datatypes::SchemaRef;use datafusion_common::config::SpillCompression;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,unwrap_or_internal_err,@@ -1278,6 +1279,23 @@ impl ExecutionPlan for SortExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let dynamic_filter = self+ .filter+ .as_ref()+ .map(|filter| filter.read().expr() as Arc<dyn PhysicalExpr>);+ crate::apply_expression_roots(+ self.expr+ .iter()+ .map(|sort_expr| &sort_expr.expr)+ .chain(dynamic_filter.iter()),+ f,+ )+ }+fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {self.filter.iter()@@ -1770,6 +1788,13 @@ mod tests {Ok(self)}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 2add3e1eb8..ac6f5d18cd 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -31,9 +31,11 @@ use crate::{check_if_same_properties,};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryConsumer;+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};@@ -281,6 +283,16 @@ impl ExecutionPlan for SortPreservingMergeExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ crate::apply_expression_roots(+ self.expr.iter().map(|sort_expr| &sort_expr.expr),+ f,+ )+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -1573,6 +1585,12 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rsindex 61a9b9cc6d..a82f8d9441 100644--- a/datafusion/physical-plan/src/streaming.rs+++ b/datafusion/physical-plan/src/streaming.rs@@ -33,8 +33,10 @@ use crate::stream::RecordBatchStreamAdapter;use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};use arrow::datatypes::{Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, internal_err, plan_err};use datafusion_execution::TaskContext;+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::projection::ProjectionMapping;use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};@@ -271,6 +273,13 @@ impl ExecutionPlan for StreamingTableExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex e8c775a786..68e6ff7eca 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -36,6 +36,7 @@ use crate::{DisplayAs, DisplayFormatType, PlanProperties};use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};use arrow_schema::{DataType, Field, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, Statistics, assert_or_internal_err, config::ConfigOptions, project_schema,};@@ -45,7 +46,9 @@ use datafusion_physical_expr::equivalence::{};use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr::utils::collect_columns;-use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, Partitioning};+use datafusion_physical_expr::{+ EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr,+};use futures::{Future, FutureExt};@@ -138,6 +141,13 @@ impl ExecutionPlan for TestMemoryExec {Vec::new()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rsindex b92008c6b2..9a3f05a6e0 100644--- a/datafusion/physical-plan/src/test/exec.rs+++ b/datafusion/physical-plan/src/test/exec.rs@@ -35,9 +35,10 @@ use std::{use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use futures::Stream;use tokio::sync::Barrier;@@ -124,6 +125,9 @@ pub struct MockExec {/// if true (the default), sends data using a separate task to ensure the/// batches are not available without this stream yielding firstuse_task: bool,+ /// if true, report unknown statistics instead of deriving them from+ /// `data` (which propagates any planted errors at planning time)+ unknown_statistics: bool,cache: Arc<PlanProperties>,}@@ -141,6 +145,7 @@ impl MockExec {data,schema,use_task: true,+ unknown_statistics: false,cache: Arc::new(cache),}}@@ -153,6 +158,17 @@ impl MockExec {self}+ /// Report unknown statistics rather than computing them from `data`.+ ///+ /// By default statistics are derived from `data`, which propagates any+ /// planted errors when statistics are requested during planning (for+ /// example when a parent node computes its properties). Use this when a+ /// planted error should only surface at execution time.+ pub fn with_unknown_statistics(mut self) -> Self {+ self.unknown_statistics = true;+ self+ }+/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.fn compute_properties(schema: SchemaRef) -> PlanProperties {PlanProperties::new(@@ -195,6 +211,13 @@ impl ExecutionPlan for MockExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -248,13 +271,14 @@ impl ExecutionPlan for MockExec {}}- // Panics if one of the batches is an error+ // Errors if one of the batches is an error, unless+ // `with_unknown_statistics` was usedfn statistics_from_inputs(&self,_input_stats: &[Arc<Statistics>],args: &StatisticsArgs,) -> Result<Arc<Statistics>> {- if args.partition().is_some() {+ if self.unknown_statistics || args.partition().is_some() {return Ok(Arc::new(Statistics::new_unknown(&self.schema)));}let data: Result<Vec<_>> = self@@ -432,6 +456,13 @@ impl ExecutionPlan for BarrierExec {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Returns a stream which yields datafn execute(&self,@@ -568,6 +599,13 @@ impl ExecutionPlan for ErrorExec {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Returns a stream which yields datafn execute(&self,@@ -647,6 +685,13 @@ impl ExecutionPlan for StatisticsExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -758,6 +803,13 @@ impl ExecutionPlan for BlockingExec {internal_err!("Children cannot be replaced in {self:?}")}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,@@ -893,6 +945,13 @@ impl ExecutionPlan for PanicExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 8d77556509..88d2628ae2 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -51,6 +51,7 @@ use arrow::datatypes::{Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::NdvFallback;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_or_internal_err, exec_err, internal_datafusion_err,};@@ -330,6 +331,13 @@ impl ExecutionPlan for UnionExec {self.inputs.iter().collect()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -703,6 +711,13 @@ impl ExecutionPlan for InterleaveExec {vec![false; self.inputs().len()]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex ababdd36a9..1877f668c5 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -44,6 +44,7 @@ use arrow::datatypes::{DataType, Int64Type, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_ord::cmp::lt;use async_trait::async_trait;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, HashMap, HashSet, Result, UnnestOptions, exec_datafusion_err, exec_err,internal_err,@@ -227,6 +228,13 @@ impl ExecutionPlan for UnnestExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex 2ca7187faf..e9a0d9f47c 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -50,6 +50,7 @@ use arrow::{};use datafusion_common::hash_utils::create_hashes;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::{evaluate_partition_ranges, get_at_indices, get_row_at_idx,};@@ -312,6 +313,21 @@ impl ExecutionPlan for BoundedWindowAggExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let expressions = self.window_expr.iter().flat_map(|window_expr| {+ let expressions = window_expr.all_expressions();+ expressions+ .args+ .into_iter()+ .chain(expressions.partition_by_exprs)+ .chain(expressions.order_by_exprs)+ });+ crate::apply_expression_roots(expressions, f)+ }+fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {let partition_bys = self.window_expr()[0].partition_by();let order_keys = self.window_expr()[0].order_by();diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex 025f642f90..bbf9a14fd5 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -45,6 +45,7 @@ use arrow::datatypes::SchemaRef;use arrow::error::ArrowError;use arrow::record_batch::RecordBatch;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::{evaluate_partition_ranges, transpose};use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;@@ -214,6 +215,21 @@ impl ExecutionPlan for WindowAggExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let expressions = self.window_expr.iter().flat_map(|window_expr| {+ let expressions = window_expr.all_expressions();+ expressions+ .args+ .into_iter()+ .chain(expressions.partition_by_exprs)+ .chain(expressions.order_by_exprs)+ });+ crate::apply_expression_roots(expressions, f)+ }+fn maintains_input_order(&self) -> Vec<bool> {vec![true]}diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rsindex c92face1e5..83cd0a15a6 100644--- a/datafusion/physical-plan/src/work_table.rs+++ b/datafusion/physical-plan/src/work_table.rs@@ -32,10 +32,11 @@ use crate::{use crate::statistics::StatisticsArgs;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryReservation;-use datafusion_physical_expr::{EquivalenceProperties, Partitioning};+use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};/// A vector of record batches with a memory reservation.#[derive(Debug)]@@ -186,6 +187,13 @@ impl ExecutionPlan for WorkTableExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 1252668f57..de684857f4 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -171,6 +171,22 @@ mod file_scan_config_serde {"serde-test"}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &Arc<dyn PhysicalExpr>,+ )+ -> Result<datafusion_common::tree_node::TreeNodeRecursion>,+ ) -> Result<datafusion_common::tree_node::TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(+ self.projection+ .iter()+ .flatten()+ .map(|proj_expr| &proj_expr.expr),+ f,+ )+ }+fn try_pushdown_projection(&self,projection: &FileProjectionExprs,diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 99ba5cf0ab..19249cf97b 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -108,6 +108,7 @@ use datafusion_common::file_options::json_writer::JsonWriterOptions;use datafusion_common::format::ExplainFormat;use datafusion_common::parsers::CompressionTypeVariant;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, JoinSide, NullEquality, Result, UnnestOptions, exec_datafusion_err,internal_datafusion_err, internal_err, not_impl_err,@@ -323,6 +324,13 @@ impl ExecutionPlan for DowncastDelegatingExec {self.inner.children()}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ self.inner.apply_expressions(f)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -4934,6 +4942,61 @@ fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {Ok(())}+#[test]+fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let aggregate = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![+ AggregateExprBuilder::new(+ datafusion::functions_aggregate::min_max::min_udaf(),+ vec![col_a],+ )+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()+ .map(Arc::new)?,+ ],+ vec![None],+ child,+ Arc::clone(&schema),+ )?) as Arc<dyn ExecutionPlan>;++ let mut config = ConfigOptions::default();+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(aggregate, &config)?;+ assert!(+ plan.downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec")+ .dynamic_expressions_produced()+ .is_empty()+ );++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DefaultPhysicalProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ assert!(+ deserialized+ .downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec")+ .dynamic_expressions_produced()+ .is_empty()+ );+ Ok(())+}+/// Test that plan containing a SortExec with dynamic filter pushdown/// can be serialized and deserialized while preserving references to the dynamic filter.#[test]@@ -5054,6 +5117,13 @@ impl ExecutionPlan for CustomExecWithExprs {vec![&self.child]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ datafusion_physical_plan::apply_expression_roots(&self.exprs, f)+ }+fn with_new_children(self: Arc<Self>,_children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.mdindex 540782e3e8..c6a316aa74 100644--- a/docs/source/library-user-guide/custom-table-providers.md+++ b/docs/source/library-user-guide/custom-table-providers.md@@ -766,6 +766,7 @@ impl DatePartitionedTable {# fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { vec![] }# fn with_new_children(self: Arc<Self>, _: Vec<Arc<dyn ExecutionPlan>>) -> Result<Arc<dyn ExecutionPlan>> { Ok(self) }# fn execute(&self, _: usize, _: Arc<TaskContext>) -> Result<SendableRecordBatchStream> { todo!() }+# fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }# }```@@ -909,6 +910,13 @@ impl ExecutionPlan for CountingExec {batch_stream,)))}++# fn apply_expressions(+# &self,+# _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,+# ) -> Result<TreeNodeRecursion> {+# Ok(TreeNodeRecursion::Continue)+# }}```diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex 2811fb4df2..73eb3dfd86 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -986,6 +986,11 @@ let plan = deserialize_bytes(&proto_bytes)?;See [PR #23827](https://github.com/apache/datafusion/pull/23827) for details.+### `ExecutionPlan::apply_expressions` is now a required method++`apply_expressions` has been added as a **required** method on the `ExecutionPlan`, `FileSource`, and `DataSource` traits. Any custom implementation of+these traits must now implement `apply_expressions`. See docs on `ExecutionPlan::apply_expressions` for migration details.+### `WindowExpr::evaluate_stateful` now takes a `WindowEvalContext``WindowExpr::evaluate_stateful` (and the provided