Apache DataFusion SQL Query Engine
Add `ExecutionPlan::apply_expressions()` (#20337)
Lía Adrianaa5f490e4c311
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rsindex 7abb39e1a7..71e589dcf6 100644--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs@@ -27,6 +27,7 @@ use async_trait::async_trait;use datafusion::arrow::array::{UInt8Builder, UInt64Builder};use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::arrow::record_batch::RecordBatch;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::datasource::{TableProvider, TableType, provider_as_source};use datafusion::error::Result;use datafusion::execution::context::TaskContext;@@ -283,4 +284,20 @@ impl ExecutionPlan for CustomExec {None,)?))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rsindex 4c05cd2fb1..1440347d44 100644--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs@@ -29,6 +29,7 @@use arrow::record_batch::RecordBatch;use arrow_schema::SchemaRef;use datafusion::common::record_batch;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::common::{exec_datafusion_err, internal_err};use datafusion::datasource::{DefaultTableSource, memory::MemTable};use datafusion::error::Result;@@ -296,4 +297,20 @@ impl ExecutionPlan for BufferingExecutionPlan {}),)))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.properties.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rsindex b4f3d4f098..df3d58b7bf 100644--- a/datafusion-examples/examples/proto/composed_extension_codec.rs+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs@@ -38,6 +38,7 @@ use std::sync::Arc;use datafusion::common::Result;use datafusion::common::internal_err;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::execution::TaskContext;use datafusion::physical_plan::{DisplayAs, ExecutionPlan};use datafusion::prelude::SessionContext;@@ -128,6 +129,15 @@ impl ExecutionPlan for ParentExec {) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {unreachable!()}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// A PhysicalExtensionCodec that can serialize and deserialize ParentExec@@ -204,6 +214,15 @@ impl ExecutionPlan for ChildExec {) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {unreachable!()}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// A PhysicalExtensionCodec that can serialize and deserialize ChildExecdiff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rsindex 895f2fdd4f..aa24f6cfbf 100644--- a/datafusion-examples/examples/relation_planner/table_sample.rs+++ b/datafusion-examples/examples/relation_planner/table_sample.rs@@ -117,7 +117,7 @@ use datafusion::{};use datafusion_common::{DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,- plan_datafusion_err, plan_err,+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,};use datafusion_expr::{UserDefinedLogicalNode, UserDefinedLogicalNodeCore,@@ -743,6 +743,22 @@ impl ExecutionPlan for SampleExec {Ok(stats)}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}/// Bernoulli sampler: includes each row with probability `(upper - lower)`.diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rsindex 484b5f805e..9b91062657 100644--- a/datafusion/catalog/src/memory/table.rs+++ b/datafusion/catalog/src/memory/table.rs@@ -32,6 +32,7 @@ use arrow::compute::{and, filter_record_batch};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::error::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};use datafusion_common_runtime::JoinSet;use datafusion_datasource::memory::{MemSink, MemorySourceConfig};@@ -46,7 +47,7 @@ use datafusion_physical_plan::repartition::RepartitionExec;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;use datafusion_physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,- PlanProperties, common,+ PhysicalExpr, PlanProperties, common,};use datafusion_session::Session;@@ -400,10 +401,7 @@ impl TableProvider for MemTable {let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;// Create physical expressions for assignments upfront (outside batch loop)- let physical_assignments: HashMap<- String,- Arc<dyn datafusion_physical_plan::PhysicalExpr>,- > = assignments+ let physical_assignments: HashMap<String, Arc<dyn PhysicalExpr>> = assignments.iter().map(|(name, expr)| {let physical_expr =@@ -638,4 +636,11 @@ impl ExecutionPlan for DmlResultExec {stream,)))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rsindex 14f3e5cf03..8d52f555bb 100644--- a/datafusion/core/src/physical_planner.rs+++ b/datafusion/core/src/physical_planner.rs@@ -3891,6 +3891,20 @@ mod tests {) -> Result<SendableRecordBatchStream> {unimplemented!("NoOpExecutionPlan::execute");}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}// Produces an execution plan where the schema is mismatched from@@ -4033,6 +4047,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for OkExtensionNode {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {@@ -4082,6 +4102,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for InvariantFailsExtensionNode {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {@@ -4203,6 +4229,12 @@ digraph {) -> Result<SendableRecordBatchStream> {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}impl DisplayAs for ExecutableInvariantFails {fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rsindex f51d0a1e36..5f20fb1c60 100644--- a/datafusion/core/tests/custom_sources_cases/mod.rs+++ b/datafusion/core/tests/custom_sources_cases/mod.rs@@ -39,6 +39,7 @@ use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;use datafusion_common::project_schema;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::PlanProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};@@ -209,6 +210,22 @@ impl ExecutionPlan for CustomExecutionPlan {.collect(),})}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[async_trait]diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rsindex 96357d3103..6bb119fcb7 100644--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs@@ -35,6 +35,7 @@ use datafusion::prelude::*;use datafusion::scalar::ScalarValue;use datafusion_catalog::Session;use datafusion_common::cast::as_primitive_array;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, internal_err, not_impl_err};use datafusion_expr::expr::{BinaryExpr, Cast};use datafusion_functions_aggregate::expr_fn::count;@@ -152,6 +153,22 @@ impl ExecutionPlan for CustomPlan {})),)))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[derive(Clone, Debug)]diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rsindex 03513ec730..eb81c9f24d 100644--- a/datafusion/core/tests/custom_sources_cases/statistics.rs+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs@@ -33,6 +33,7 @@ use datafusion::{scalar::ScalarValue,};use datafusion_catalog::Session;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{project_schema, stats::Precision};use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};@@ -188,6 +189,22 @@ impl ExecutionPlan for StatisticsValidation {Ok(self.stats.clone())}}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}fn init_ctx(stats: Statistics, schema: Schema) -> Result<SessionContext> {diff --git a/datafusion/core/tests/fuzz_cases/once_exec.rs b/datafusion/core/tests/fuzz_cases/once_exec.rsindex 69edf9be1d..eed172f09f 100644--- a/datafusion/core/tests/fuzz_cases/once_exec.rs+++ b/datafusion/core/tests/fuzz_cases/once_exec.rs@@ -17,6 +17,7 @@use arrow_schema::SchemaRef;use datafusion_common::internal_datafusion_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};@@ -110,4 +111,20 @@ impl ExecutionPlan for OnceExec {stream.ok_or_else(|| internal_datafusion_err!("Stream already consumed"))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> datafusion_common::Result<TreeNodeRecursion>,+ ) -> datafusion_common::Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rsindex 5df634c70b..993798ff75 100644--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs@@ -40,7 +40,9 @@ use datafusion::prelude::{SessionConfig, SessionContext};use datafusion_common::ScalarValue;use datafusion_common::config::CsvOptions;use datafusion_common::error::Result;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_datasource::file_groups::FileGroup;use datafusion_datasource::file_scan_config::FileScanConfigBuilder;use datafusion_expr::{JoinType, Operator};@@ -202,6 +204,20 @@ impl ExecutionPlan for SortRequiredExec {)))}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex ac64727481..64d1c7bf10 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -4087,6 +4087,107 @@ async fn test_filter_with_projection_pushdown() {assert_batches_eq!(expected, &result);}+/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree+#[tokio::test]+async fn test_discover_dynamic_filters_via_expressions_api() {+ use datafusion_common::JoinType;+ use datafusion_common::tree_node::TreeNodeRecursion;+ use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;+ use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};++ fn count_dynamic_filters(plan: &Arc<dyn ExecutionPlan>) -> usize {+ let mut count = 0;++ // Check expressions from this node using apply_expressions+ let _ = plan.apply_expressions(&mut |expr| {+ if let Some(_df) = expr.as_any().downcast_ref::<DynamicFilterPhysicalExpr>() {+ count += 1;+ }+ Ok(TreeNodeRecursion::Continue)+ });++ // Recursively visit children+ for child in plan.children() {+ count += count_dynamic_filters(child);+ }++ count+ }++ // Create build side (left)+ let build_batches =+ vec![record_batch!(("a", Utf8, ["foo", "bar"]), ("b", Int32, [1, 2])).unwrap()];+ let build_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("b", DataType::Int32, false),+ ]));+ let build_scan = TestScanBuilder::new(build_schema.clone())+ .with_support(true)+ .with_batches(build_batches)+ .build();++ // Create probe side (right)+ let probe_batches = vec![+ record_batch!(+ ("a", Utf8, ["foo", "bar", "baz", "qux"]),+ ("c", Float64, [1.0, 2.0, 3.0, 4.0])+ )+ .unwrap(),+ ];+ let probe_schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Utf8, false),+ Field::new("c", DataType::Float64, false),+ ]));+ let probe_scan = TestScanBuilder::new(probe_schema.clone())+ .with_support(true)+ .with_batches(probe_batches)+ .build();++ // Create HashJoinExec+ let plan = Arc::new(+ HashJoinExec::try_new(+ build_scan,+ probe_scan,+ vec![(+ col("a", &build_schema).unwrap(),+ col("a", &probe_schema).unwrap(),+ )],+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ datafusion_common::NullEquality::NullEqualsNothing,+ false,+ )+ .unwrap(),+ ) as Arc<dyn ExecutionPlan>;++ // Before optimization: no dynamic filters+ let count_before = count_dynamic_filters(&plan);+ assert_eq!(+ count_before, 0,+ "Before optimization, should have no dynamic filters"+ );++ // Apply filter pushdown optimization (this creates dynamic filters)+ let mut config = ConfigOptions::default();+ config.optimizer.enable_dynamic_filter_pushdown = true;+ config.execution.parquet.pushdown_filters = true;+ let optimized_plan = FilterPushdown::new_post_optimization()+ .optimize(plan, &config)+ .unwrap();++ // After optimization: should discover dynamic filters+ // We expect 2 dynamic filters:+ // 1. In the HashJoinExec (producer)+ // 2. In the DataSourceExec (consumer, pushed down to the probe side)+ let count_after = count_dynamic_filters(&optimized_plan);+ assert_eq!(+ count_after, 2,+ "After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}"+ );+}+#[tokio::test]async fn test_hashjoin_dynamic_filter_pushdown_left_join() {use datafusion_common::JoinType;diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rsindex ef0bbfc7f4..17156022da 100644--- a/datafusion/core/tests/physical_optimizer/join_selection.rs+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs@@ -26,6 +26,7 @@ use std::{use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{ColumnStatistics, JoinType, ScalarValue, stats::Precision};use datafusion_common::{JoinSide, NullEquality};use datafusion_common::{Result, Statistics};@@ -1078,6 +1079,20 @@ impl ExecutionPlan for UnboundedExec {batch: self.batch.clone(),}))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[derive(Eq, PartialEq, Debug)]@@ -1183,6 +1198,20 @@ impl ExecutionPlan for StatisticsExec {self.stats.clone()})}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[test]diff --git a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs b/datafusion/core/tests/physical_optimizer/pushdown_utils.rsindex 91ae6c414e..ce2cb04b64 100644--- a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs+++ b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs@@ -18,6 +18,7 @@use arrow::datatypes::SchemaRef;use arrow::{array::RecordBatch, compute::concat_batches};use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, config::ConfigOptions, internal_err};use datafusion_datasource::{PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,@@ -239,6 +240,25 @@ impl FileSource for TestSource {fn table_schema(&self) -> &datafusion_datasource::TableSchema {&self.table_schema}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit predicate (filter) expression if present+ if let Some(predicate) = &self.predicate {+ f(predicate.as_ref())?;+ }++ // Visit projection expressions if present+ if let Some(projection) = &self.projection {+ for proj_expr in projection {+ f(proj_expr.expr.as_ref())?;+ }+ }++ Ok(TreeNodeRecursion::Continue)+ }}#[derive(Debug, Clone)]@@ -558,4 +578,13 @@ impl ExecutionPlan for TestNode {Ok(res)}}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit the predicate expression+ f(self.predicate.as_ref())?;+ Ok(TreeNodeRecursion::Continue)+ }}diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rsindex f8c91ba272..7b000f69a6 100644--- a/datafusion/core/tests/physical_optimizer/test_utils.rs+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs@@ -31,7 +31,9 @@ use datafusion::datasource::physical_plan::ParquetSource;use datafusion::datasource::source::DataSourceExec;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;use datafusion_common::{ColumnStatistics, JoinType, NullEquality, Result, Statistics, internal_err,@@ -492,6 +494,20 @@ impl ExecutionPlan for RequirementsTestExec {) -> Result<SendableRecordBatchStream> {unimplemented!("Test exec does not support execution")}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in required_input_ordering if present+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = &self.required_input_ordering {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}/// A [`PlanContext`] object is susceptible to being left in an inconsistent state after@@ -963,6 +979,28 @@ impl ExecutionPlan for TestScan {inner: Arc::new(new_scan),})}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in output_ordering+ let mut tnr = TreeNodeRecursion::Continue;+ for ordering in &self.output_ordering {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }++ // Visit expressions in requested_ordering if present+ if let Some(ordering) = &self.requested_ordering {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }++ Ok(tnr)+ }}/// Helper function to create a TestScan with orderingdiff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rsindex 4d2a31ca1f..2a2aed82f0 100644--- a/datafusion/core/tests/user_defined/insert_operation.rs+++ b/datafusion/core/tests/user_defined/insert_operation.rs@@ -25,6 +25,7 @@ use datafusion::{};use datafusion_catalog::{Session, TableProvider};use datafusion_common::config::Dialect;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_expr::{Expr, TableType, dml::InsertOp};use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use datafusion_physical_plan::execution_plan::SchedulingType;@@ -184,6 +185,22 @@ impl ExecutionPlan for TestInsertExec {) -> Result<datafusion_execution::SendableRecordBatchStream> {unimplemented!("TestInsertExec is a stub for testing.")}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.plan_properties.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}fn make_count_schema() -> SchemaRef {diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rsindex f97923ffc5..6e4ed69e50 100644--- a/datafusion/core/tests/user_defined/user_defined_plan.rs+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs@@ -90,7 +90,9 @@ use datafusion::{prelude::{SessionConfig, SessionContext},};use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::{ScalarValue, assert_eq_or_internal_err, assert_or_internal_err};use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};use datafusion_optimizer::AnalyzerRule;@@ -746,6 +748,22 @@ impl ExecutionPlan for TopKExec {state: BTreeMap::new(),}))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion::physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}// A very specialized TopK implementationdiff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rsindex 99446cb876..0226dadd3d 100644--- a/datafusion/datasource-arrow/src/source.rs+++ b/datafusion/datasource-arrow/src/source.rs@@ -40,6 +40,7 @@ use arrow::buffer::Buffer;use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};use datafusion_common::error::Result;use datafusion_common::exec_datafusion_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::PartitionedFile;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -396,6 +397,20 @@ impl FileSource for ArrowSource {fn projection(&self) -> Option<&ProjectionExprs> {Some(&self.projection.source)}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit projection expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for proj_expr in &self.projection.source {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }}/// `FileOpener` wrapper for both Arrow IPC file and stream formatsdiff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rsindex bd9ff2a7a8..525d72e620 100644--- a/datafusion/datasource-avro/src/source.rs+++ b/datafusion/datasource-avro/src/source.rs@@ -23,6 +23,7 @@ use std::sync::Arc;use crate::avro_to_arrow::Reader as AvroReader;use datafusion_common::error::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -139,6 +140,20 @@ impl FileSource for AvroSource {) -> Result<Option<FileScanConfig>> {Ok(None)}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit projection expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for proj_expr in &self.projection.source {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }}mod private {diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rsindex 37e3ef23b6..77a0dc9cf7 100644--- a/datafusion/datasource-csv/src/source.rs+++ b/datafusion/datasource-csv/src/source.rs@@ -35,6 +35,7 @@ use datafusion_datasource::{use arrow::csv;use datafusion_common::config::CsvOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result};use datafusion_common_runtime::JoinSet;use datafusion_datasource::file::FileSource;@@ -314,6 +315,20 @@ impl FileSource for CsvSource {DisplayFormatType::TreeRender => Ok(()),}}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit projection expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for proj_expr in &self.projection.source {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }}impl FileOpener for CsvOpener {diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rsindex 52a38f4994..2f1d5abbee 100644--- a/datafusion/datasource-json/src/source.rs+++ b/datafusion/datasource-json/src/source.rs@@ -27,6 +27,7 @@ use crate::file_format::JsonDecoder;use crate::utils::{ChannelReader, JsonArrayToNdjsonReader};use datafusion_common::error::{DataFusionError, Result};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common_runtime::{JoinSet, SpawnedTask};use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};use datafusion_datasource::file_compression_type::FileCompressionType;@@ -236,6 +237,20 @@ impl FileSource for JsonSource {fn file_type(&self) -> &str {"json"}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit projection expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for proj_expr in &self.projection.source {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }}impl FileOpener for JsonOpener {diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rsindex 75d87a4cd1..1355eff5b9 100644--- a/datafusion/datasource-parquet/src/source.rs+++ b/datafusion/datasource-parquet/src/source.rs@@ -35,6 +35,7 @@ use datafusion_datasource::file_stream::FileOpener;use arrow::datatypes::TimeUnit;use datafusion_common::DataFusionError;use datafusion_common::config::TableParquetOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_datasource::TableSchema;use datafusion_datasource::file::FileSource;use datafusion_datasource::file_scan_config::FileScanConfig;@@ -816,6 +817,26 @@ impl FileSource for ParquetSource {// - Detection of exact ordering (return Exact to remove Sort operator)// - Partial sort pushdown for prefix matches}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn PhysicalExpr,+ ) -> datafusion_common::Result<TreeNodeRecursion>,+ ) -> datafusion_common::Result<TreeNodeRecursion> {+ // Visit predicate (filter) expression if present+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(predicate) = &self.predicate {+ tnr = tnr.visit_sibling(|| f(predicate.as_ref()))?;+ }++ // Visit projection expressions+ for proj_expr in &self.projection {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }++ Ok(tnr)+ }}#[cfg(test)]diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rsindex a0f82ff7a9..b5a6760cae 100644--- a/datafusion/datasource/src/file.rs+++ b/datafusion/datasource/src/file.rs@@ -28,6 +28,7 @@ use crate::file_stream::FileOpener;#[expect(deprecated)]use crate::schema_adapter::SchemaAdapterFactory;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, not_impl_err};use datafusion_physical_expr::projection::ProjectionExprs;use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};@@ -319,4 +320,25 @@ pub trait FileSource: Send + Sync {fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {None}++ /// Apply a function to all physical expressions used by this file source.+ ///+ /// This includes:+ /// - Filter predicates (which may contain dynamic filters)+ /// - Projection expressions+ ///+ /// The function `f` is called once for each expression. The function should+ /// return `TreeNodeRecursion::Continue` to continue visiting other expressions,+ /// or `TreeNodeRecursion::Stop` to stop visiting expressions early.+ ///+ /// Implementations must explicitly visit all expressions. There is no default+ /// implementation to ensure that all FileSource implementations handle this correctly.+ ///+ /// See [`ExecutionPlan::apply_expressions`] for more details and examples.+ ///+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;}diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rsindex c3e5cabce7..524e091381 100644--- a/datafusion/datasource/src/file_scan_config.rs+++ b/datafusion/datasource/src/file_scan_config.rs@@ -27,6 +27,7 @@ use crate::{use arrow::datatypes::FieldRef;use arrow::datatypes::{DataType, Schema, SchemaRef};use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,};@@ -78,7 +79,9 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::/// # use arrow::datatypes::{Field, Fields, DataType, Schema, SchemaRef};/// # use object_store::ObjectStore;/// # use datafusion_common::Result;+/// # use datafusion_common::tree_node::TreeNodeRecursion;/// # use datafusion_datasource::file::FileSource;+/// # use datafusion_physical_plan::PhysicalExpr;/// # use datafusion_datasource::file_groups::FileGroup;/// # use datafusion_datasource::PartitionedFile;/// # use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};@@ -109,6 +112,7 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::/// # fn file_type(&self) -> &str { "parquet" }/// # // Note that this implementation drops the projection on the floor, it is not complete!/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }+/// # fn apply_expressions(&self, _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }/// # }/// # impl ParquetSource {/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }@@ -923,6 +927,14 @@ impl DataSource for FileScanConfig {};Some(Arc::new(new_config))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Delegate to the file source+ self.file_source.apply_expressions(f)+ }}impl FileScanConfig {@@ -1584,6 +1596,13 @@ mod tests {inner: Arc::new(self.clone()) as Arc<dyn FileSource>,})}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}#[test]diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rsindex 1d12bb3200..e6add2a42c 100644--- a/datafusion/datasource/src/memory.rs+++ b/datafusion/datasource/src/memory.rs@@ -29,6 +29,7 @@ use crate::source::{DataSource, DataSourceExec};use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::datatypes::{Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, ScalarValue, assert_or_internal_err, plan_err, project_schema,};@@ -252,6 +253,20 @@ impl DataSource for MemorySourceConfig {}).transpose()}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in sort_information+ let mut tnr = TreeNodeRecursion::Continue;+ for ordering in &self.sort_information {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}impl MemorySourceConfig {diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rsindex f149109dff..155c951fe5 100644--- a/datafusion/datasource/src/sink.rs+++ b/datafusion/datasource/src/sink.rs@@ -24,9 +24,10 @@ use std::sync::Arc;use arrow::array::{ArrayRef, RecordBatch, UInt64Array};use arrow::datatypes::{DataType, Field, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::{Distribution, EquivalenceProperties};+use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr};use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequirements};use datafusion_physical_plan::metrics::MetricsSet;use datafusion_physical_plan::stream::RecordBatchStreamAdapter;@@ -219,6 +220,19 @@ impl ExecutionPlan for DataSinkExec {)))}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to sort order requirements if present+ if let Some(sort_order) = &self.sort_order {+ for req in sort_order.iter() {+ f(req.expr.as_ref())?;+ }+ }+ Ok(TreeNodeRecursion::Continue)+ }+/// Execute the plan and return a stream of `RecordBatch`es for/// the specified partition.fn execute(diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rsindex 05028ed0f4..bdb45e8e3b 100644--- a/datafusion/datasource/src/source.rs+++ b/datafusion/datasource/src/source.rs@@ -37,6 +37,7 @@ use itertools::Itertools;use crate::file_scan_config::FileScanConfig;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, Result, Statistics};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};@@ -211,6 +212,22 @@ pub trait DataSource: Send + Sync + Debug {fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {None}++ /// Apply a closure to each expression used by this data source.+ ///+ /// This includes filter predicates (which may contain dynamic filters) and any+ /// other expressions used during data scanning.+ ///+ /// Implementations must override this method. If the data source has no expressions,+ /// return `Ok(TreeNodeRecursion::Continue)` immediately.+ ///+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.+ ///+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;}/// [`ExecutionPlan`] that reads one or more files@@ -262,6 +279,14 @@ impl ExecutionPlan for DataSourceExec {Vec::new()}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Delegate to the underlying data source+ self.data_source.apply_expressions(f)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rsindex c8d5dd54cb..3a9e78943b 100644--- a/datafusion/datasource/src/test_util.rs+++ b/datafusion/datasource/src/test_util.rs@@ -22,7 +22,7 @@ use crate::{use std::sync::Arc;use arrow::datatypes::Schema;-use datafusion_common::Result;+use datafusion_common::{Result, tree_node::TreeNodeRecursion};use datafusion_physical_expr::{PhysicalExpr, expressions::Column};use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;use object_store::ObjectStore;@@ -119,6 +119,13 @@ impl FileSource for MockSource {) -> Option<&datafusion_physical_plan::projection::ProjectionExprs> {Some(&self.projection.source)}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}/// Create a column expressiondiff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rsindex 524d8b4b6b..064e4a8953 100644--- a/datafusion/ffi/src/execution_plan.rs+++ b/datafusion/ffi/src/execution_plan.rs@@ -21,6 +21,7 @@ use std::sync::Arc;use abi_stable::StableAbi;use abi_stable::std_types::{RString, RVec};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result};use datafusion_execution::{SendableRecordBatchStream, TaskContext};use datafusion_physical_plan::{@@ -293,6 +294,22 @@ impl ExecutionPlan for ForeignExecutionPlan {.map(|stream| Pin::new(Box::new(stream)) as SendableRecordBatchStream)}}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.properties.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[cfg(test)]@@ -367,6 +384,22 @@ pub(crate) mod tests {) -> Result<SendableRecordBatchStream> {unimplemented!()}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.props.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}#[test]diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rsindex 8370cf19e6..e9fa31a7fc 100644--- a/datafusion/ffi/src/tests/async_provider.rs+++ b/datafusion/ffi/src/tests/async_provider.rs@@ -33,6 +33,7 @@ use arrow::array::RecordBatch;use arrow::datatypes::Schema;use async_trait::async_trait;use datafusion_catalog::TableProvider;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, exec_err};use datafusion_execution::RecordBatchStream;use datafusion_expr::Expr;@@ -219,6 +220,22 @@ impl ExecutionPlan for AsyncTestExecutionPlan {batch_receiver: self.batch_receiver.resubscribe(),}))}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_plan::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in the output ordering from equivalence properties+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.properties.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }}impl datafusion_physical_plan::DisplayAs for AsyncTestExecutionPlan {diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rsindex ef8946f9a4..6925c036e4 100644--- a/datafusion/physical-optimizer/src/ensure_coop.rs+++ b/datafusion/physical-optimizer/src/ensure_coop.rs@@ -264,11 +264,12 @@ mod tests {async fn test_eager_evaluation_resets_cooperative_context() {// Test that cooperative context is reset when encountering an eager evaluation boundary.use arrow::datatypes::Schema;+ use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;use datafusion_physical_plan::{- DisplayAs, DisplayFormatType, Partitioning, PlanProperties,+ DisplayAs, DisplayFormatType, Partitioning, PhysicalExpr, PlanProperties,SendableRecordBatchStream,execution_plan::{Boundedness, EmissionType},};@@ -351,6 +352,13 @@ mod tests {) -> Result<SendableRecordBatchStream> {internal_err!("DummyExec does not support execution")}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}// Build a plan similar to the original test:diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rsindex 75721951f8..b2212c94bc 100644--- a/datafusion/physical-optimizer/src/output_requirements.rs+++ b/datafusion/physical-optimizer/src/output_requirements.rs@@ -27,7 +27,9 @@ use std::sync::Arc;use crate::PhysicalOptimizerRule;use datafusion_common::config::ConfigOptions;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};use datafusion_common::{Result, Statistics};use datafusion_execution::TaskContext;use datafusion_physical_expr::Distribution;@@ -296,6 +298,36 @@ impl ExecutionPlan for OutputRequirementExec {fn fetch(&self) -> Option<usize> {self.fetch}++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(+ &dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr,+ ) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Visit expressions in order_requirement+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(order_reqs) = &self.order_requirement {+ let lexes = match order_reqs {+ OrderingRequirements::Hard(alternatives) => alternatives,+ OrderingRequirements::Soft(alternatives) => alternatives,+ };+ for lex in lexes {+ for sort_expr in lex {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ }++ // Visit expressions in dist_requirement if it's HashPartitioned+ if let Distribution::HashPartitioned(exprs) = &self.dist_requirement {+ for expr in exprs {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ }++ Ok(tnr)+ }}impl PhysicalOptimizerRule for OutputRequirements {diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 8599993851..561e9d1d05 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -45,6 +45,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_schema::FieldRef;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraint, Constraints, Result, ScalarValue, assert_eq_or_internal_err, not_impl_err,};@@ -1382,6 +1383,36 @@ impl ExecutionPlan for AggregateExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to group by expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for expr in self.group_by.input_exprs() {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }++ // Apply to aggregate expressions+ for aggr in self.aggr_expr.iter() {+ for expr in aggr.expressions() {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ }++ // Apply to filter expressions (FILTER WHERE clauses)+ for filter in self.filter_expr.iter().flatten() {+ tnr = tnr.visit_sibling(|| f(filter.as_ref()))?;+ }++ // Apply to dynamic filter expression if present+ if let Some(dyn_filter) = &self.dynamic_filter {+ tnr = tnr.visit_sibling(|| f(dyn_filter.filter.as_ref()))?;+ }++ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -2479,6 +2510,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rsindex eca31ea0e1..4aa78055da 100644--- a/datafusion/physical-plan/src/analyze.rs+++ b/datafusion/physical-plan/src/analyze.rs@@ -31,9 +31,11 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::instant::Instant;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::PhysicalExpr;use futures::StreamExt;@@ -143,6 +145,13 @@ impl ExecutionPlan for AnalyzeExec {vec![Distribution::UnspecifiedDistribution]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rsindex 72741f4314..abfe870f52 100644--- a/datafusion/physical-plan/src/async_func.rs+++ b/datafusion/physical-plan/src/async_func.rs@@ -24,7 +24,8 @@ use crate::{};use arrow::array::RecordBatch;use arrow_schema::{Fields, Schema, SchemaRef};-use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};+use datafusion_common::tree_node::TreeNodeRecursion;+use datafusion_common::tree_node::{Transformed, TreeNode};use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};use datafusion_physical_expr::ScalarFunctionExpr;@@ -169,6 +170,13 @@ impl ExecutionPlan for AsyncFuncExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rsindex a59d062929..8477f39a9b 100644--- a/datafusion/physical-plan/src/buffer.rs+++ b/datafusion/physical-plan/src/buffer.rs@@ -31,6 +31,7 @@ use crate::{};use arrow::array::RecordBatch;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, Statistics, internal_err, plan_err};use datafusion_common_runtime::SpawnedTask;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};@@ -172,6 +173,13 @@ impl ExecutionPlan for BufferExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rsindex 663b0b51ea..d26d1f9bea 100644--- a/datafusion/physical-plan/src/coalesce_batches.rs+++ b/datafusion/physical-plan/src/coalesce_batches.rs@@ -33,6 +33,7 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_execution::TaskContext;use datafusion_physical_expr::PhysicalExpr;@@ -187,6 +188,13 @@ impl ExecutionPlan for CoalesceBatchesExec {vec![false]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rsindex 39906d3680..3dffdd6a99 100644--- a/datafusion/physical-plan/src/coalesce_partitions.rs+++ b/datafusion/physical-plan/src/coalesce_partitions.rs@@ -35,6 +35,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_properuse datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::PhysicalExpr;@@ -158,6 +159,13 @@ impl ExecutionPlan for CoalescePartitionsExec {vec![false]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rsindex 5f0040b3dd..c7a5432994 100644--- a/datafusion/physical-plan/src/coop.rs+++ b/datafusion/physical-plan/src/coop.rs@@ -71,6 +71,7 @@//! that report [`SchedulingType::NonCooperative`] in their [plan properties](ExecutionPlan::properties).use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_physical_expr::PhysicalExpr;#[cfg(datafusion_coop = "tokio_fallback")]use futures::Future;@@ -281,6 +282,13 @@ impl ExecutionPlan for CooperativeExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rsindex 44148f2d0e..c69ac03178 100644--- a/datafusion/physical-plan/src/display.rs+++ b/datafusion/physical-plan/src/display.rs@@ -1120,8 +1120,11 @@ mod tests {use std::fmt::Write;use std::sync::Arc;- use datafusion_common::{Result, Statistics, internal_datafusion_err};+ use datafusion_common::{+ Result, Statistics, internal_datafusion_err, tree_node::TreeNodeRecursion,+ };use datafusion_execution::{SendableRecordBatchStream, TaskContext};+ use datafusion_physical_expr::PhysicalExpr;use crate::{DisplayAs, ExecutionPlan, PlanProperties};@@ -1168,6 +1171,13 @@ mod tests {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_: usize,diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rsindex e4d4da4e88..db3f5161a9 100644--- a/datafusion/physical-plan/src/empty.rs+++ b/datafusion/physical-plan/src/empty.rs@@ -30,9 +30,10 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use crate::execution_plan::SchedulingType;use log::trace;@@ -123,6 +124,13 @@ impl ExecutionPlan for EmptyExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rsindex 681a1345d8..0a9348b68a 100644--- a/datafusion/physical-plan/src/execution_plan.rs+++ b/datafusion/physical-plan/src/execution_plan.rs@@ -27,7 +27,9 @@ pub use crate::stream::EmptyRecordBatchStream;use arrow_schema::Schema;pub use datafusion_common::hash_utils;-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};+use datafusion_common::tree_node::{+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,+};pub use datafusion_common::utils::project_schema;pub use datafusion_common::{ColumnStatistics, Statistics, internal_err};pub use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};@@ -205,6 +207,80 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {/// joins).fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>>;+ /// Apply a closure `f` to each expression (non-recursively) in the current+ /// physical plan node. This does not include expressions in any children.+ ///+ /// The closure `f` is applied to expressions in the order they appear in the plan.+ /// The closure can return `TreeNodeRecursion::Continue` to continue visiting,+ /// `TreeNodeRecursion::Stop` to stop visiting immediately, or `TreeNodeRecursion::Jump`+ /// to skip any remaining expressions (though typically all expressions are visited).+ ///+ /// The expressions visited do not necessarily represent or even contribute+ /// to the output schema of this node. For example, `FilterExec` visits the+ /// filter predicate even though the output of a Filter has the same columns+ /// as the input.+ ///+ /// # Example Usage+ /// ```+ /// # use std::sync::Arc;+ /// # use datafusion_physical_plan::ExecutionPlan;+ /// # use datafusion_common::tree_node::TreeNodeRecursion;+ /// # fn example(plan: Arc<dyn ExecutionPlan>) -> datafusion_common::Result<()> {+ /// // Count the number of expressions+ /// let mut count = 0;+ /// plan.apply_expressions(&mut |_expr| {+ /// count += 1;+ /// Ok(TreeNodeRecursion::Continue)+ /// })?;+ /// # Ok(())+ /// # }+ /// ```+ ///+ /// # Implementation Examples+ ///+ /// ## Node with no expressions (e.g., EmptyExec, MemoryExec)+ /// ```ignore+ /// fn apply_expressions(+ /// &self,+ /// _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ /// ) -> Result<TreeNodeRecursion> {+ /// Ok(TreeNodeRecursion::Continue)+ /// }+ /// ```+ ///+ /// ## Node with a single expression (e.g., FilterExec)+ /// ```ignore+ /// fn apply_expressions(+ /// &self,+ /// f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ /// ) -> Result<TreeNodeRecursion> {+ /// f(self.predicate.as_ref())+ /// }+ /// ```+ ///+ /// ## Node with multiple expressions (e.g., ProjectionExec, JoinExec)+ ///+ /// Use [`TreeNodeRecursion::visit_sibling`] when iterating over multiple+ /// expressions. This correctly propagates [`TreeNodeRecursion::Stop`]: if+ /// `f` returns `Stop` for an earlier expression, `visit_sibling` short-circuits+ /// and skips the remaining ones.+ /// ```ignore+ /// fn apply_expressions(+ /// &self,+ /// f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ /// ) -> Result<TreeNodeRecursion> {+ /// let mut tnr = TreeNodeRecursion::Continue;+ /// for expr in &self.expressions {+ /// tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ /// }+ /// Ok(tnr)+ /// }+ /// ```+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion>;+/// Returns a new `ExecutionPlan` where all existing children were replaced/// by the `children`, in orderfn with_new_children(@@ -1549,6 +1625,13 @@ mod tests {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,@@ -1605,6 +1688,13 @@ mod tests {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -1625,6 +1715,110 @@ mod tests {}}+ /// A test node that holds a fixed list of expressions, used to test+ /// `apply_expressions` behavior.+ #[derive(Debug)]+ struct MultiExprExec {+ exprs: Vec<Arc<dyn PhysicalExpr>>,+ }++ impl DisplayAs for MultiExprExec {+ fn fmt_as(+ &self,+ _t: DisplayFormatType,+ _f: &mut std::fmt::Formatter,+ ) -> std::fmt::Result {+ unimplemented!()+ }+ }++ impl ExecutionPlan for MultiExprExec {+ fn name(&self) -> &'static str {+ "MultiExprExec"+ }++ fn as_any(&self) -> &dyn Any {+ self+ }++ fn properties(&self) -> &std::sync::Arc<PlanProperties> {+ unimplemented!()+ }++ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {+ vec![]+ }++ fn with_new_children(+ self: Arc<Self>,+ _: Vec<Arc<dyn ExecutionPlan>>,+ ) -> Result<Arc<dyn ExecutionPlan>> {+ unimplemented!()+ }++ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for expr in &self.exprs {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ Ok(tnr)+ }++ fn execute(+ &self,+ _partition: usize,+ _context: Arc<TaskContext>,+ ) -> Result<SendableRecordBatchStream> {+ unimplemented!()+ }++ fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {+ unimplemented!()+ }+ }++ /// Returns a simple literal `Arc<dyn PhysicalExpr>` for use in tests.+ fn lit_expr(val: i64) -> Arc<dyn PhysicalExpr> {+ use datafusion_physical_expr::expressions::Literal;+ Arc::new(Literal::new(datafusion_common::ScalarValue::Int64(Some(+ val,+ ))))+ }++ /// `apply_expressions` visits all expressions when `f` always returns `Continue`.+ #[test]+ fn test_apply_expressions_continue_visits_all() -> Result<()> {+ let plan = MultiExprExec {+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],+ };+ let mut visited = 0usize;+ plan.apply_expressions(&mut |_expr| {+ visited += 1;+ Ok(TreeNodeRecursion::Continue)+ })?;+ assert_eq!(visited, 3);+ Ok(())+ }++ #[test]+ fn test_apply_expressions_stop_halts_early() -> Result<()> {+ let plan = MultiExprExec {+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],+ };+ let mut visited = 0usize;+ let tnr = plan.apply_expressions(&mut |_expr| {+ visited += 1;+ Ok(TreeNodeRecursion::Stop)+ })?;+ // Only the first expression is visited; the rest are skipped.+ assert_eq!(visited, 1);+ assert_eq!(tnr, TreeNodeRecursion::Stop);+ Ok(())+ }+#[test]fn test_execution_plan_name() {let schema1 = Arc::new(Schema::empty());diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rsindex bf21b04846..fa684f3483 100644--- a/datafusion/physical-plan/src/explain.rs+++ b/datafusion/physical-plan/src/explain.rs@@ -27,9 +27,10 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};use datafusion_common::display::StringifiedPlan;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use log::trace;@@ -121,6 +122,13 @@ impl ExecutionPlan for ExplainExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rsindex ecea4e6ebe..4622439b98 100644--- a/datafusion/physical-plan/src/filter.rs+++ b/datafusion/physical-plan/src/filter.rs@@ -51,6 +51,7 @@ use arrow::record_batch::RecordBatch;use datafusion_common::cast::as_boolean_array;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, ScalarValue, internal_err, plan_err, project_schema,};@@ -506,6 +507,13 @@ impl ExecutionPlan for FilterExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ f(self.predicate.as_ref())+ }+fn maintains_input_order(&self) -> Vec<bool> {// Tell optimizer this operator doesn't reorder its inputvec![true]diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rsindex 342cb7e70a..bdad207503 100644--- a/datafusion/physical-plan/src/joins/cross_join.rs+++ b/datafusion/physical-plan/src/joins/cross_join.rs@@ -41,11 +41,13 @@ use arrow::array::{RecordBatch, RecordBatchOptions};use arrow::compute::concat_batches;use arrow::datatypes::{Fields, Schema, SchemaRef};use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinType, Result, ScalarValue, assert_eq_or_internal_err, internal_err,};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::equivalence::join_equivalence_properties;use async_trait::async_trait;@@ -285,6 +287,14 @@ impl ExecutionPlan for CrossJoinExec {Some(self.metrics.clone_inner())}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // CrossJoin has no join conditions or expressions+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex b6ce472bbe..b5e8d3c1e3 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -69,6 +69,7 @@ use arrow::record_batch::RecordBatch;use arrow::util::bit_util;use arrow_schema::{DataType, Schema};use datafusion_common::config::ConfigOptions;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::memory::estimate_memory_size;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,@@ -1231,6 +1232,30 @@ impl ExecutionPlan for HashJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to join key expressions from both sides+ let mut tnr = TreeNodeRecursion::Continue;+ for (left, right) in &self.on {+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;+ }++ // Apply to join filter expression if present+ if let Some(filter) = &self.filter {+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;+ }++ // Apply to dynamic filter expression if present+ if let Some(df) = &self.dynamic_filter {+ tnr = tnr.visit_sibling(|| f(df.filter.as_ref()))?;+ }++ Ok(tnr)+ }+/// Creates a new HashJoinExec with different children while preserving configuration.////// This method is called during query optimization when the optimizer creates newdiff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex 4fb7dabf67..1ba6ee23ca 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -61,6 +61,7 @@ use arrow::datatypes::{Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_schema::DataType;use datafusion_common::cast::as_boolean_array;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, Result, ScalarValue, Statistics, arrow_err, assert_eq_or_internal_err,internal_datafusion_err, internal_err, project_schema, unwrap_or_internal_err,@@ -557,6 +558,17 @@ impl ExecutionPlan for NestedLoopJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to join filter expressions if present+ if let Some(filter) = &self.filter {+ f(filter.expression().as_ref())?;+ }+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rsindex abb6e34aa2..fb1c4b1605 100644--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs@@ -23,6 +23,7 @@ use arrow::{};use arrow_schema::{SchemaRef, SortOptions};use datafusion_common::not_impl_err;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, Result, internal_err};use datafusion_execution::{SendableRecordBatchStream,@@ -511,6 +512,14 @@ impl ExecutionPlan for PiecewiseMergeJoinExec {vec![&self.buffered, &self.streamed]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to the two expressions being compared in the range predicate+ f(self.on.0.as_ref())?.visit_sibling(|| f(self.on.1.as_ref()))+ }+fn required_input_distribution(&self) -> Vec<Distribution> {vec![Distribution::SinglePartition,diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rsindex b34e811f91..7a01888c28 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs@@ -44,6 +44,7 @@ use crate::{use arrow::compute::SortOptions;use arrow::datatypes::SchemaRef;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err, internal_err,plan_err,@@ -450,6 +451,23 @@ impl ExecutionPlan for SortMergeJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to join keys from both sides+ let mut tnr = TreeNodeRecursion::Continue;+ for (left, right) in &self.on {+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;+ }+ // Apply to join filter expressions if present+ if let Some(filter) = &self.filter {+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rsindex 29917dd2cc..eeb160ae51 100644--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs@@ -66,6 +66,7 @@ use arrow::compute::concat_batches;use arrow::datatypes::{ArrowNativeType, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::hash_utils::create_hashes;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::bisect;use datafusion_common::{HashSet, JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err,@@ -464,6 +465,23 @@ impl ExecutionPlan for SymmetricHashJoinExec {vec![&self.left, &self.right]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to join keys from both sides+ let mut tnr = TreeNodeRecursion::Continue;+ for (left, right) in &self.on {+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;+ }+ // Apply to join filter expressions if present+ if let Some(filter) = &self.filter {+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rsindex a78e5c067f..63a722ef26 100644--- a/datafusion/physical-plan/src/limit.rs+++ b/datafusion/physical-plan/src/limit.rs@@ -35,10 +35,11 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::LexOrdering;+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use futures::stream::{Stream, StreamExt};use log::trace;@@ -178,6 +179,20 @@ impl ExecutionPlan for GlobalLimitExec {vec![false]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to required ordering expressions if present+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = &self.required_ordering {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -353,6 +368,20 @@ impl ExecutionPlan for LocalLimitExec {vec![true]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to required ordering expressions if present+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = &self.required_ordering {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rsindex 90fd3f24cf..d607f2b440 100644--- a/datafusion/physical-plan/src/memory.rs+++ b/datafusion/physical-plan/src/memory.rs@@ -32,10 +32,11 @@ use crate::{use arrow::array::RecordBatch;use arrow::datatypes::SchemaRef;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, assert_or_internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryReservation;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use futures::Stream;@@ -315,6 +316,13 @@ impl ExecutionPlan for LazyMemoryExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rsindex 5dbd7b3032..68fe62f168 100644--- a/datafusion/physical-plan/src/placeholder_row.rs+++ b/datafusion/physical-plan/src/placeholder_row.rs@@ -30,9 +30,11 @@ use crate::{use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_or_internal_err};use datafusion_execution::TaskContext;use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::PhysicalExpr;use log::trace;@@ -140,6 +142,13 @@ impl ExecutionPlan for PlaceholderRowExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rsindex db3a71fc70..a3d940df13 100644--- a/datafusion/physical-plan/src/projection.rs+++ b/datafusion/physical-plan/src/projection.rs@@ -318,6 +318,17 @@ impl ExecutionPlan for ProjectionExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for proj_expr in self.projector.projection().as_ref().iter() {+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rsindex 995aa4822a..049aa9563d 100644--- a/datafusion/physical-plan/src/recursive_query.rs+++ b/datafusion/physical-plan/src/recursive_query.rs@@ -36,10 +36,12 @@ use arrow::array::{BooleanArray, BooleanBuilder};use arrow::compute::filter_record_batch;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};use datafusion_common::{Result, internal_datafusion_err, not_impl_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::{EquivalenceProperties, Partitioning};use futures::{Stream, StreamExt, ready};@@ -151,6 +153,13 @@ impl ExecutionPlan for RecursiveQueryExec {vec![&self.static_term, &self.recursive_term]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+// TODO: control these hints and see whether we can// infer some from the child plans (static/recursive terms).fn maintains_input_order(&self) -> Vec<bool> {diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex da4329e2cc..081f10d482 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -49,6 +49,7 @@ use arrow::compute::take_arrays;use arrow::datatypes::{SchemaRef, UInt32Type};use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::transpose;use datafusion_common::{ColumnStatistics, DataFusionError, HashMap, assert_or_internal_err,@@ -914,6 +915,21 @@ impl ExecutionPlan for RepartitionExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to hash partition expressions if this is a hash repartition+ if let Partitioning::Hash(exprs, _) = self.partitioning() {+ let mut tnr = TreeNodeRecursion::Continue;+ for expr in exprs {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ return Ok(tnr);+ }+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rsindex 0dbb75f2ef..6df3babb8e 100644--- a/datafusion/physical-plan/src/sorts/partial_sort.rs+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs@@ -69,9 +69,10 @@ use arrow::compute::concat_batches;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::evaluate_partition_ranges;use datafusion_execution::{RecordBatchStream, TaskContext};-use datafusion_physical_expr::LexOrdering;+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};use futures::{Stream, StreamExt, ready};use log::trace;@@ -287,6 +288,17 @@ impl ExecutionPlan for PartialSortExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for sort_expr in &self.expr {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex b3ea548d53..6447b3c310 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -58,6 +58,7 @@ use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray};use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};use arrow::datatypes::SchemaRef;use datafusion_common::config::SpillCompression;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,unwrap_or_internal_err,@@ -1194,6 +1195,25 @@ impl ExecutionPlan for SortExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to sort expressions+ let mut tnr = TreeNodeRecursion::Continue;+ for sort_expr in &self.expr {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }++ // Apply to dynamic filter expression if present (when fetch is Some, TopK mode)+ if let Some(filter) = &self.filter {+ let filter_guard = filter.read();+ tnr = tnr.visit_sibling(|| f(filter_guard.expr().as_ref()))?;+ }++ Ok(tnr)+ }+fn benefits_from_input_partitioning(&self) -> Vec<bool> {vec![false]}@@ -1490,6 +1510,13 @@ mod tests {Ok(self)}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rsindex 763b72a660..7cbc21f8cb 100644--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs@@ -31,9 +31,11 @@ use crate::{check_if_same_properties,};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryConsumer;+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};use crate::execution_plan::{EvaluationType, SchedulingType};@@ -290,6 +292,17 @@ impl ExecutionPlan for SortPreservingMergeExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for sort_expr in &self.expr {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,@@ -1413,6 +1426,12 @@ mod tests {fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rsindex 1535482374..5a1206629a 100644--- a/datafusion/physical-plan/src/streaming.rs+++ b/datafusion/physical-plan/src/streaming.rs@@ -34,8 +34,10 @@ use crate::stream::RecordBatchStreamAdapter;use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};use arrow::datatypes::{Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, internal_err, plan_err};use datafusion_execution::TaskContext;+use datafusion_physical_expr::PhysicalExpr;use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};use async_trait::async_trait;@@ -248,6 +250,13 @@ impl ExecutionPlan for StreamingTableExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rsindex 0e7b900eb6..499c10413a 100644--- a/datafusion/physical-plan/src/test.rs+++ b/datafusion/physical-plan/src/test.rs@@ -36,6 +36,7 @@ use crate::{DisplayAs, DisplayFormatType, PlanProperties};use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};use arrow_schema::{DataType, Field, Schema, SchemaRef};+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, Statistics, assert_or_internal_err, config::ConfigOptions, project_schema,};@@ -45,7 +46,9 @@ use datafusion_physical_expr::equivalence::{};use datafusion_physical_expr::expressions::Column;use datafusion_physical_expr::utils::collect_columns;-use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, Partitioning};+use datafusion_physical_expr::{+ EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr,+};use futures::{Future, FutureExt};@@ -142,6 +145,20 @@ impl ExecutionPlan for TestMemoryExec {Vec::new()}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ // Apply to all sort information orderings+ let mut tnr = TreeNodeRecursion::Continue;+ for ordering in &self.sort_information {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rsindex d628fb819f..7b31855ce8 100644--- a/datafusion/physical-plan/src/test/exec.rs+++ b/datafusion/physical-plan/src/test/exec.rs@@ -36,9 +36,10 @@ use std::{use arrow::datatypes::{DataType, Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{DataFusionError, Result, internal_err};use datafusion_execution::TaskContext;-use datafusion_physical_expr::EquivalenceProperties;+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};use futures::Stream;use tokio::sync::Barrier;@@ -200,6 +201,13 @@ impl ExecutionPlan for MockExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -437,6 +445,13 @@ impl ExecutionPlan for BarrierExec {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Returns a stream which yields datafn execute(&self,@@ -573,6 +588,13 @@ impl ExecutionPlan for ErrorExec {unimplemented!()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+/// Returns a stream which yields datafn execute(&self,@@ -656,6 +678,13 @@ impl ExecutionPlan for StatisticsExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,@@ -767,6 +796,13 @@ impl ExecutionPlan for BlockingExec {internal_err!("Children cannot be replaced in {self:?}")}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn execute(&self,_partition: usize,@@ -906,6 +942,13 @@ impl ExecutionPlan for PanicExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rsindex 9fc02e730d..5c4f821c98 100644--- a/datafusion/physical-plan/src/union.rs+++ b/datafusion/physical-plan/src/union.rs@@ -50,6 +50,7 @@ use arrow::datatypes::{Field, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::config::ConfigOptions;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_or_internal_err, exec_err, internal_datafusion_err,};@@ -273,6 +274,13 @@ impl ExecutionPlan for UnionExec {self.inputs.iter().collect()}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,@@ -589,6 +597,13 @@ impl ExecutionPlan for InterleaveExec {vec![false; self.inputs().len()]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rsindex 422a9dd0d3..72faeb83c5 100644--- a/datafusion/physical-plan/src/unnest.rs+++ b/datafusion/physical-plan/src/unnest.rs@@ -42,6 +42,7 @@ use arrow::datatypes::{DataType, Int64Type, Schema, SchemaRef};use arrow::record_batch::RecordBatch;use arrow_ord::cmp::lt;use async_trait::async_trait;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Constraints, HashMap, HashSet, Result, UnnestOptions, exec_datafusion_err, exec_err,internal_err,@@ -240,6 +241,13 @@ impl ExecutionPlan for UnnestExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,mut children: Vec<Arc<dyn ExecutionPlan>>,diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rsindex a31268b9c6..0a3d592776 100644--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs@@ -48,6 +48,7 @@ use arrow::{};use datafusion_common::hash_utils::create_hashes;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::{evaluate_partition_ranges, get_at_indices, get_row_at_idx,};@@ -323,6 +324,19 @@ impl ExecutionPlan for BoundedWindowAggExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for window_expr in &self.window_expr {+ for expr in window_expr.expressions() {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {let partition_bys = self.window_expr()[0].partition_by();let order_keys = self.window_expr()[0].order_by();diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rsindex 0a146d51d6..01710d5b3c 100644--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs@@ -41,6 +41,7 @@ use arrow::datatypes::SchemaRef;use arrow::error::ArrowError;use arrow::record_batch::RecordBatch;use datafusion_common::stats::Precision;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::utils::{evaluate_partition_ranges, transpose};use datafusion_common::{Result, assert_eq_or_internal_err};use datafusion_execution::TaskContext;@@ -225,6 +226,19 @@ impl ExecutionPlan for WindowAggExec {vec![&self.input]}+ fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for window_expr in &self.window_expr {+ for expr in window_expr.expressions() {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ }+ Ok(tnr)+ }+fn maintains_input_order(&self) -> Vec<bool> {vec![true]}diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rsindex 4c7f77e0ff..4e8cfeea39 100644--- a/datafusion/physical-plan/src/work_table.rs+++ b/datafusion/physical-plan/src/work_table.rs@@ -31,10 +31,11 @@ use crate::{use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;+use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};use datafusion_execution::TaskContext;use datafusion_execution::memory_pool::MemoryReservation;-use datafusion_physical_expr::{EquivalenceProperties, Partitioning};+use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};/// A vector of record batches with a memory reservation.#[derive(Debug)]@@ -189,6 +190,13 @@ impl ExecutionPlan for WorkTableExec {vec![]}+ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }+fn with_new_children(self: Arc<Self>,_: Vec<Arc<dyn ExecutionPlan>>,diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.mdindex 50005a7527..70b6be3ae2 100644--- a/docs/source/library-user-guide/custom-table-providers.md+++ b/docs/source/library-user-guide/custom-table-providers.md@@ -52,11 +52,12 @@ use std::any::Any;use std::sync::{Arc, Mutex};use std::collections::{BTreeMap, HashMap};use datafusion::common::Result;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};use datafusion::physical_plan::expressions::PhysicalSortExpr;use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream, DisplayAs, DisplayFormatType,- Statistics, PlanProperties+ Statistics, PlanProperties, PhysicalExpr};use datafusion::execution::context::TaskContext;use datafusion::arrow::array::{UInt64Builder, UInt8Builder};@@ -153,6 +154,13 @@ impl ExecutionPlan for CustomExec {None,)?))}++ fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+ ) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+ }}```@@ -277,12 +285,20 @@ The `scan` method of the `TableProvider` returns a `Result<Arc<dyn ExecutionPlan# None,# )?))# }+#+# fn apply_expressions(+# &self,+# _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+# ) -> Result<TreeNodeRecursion> {+# Ok(TreeNodeRecursion::Continue)+# }# }use async_trait::async_trait;+use datafusion::common::tree_node::TreeNodeRecursion;use datafusion::logical_expr::expr::Expr;use datafusion::datasource::{TableProvider, TableType};-use datafusion::physical_plan::project_schema;+use datafusion::physical_plan::{project_schema, PhysicalExpr};use datafusion::catalog::Session;impl CustomExec {@@ -469,12 +485,20 @@ This will allow you to use the custom table provider in DataFusion. For example,# None,# )?))# }+#+# fn apply_expressions(+# &self,+# _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+# ) -> Result<TreeNodeRecursion> {+# Ok(TreeNodeRecursion::Continue)+# }# }# use async_trait::async_trait;+# use datafusion::common::tree_node::TreeNodeRecursion;# use datafusion::logical_expr::expr::Expr;# use datafusion::datasource::{TableProvider, TableType};-# use datafusion::physical_plan::project_schema;+# use datafusion::physical_plan::{project_schema, PhysicalExpr};# use datafusion::catalog::Session;## impl CustomExec {diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.mdnew file mode 100644index 0000000000..85cb8e9dd0--- /dev/null+++ b/docs/source/library-user-guide/upgrading/54.0.0.md@@ -0,0 +1,93 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Upgrade Guides++## DataFusion 54.0.0++**Note:** DataFusion `54.0.0` has not been released yet. The information provided+in this section pertains to features and changes that have already been merged+to the main branch and are awaiting release in this version.++### `ExecutionPlan::apply_expressions` is now a required method++`apply_expressions` has been added as a **required** method on the `ExecutionPlan` trait (no default implementation). The same applies to the `FileSource` and `DataSource` traits. Any custom implementation of these traits must now implement `apply_expressions`.++**Who is affected:**++- Users who implement custom `ExecutionPlan` nodes+- Users who implement custom `FileSource` or `DataSource` sources++**Migration guide:**++Add `apply_expressions` to your implementation. Call `f` on each top-level `PhysicalExpr` your node owns, using `visit_sibling` to correctly propagate `TreeNodeRecursion`:++**Node with no expressions:**++```rust,ignore+fn apply_expressions(+ &self,+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+) -> Result<TreeNodeRecursion> {+ Ok(TreeNodeRecursion::Continue)+}+```++**Node with a single expression:**++```rust,ignore+fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+) -> Result<TreeNodeRecursion> {+ f(self.predicate.as_ref())+}+```++**Node with multiple expressions:**++```rust,ignore+fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ for expr in &self.expressions {+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;+ }+ Ok(tnr)+}+```++**Node whose only expressions are in `output_ordering()` (e.g. a synthetic test node with no owned expression fields):**++```rust,ignore+fn apply_expressions(+ &self,+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,+) -> Result<TreeNodeRecursion> {+ let mut tnr = TreeNodeRecursion::Continue;+ if let Some(ordering) = self.cache.output_ordering() {+ for sort_expr in ordering {+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;+ }+ }+ Ok(tnr)+}+```diff --git a/docs/source/library-user-guide/upgrading/index.rst b/docs/source/library-user-guide/upgrading/index.rstindex 16bb33b759..1ed5eca2a5 100644--- a/docs/source/library-user-guide/upgrading/index.rst+++ b/docs/source/library-user-guide/upgrading/index.rst@@ -21,6 +21,7 @@ Upgrade Guides.. toctree:::maxdepth: 1+ DataFusion 54.0.0 <54.0.0>DataFusion 53.0.0 <53.0.0>DataFusion 52.0.0 <52.0.0>DataFusion 51.0.0 <51.0.0>