Skip to content

Apache DataFusion SQL Query Engine

Add `ExecutionPlan::apply_expressions()` (#20337)

Lía Adrianaa5f490e4c311
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
index 7abb39e1a7..71e589dcf6 100644
--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
@@ -27,6 +27,7 @@ use async_trait::async_trait;
use datafusion::arrow::array::{UInt8Builder, UInt64Builder};
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::datasource::{TableProvider, TableType, provider_as_source};
use datafusion::error::Result;
use datafusion::execution::context::TaskContext;
@@ -283,4 +284,20 @@ impl ExecutionPlan for CustomExec {
None,
)?))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
index 4c05cd2fb1..1440347d44 100644
--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
@@ -29,6 +29,7 @@
use arrow::record_batch::RecordBatch;
use arrow_schema::SchemaRef;
use datafusion::common::record_batch;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::common::{exec_datafusion_err, internal_err};
use datafusion::datasource::{DefaultTableSource, memory::MemTable};
use datafusion::error::Result;
@@ -296,4 +297,20 @@ impl ExecutionPlan for BufferingExecutionPlan {
}),
)))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.properties.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rs
index b4f3d4f098..df3d58b7bf 100644
--- a/datafusion-examples/examples/proto/composed_extension_codec.rs
+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs
@@ -38,6 +38,7 @@ use std::sync::Arc;
use datafusion::common::Result;
use datafusion::common::internal_err;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::execution::TaskContext;
use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
use datafusion::prelude::SessionContext;
@@ -128,6 +129,15 @@ impl ExecutionPlan for ParentExec {
) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
unreachable!()
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// A PhysicalExtensionCodec that can serialize and deserialize ParentExec
@@ -204,6 +214,15 @@ impl ExecutionPlan for ChildExec {
) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
unreachable!()
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// A PhysicalExtensionCodec that can serialize and deserialize ChildExec
diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs
index 895f2fdd4f..aa24f6cfbf 100644
--- a/datafusion-examples/examples/relation_planner/table_sample.rs
+++ b/datafusion-examples/examples/relation_planner/table_sample.rs
@@ -117,7 +117,7 @@ use datafusion::{
};
use datafusion_common::{
DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
- plan_datafusion_err, plan_err,
+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,
};
use datafusion_expr::{
UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
@@ -743,6 +743,22 @@ impl ExecutionPlan for SampleExec {
Ok(stats)
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
/// Bernoulli sampler: includes each row with probability `(upper - lower)`.
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs
index 484b5f805e..9b91062657 100644
--- a/datafusion/catalog/src/memory/table.rs
+++ b/datafusion/catalog/src/memory/table.rs
@@ -32,6 +32,7 @@ use arrow::compute::{and, filter_record_batch};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::error::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};
use datafusion_common_runtime::JoinSet;
use datafusion_datasource::memory::{MemSink, MemorySourceConfig};
@@ -46,7 +47,7 @@ use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning,
- PlanProperties, common,
+ PhysicalExpr, PlanProperties, common,
};
use datafusion_session::Session;
@@ -400,10 +401,7 @@ impl TableProvider for MemTable {
let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
// Create physical expressions for assignments upfront (outside batch loop)
- let physical_assignments: HashMap<
- String,
- Arc<dyn datafusion_physical_plan::PhysicalExpr>,
- > = assignments
+ let physical_assignments: HashMap<String, Arc<dyn PhysicalExpr>> = assignments
.iter()
.map(|(name, expr)| {
let physical_expr =
@@ -638,4 +636,11 @@ impl ExecutionPlan for DmlResultExec {
stream,
)))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 14f3e5cf03..8d52f555bb 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -3891,6 +3891,20 @@ mod tests {
) -> Result<SendableRecordBatchStream> {
unimplemented!("NoOpExecutionPlan::execute");
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
// Produces an execution plan where the schema is mismatched from
@@ -4033,6 +4047,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for OkExtensionNode {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
@@ -4082,6 +4102,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for InvariantFailsExtensionNode {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
@@ -4203,6 +4229,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for ExecutableInvariantFails {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rs
index f51d0a1e36..5f20fb1c60 100644
--- a/datafusion/core/tests/custom_sources_cases/mod.rs
+++ b/datafusion/core/tests/custom_sources_cases/mod.rs
@@ -39,6 +39,7 @@ use datafusion_catalog::Session;
use datafusion_common::cast::as_primitive_array;
use datafusion_common::project_schema;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::PlanProperties;
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -209,6 +210,22 @@ impl ExecutionPlan for CustomExecutionPlan {
.collect(),
})
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[async_trait]
diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
index 96357d3103..6bb119fcb7 100644
--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
@@ -35,6 +35,7 @@ use datafusion::prelude::*;
use datafusion::scalar::ScalarValue;
use datafusion_catalog::Session;
use datafusion_common::cast::as_primitive_array;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, internal_err, not_impl_err};
use datafusion_expr::expr::{BinaryExpr, Cast};
use datafusion_functions_aggregate::expr_fn::count;
@@ -152,6 +153,22 @@ impl ExecutionPlan for CustomPlan {
})),
)))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[derive(Clone, Debug)]
diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rs
index 03513ec730..eb81c9f24d 100644
--- a/datafusion/core/tests/custom_sources_cases/statistics.rs
+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs
@@ -33,6 +33,7 @@ use datafusion::{
scalar::ScalarValue,
};
use datafusion_catalog::Session;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{project_schema, stats::Precision};
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -188,6 +189,22 @@ impl ExecutionPlan for StatisticsValidation {
Ok(self.stats.clone())
}
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
fn init_ctx(stats: Statistics, schema: Schema) -> Result<SessionContext> {
diff --git a/datafusion/core/tests/fuzz_cases/once_exec.rs b/datafusion/core/tests/fuzz_cases/once_exec.rs
index 69edf9be1d..eed172f09f 100644
--- a/datafusion/core/tests/fuzz_cases/once_exec.rs
+++ b/datafusion/core/tests/fuzz_cases/once_exec.rs
@@ -17,6 +17,7 @@
use arrow_schema::SchemaRef;
use datafusion_common::internal_datafusion_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -110,4 +111,20 @@ impl ExecutionPlan for OnceExec {
stream.ok_or_else(|| internal_datafusion_err!("Stream already consumed"))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> datafusion_common::Result<TreeNodeRecursion>,
+ ) -> datafusion_common::Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index 5df634c70b..993798ff75 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -40,7 +40,9 @@ use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::ScalarValue;
use datafusion_common::config::CsvOptions;
use datafusion_common::error::Result;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_datasource::file_groups::FileGroup;
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_expr::{JoinType, Operator};
@@ -202,6 +204,20 @@ impl ExecutionPlan for SortRequiredExec {
)))
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn execute(
&self,
_partition: usize,
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index ac64727481..64d1c7bf10 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -4087,6 +4087,107 @@ async fn test_filter_with_projection_pushdown() {
assert_batches_eq!(expected, &result);
}
+/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree
+#[tokio::test]
+async fn test_discover_dynamic_filters_via_expressions_api() {
+ use datafusion_common::JoinType;
+ use datafusion_common::tree_node::TreeNodeRecursion;
+ use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
+ use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
+
+ fn count_dynamic_filters(plan: &Arc<dyn ExecutionPlan>) -> usize {
+ let mut count = 0;
+
+ // Check expressions from this node using apply_expressions
+ let _ = plan.apply_expressions(&mut |expr| {
+ if let Some(_df) = expr.as_any().downcast_ref::<DynamicFilterPhysicalExpr>() {
+ count += 1;
+ }
+ Ok(TreeNodeRecursion::Continue)
+ });
+
+ // Recursively visit children
+ for child in plan.children() {
+ count += count_dynamic_filters(child);
+ }
+
+ count
+ }
+
+ // Create build side (left)
+ let build_batches =
+ vec![record_batch!(("a", Utf8, ["foo", "bar"]), ("b", Int32, [1, 2])).unwrap()];
+ let build_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let build_scan = TestScanBuilder::new(build_schema.clone())
+ .with_support(true)
+ .with_batches(build_batches)
+ .build();
+
+ // Create probe side (right)
+ let probe_batches = vec![
+ record_batch!(
+ ("a", Utf8, ["foo", "bar", "baz", "qux"]),
+ ("c", Float64, [1.0, 2.0, 3.0, 4.0])
+ )
+ .unwrap(),
+ ];
+ let probe_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("c", DataType::Float64, false),
+ ]));
+ let probe_scan = TestScanBuilder::new(probe_schema.clone())
+ .with_support(true)
+ .with_batches(probe_batches)
+ .build();
+
+ // Create HashJoinExec
+ let plan = Arc::new(
+ HashJoinExec::try_new(
+ build_scan,
+ probe_scan,
+ vec![(
+ col("a", &build_schema).unwrap(),
+ col("a", &probe_schema).unwrap(),
+ )],
+ None,
+ &JoinType::Inner,
+ None,
+ PartitionMode::CollectLeft,
+ datafusion_common::NullEquality::NullEqualsNothing,
+ false,
+ )
+ .unwrap(),
+ ) as Arc<dyn ExecutionPlan>;
+
+ // Before optimization: no dynamic filters
+ let count_before = count_dynamic_filters(&plan);
+ assert_eq!(
+ count_before, 0,
+ "Before optimization, should have no dynamic filters"
+ );
+
+ // Apply filter pushdown optimization (this creates dynamic filters)
+ let mut config = ConfigOptions::default();
+ config.optimizer.enable_dynamic_filter_pushdown = true;
+ config.execution.parquet.pushdown_filters = true;
+ let optimized_plan = FilterPushdown::new_post_optimization()
+ .optimize(plan, &config)
+ .unwrap();
+
+ // After optimization: should discover dynamic filters
+ // We expect 2 dynamic filters:
+ // 1. In the HashJoinExec (producer)
+ // 2. In the DataSourceExec (consumer, pushed down to the probe side)
+ let count_after = count_dynamic_filters(&optimized_plan);
+ assert_eq!(
+ count_after, 2,
+ "After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}"
+ );
+}
+
#[tokio::test]
async fn test_hashjoin_dynamic_filter_pushdown_left_join() {
use datafusion_common::JoinType;
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs
index ef0bbfc7f4..17156022da 100644
--- a/datafusion/core/tests/physical_optimizer/join_selection.rs
+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs
@@ -26,6 +26,7 @@ use std::{
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{ColumnStatistics, JoinType, ScalarValue, stats::Precision};
use datafusion_common::{JoinSide, NullEquality};
use datafusion_common::{Result, Statistics};
@@ -1078,6 +1079,20 @@ impl ExecutionPlan for UnboundedExec {
batch: self.batch.clone(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[derive(Eq, PartialEq, Debug)]
@@ -1183,6 +1198,20 @@ impl ExecutionPlan for StatisticsExec {
self.stats.clone()
})
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[test]
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
index 91ae6c414e..ce2cb04b64 100644
--- a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
@@ -18,6 +18,7 @@
use arrow::datatypes::SchemaRef;
use arrow::{array::RecordBatch, compute::concat_batches};
use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, config::ConfigOptions, internal_err};
use datafusion_datasource::{
PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,
@@ -239,6 +240,25 @@ impl FileSource for TestSource {
fn table_schema(&self) -> &datafusion_datasource::TableSchema {
&self.table_schema
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit predicate (filter) expression if present
+ if let Some(predicate) = &self.predicate {
+ f(predicate.as_ref())?;
+ }
+
+ // Visit projection expressions if present
+ if let Some(projection) = &self.projection {
+ for proj_expr in projection {
+ f(proj_expr.expr.as_ref())?;
+ }
+ }
+
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[derive(Debug, Clone)]
@@ -558,4 +578,13 @@ impl ExecutionPlan for TestNode {
Ok(res)
}
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit the predicate expression
+ f(self.predicate.as_ref())?;
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index f8c91ba272..7b000f69a6 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -31,7 +31,9 @@ use datafusion::datasource::physical_plan::ParquetSource;
use datafusion::datasource::source::DataSourceExec;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
use datafusion_common::{
ColumnStatistics, JoinType, NullEquality, Result, Statistics, internal_err,
@@ -492,6 +494,20 @@ impl ExecutionPlan for RequirementsTestExec {
) -> Result<SendableRecordBatchStream> {
unimplemented!("Test exec does not support execution")
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in required_input_ordering if present
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = &self.required_input_ordering {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
/// A [`PlanContext`] object is susceptible to being left in an inconsistent state after
@@ -963,6 +979,28 @@ impl ExecutionPlan for TestScan {
inner: Arc::new(new_scan),
})
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in output_ordering
+ let mut tnr = TreeNodeRecursion::Continue;
+ for ordering in &self.output_ordering {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+
+ // Visit expressions in requested_ordering if present
+ if let Some(ordering) = &self.requested_ordering {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+
+ Ok(tnr)
+ }
}
/// Helper function to create a TestScan with ordering
diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rs
index 4d2a31ca1f..2a2aed82f0 100644
--- a/datafusion/core/tests/user_defined/insert_operation.rs
+++ b/datafusion/core/tests/user_defined/insert_operation.rs
@@ -25,6 +25,7 @@ use datafusion::{
};
use datafusion_catalog::{Session, TableProvider};
use datafusion_common::config::Dialect;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_expr::{Expr, TableType, dml::InsertOp};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use datafusion_physical_plan::execution_plan::SchedulingType;
@@ -184,6 +185,22 @@ impl ExecutionPlan for TestInsertExec {
) -> Result<datafusion_execution::SendableRecordBatchStream> {
unimplemented!("TestInsertExec is a stub for testing.")
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.plan_properties.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
fn make_count_schema() -> SchemaRef {
diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rs
index f97923ffc5..6e4ed69e50 100644
--- a/datafusion/core/tests/user_defined/user_defined_plan.rs
+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs
@@ -90,7 +90,9 @@ use datafusion::{
prelude::{SessionConfig, SessionContext},
};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::{ScalarValue, assert_eq_or_internal_err, assert_or_internal_err};
use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};
use datafusion_optimizer::AnalyzerRule;
@@ -746,6 +748,22 @@ impl ExecutionPlan for TopKExec {
state: BTreeMap::new(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion::physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
// A very specialized TopK implementation
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs
index 99446cb876..0226dadd3d 100644
--- a/datafusion/datasource-arrow/src/source.rs
+++ b/datafusion/datasource-arrow/src/source.rs
@@ -40,6 +40,7 @@ use arrow::buffer::Buffer;
use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};
use datafusion_common::error::Result;
use datafusion_common::exec_datafusion_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::PartitionedFile;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -396,6 +397,20 @@ impl FileSource for ArrowSource {
fn projection(&self) -> Option<&ProjectionExprs> {
Some(&self.projection.source)
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit projection expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for proj_expr in &self.projection.source {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
}
/// `FileOpener` wrapper for both Arrow IPC file and stream formats
diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs
index bd9ff2a7a8..525d72e620 100644
--- a/datafusion/datasource-avro/src/source.rs
+++ b/datafusion/datasource-avro/src/source.rs
@@ -23,6 +23,7 @@ use std::sync::Arc;
use crate::avro_to_arrow::Reader as AvroReader;
use datafusion_common::error::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -139,6 +140,20 @@ impl FileSource for AvroSource {
) -> Result<Option<FileScanConfig>> {
Ok(None)
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit projection expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for proj_expr in &self.projection.source {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
}
mod private {
diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs
index 37e3ef23b6..77a0dc9cf7 100644
--- a/datafusion/datasource-csv/src/source.rs
+++ b/datafusion/datasource-csv/src/source.rs
@@ -35,6 +35,7 @@ use datafusion_datasource::{
use arrow::csv;
use datafusion_common::config::CsvOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result};
use datafusion_common_runtime::JoinSet;
use datafusion_datasource::file::FileSource;
@@ -314,6 +315,20 @@ impl FileSource for CsvSource {
DisplayFormatType::TreeRender => Ok(()),
}
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit projection expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for proj_expr in &self.projection.source {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
}
impl FileOpener for CsvOpener {
diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rs
index 52a38f4994..2f1d5abbee 100644
--- a/datafusion/datasource-json/src/source.rs
+++ b/datafusion/datasource-json/src/source.rs
@@ -27,6 +27,7 @@ use crate::file_format::JsonDecoder;
use crate::utils::{ChannelReader, JsonArrayToNdjsonReader};
use datafusion_common::error::{DataFusionError, Result};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common_runtime::{JoinSet, SpawnedTask};
use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};
use datafusion_datasource::file_compression_type::FileCompressionType;
@@ -236,6 +237,20 @@ impl FileSource for JsonSource {
fn file_type(&self) -> &str {
"json"
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit projection expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for proj_expr in &self.projection.source {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
}
impl FileOpener for JsonOpener {
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index 75d87a4cd1..1355eff5b9 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -35,6 +35,7 @@ use datafusion_datasource::file_stream::FileOpener;
use arrow::datatypes::TimeUnit;
use datafusion_common::DataFusionError;
use datafusion_common::config::TableParquetOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -816,6 +817,26 @@ impl FileSource for ParquetSource {
// - Detection of exact ordering (return Exact to remove Sort operator)
// - Partial sort pushdown for prefix matches
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn PhysicalExpr,
+ ) -> datafusion_common::Result<TreeNodeRecursion>,
+ ) -> datafusion_common::Result<TreeNodeRecursion> {
+ // Visit predicate (filter) expression if present
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(predicate) = &self.predicate {
+ tnr = tnr.visit_sibling(|| f(predicate.as_ref()))?;
+ }
+
+ // Visit projection expressions
+ for proj_expr in &self.projection {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+
+ Ok(tnr)
+ }
}
#[cfg(test)]
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs
index a0f82ff7a9..b5a6760cae 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -28,6 +28,7 @@ use crate::file_stream::FileOpener;
#[expect(deprecated)]
use crate::schema_adapter::SchemaAdapterFactory;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, not_impl_err};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};
@@ -319,4 +320,25 @@ pub trait FileSource: Send + Sync {
fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> {
None
}
+
+ /// Apply a function to all physical expressions used by this file source.
+ ///
+ /// This includes:
+ /// - Filter predicates (which may contain dynamic filters)
+ /// - Projection expressions
+ ///
+ /// The function `f` is called once for each expression. The function should
+ /// return `TreeNodeRecursion::Continue` to continue visiting other expressions,
+ /// or `TreeNodeRecursion::Stop` to stop visiting expressions early.
+ ///
+ /// Implementations must explicitly visit all expressions. There is no default
+ /// implementation to ensure that all FileSource implementations handle this correctly.
+ ///
+ /// See [`ExecutionPlan::apply_expressions`] for more details and examples.
+ ///
+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
}
diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs
index c3e5cabce7..524e091381 100644
--- a/datafusion/datasource/src/file_scan_config.rs
+++ b/datafusion/datasource/src/file_scan_config.rs
@@ -27,6 +27,7 @@ use crate::{
use arrow::datatypes::FieldRef;
use arrow::datatypes::{DataType, Schema, SchemaRef};
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,
};
@@ -78,7 +79,9 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::
/// # use arrow::datatypes::{Field, Fields, DataType, Schema, SchemaRef};
/// # use object_store::ObjectStore;
/// # use datafusion_common::Result;
+/// # use datafusion_common::tree_node::TreeNodeRecursion;
/// # use datafusion_datasource::file::FileSource;
+/// # use datafusion_physical_plan::PhysicalExpr;
/// # use datafusion_datasource::file_groups::FileGroup;
/// # use datafusion_datasource::PartitionedFile;
/// # use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
@@ -109,6 +112,7 @@ use std::{any::Any, fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::
/// # fn file_type(&self) -> &str { "parquet" }
/// # // Note that this implementation drops the projection on the floor, it is not complete!
/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }
+/// # fn apply_expressions(&self, _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }
/// # }
/// # impl ParquetSource {
/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }
@@ -923,6 +927,14 @@ impl DataSource for FileScanConfig {
};
Some(Arc::new(new_config))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Delegate to the file source
+ self.file_source.apply_expressions(f)
+ }
}
impl FileScanConfig {
@@ -1584,6 +1596,13 @@ mod tests {
inner: Arc::new(self.clone()) as Arc<dyn FileSource>,
})
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[test]
diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs
index 1d12bb3200..e6add2a42c 100644
--- a/datafusion/datasource/src/memory.rs
+++ b/datafusion/datasource/src/memory.rs
@@ -29,6 +29,7 @@ use crate::source::{DataSource, DataSourceExec};
use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::datatypes::{Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, ScalarValue, assert_or_internal_err, plan_err, project_schema,
};
@@ -252,6 +253,20 @@ impl DataSource for MemorySourceConfig {
})
.transpose()
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in sort_information
+ let mut tnr = TreeNodeRecursion::Continue;
+ for ordering in &self.sort_information {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
impl MemorySourceConfig {
diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rs
index f149109dff..155c951fe5 100644
--- a/datafusion/datasource/src/sink.rs
+++ b/datafusion/datasource/src/sink.rs
@@ -24,9 +24,10 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, RecordBatch, UInt64Array};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::{Distribution, EquivalenceProperties};
+use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr};
use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequirements};
use datafusion_physical_plan::metrics::MetricsSet;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
@@ -219,6 +220,19 @@ impl ExecutionPlan for DataSinkExec {
)))
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to sort order requirements if present
+ if let Some(sort_order) = &self.sort_order {
+ for req in sort_order.iter() {
+ f(req.expr.as_ref())?;
+ }
+ }
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Execute the plan and return a stream of `RecordBatch`es for
/// the specified partition.
fn execute(
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 05028ed0f4..bdb45e8e3b 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -37,6 +37,7 @@ use itertools::Itertools;
use crate::file_scan_config::FileScanConfig;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Constraints, Result, Statistics};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
@@ -211,6 +212,22 @@ pub trait DataSource: Send + Sync + Debug {
fn with_preserve_order(&self, _preserve_order: bool) -> Option<Arc<dyn DataSource>> {
None
}
+
+ /// Apply a closure to each expression used by this data source.
+ ///
+ /// This includes filter predicates (which may contain dynamic filters) and any
+ /// other expressions used during data scanning.
+ ///
+ /// Implementations must override this method. If the data source has no expressions,
+ /// return `Ok(TreeNodeRecursion::Continue)` immediately.
+ ///
+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.
+ ///
+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
}
/// [`ExecutionPlan`] that reads one or more files
@@ -262,6 +279,14 @@ impl ExecutionPlan for DataSourceExec {
Vec::new()
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Delegate to the underlying data source
+ self.data_source.apply_expressions(f)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rs
index c8d5dd54cb..3a9e78943b 100644
--- a/datafusion/datasource/src/test_util.rs
+++ b/datafusion/datasource/src/test_util.rs
@@ -22,7 +22,7 @@ use crate::{
use std::sync::Arc;
use arrow::datatypes::Schema;
-use datafusion_common::Result;
+use datafusion_common::{Result, tree_node::TreeNodeRecursion};
use datafusion_physical_expr::{PhysicalExpr, expressions::Column};
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
use object_store::ObjectStore;
@@ -119,6 +119,13 @@ impl FileSource for MockSource {
) -> Option<&datafusion_physical_plan::projection::ProjectionExprs> {
Some(&self.projection.source)
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// Create a column expression
diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs
index 524d8b4b6b..064e4a8953 100644
--- a/datafusion/ffi/src/execution_plan.rs
+++ b/datafusion/ffi/src/execution_plan.rs
@@ -21,6 +21,7 @@ use std::sync::Arc;
use abi_stable::StableAbi;
use abi_stable::std_types::{RString, RVec};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_plan::{
@@ -293,6 +294,22 @@ impl ExecutionPlan for ForeignExecutionPlan {
.map(|stream| Pin::new(Box::new(stream)) as SendableRecordBatchStream)
}
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.properties.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[cfg(test)]
@@ -367,6 +384,22 @@ pub(crate) mod tests {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.props.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
#[test]
diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rs
index 8370cf19e6..e9fa31a7fc 100644
--- a/datafusion/ffi/src/tests/async_provider.rs
+++ b/datafusion/ffi/src/tests/async_provider.rs
@@ -33,6 +33,7 @@ use arrow::array::RecordBatch;
use arrow::datatypes::Schema;
use async_trait::async_trait;
use datafusion_catalog::TableProvider;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, exec_err};
use datafusion_execution::RecordBatchStream;
use datafusion_expr::Expr;
@@ -219,6 +220,22 @@ impl ExecutionPlan for AsyncTestExecutionPlan {
batch_receiver: self.batch_receiver.resubscribe(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_plan::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in the output ordering from equivalence properties
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.properties.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
}
impl datafusion_physical_plan::DisplayAs for AsyncTestExecutionPlan {
diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rs
index ef8946f9a4..6925c036e4 100644
--- a/datafusion/physical-optimizer/src/ensure_coop.rs
+++ b/datafusion/physical-optimizer/src/ensure_coop.rs
@@ -264,11 +264,12 @@ mod tests {
async fn test_eager_evaluation_resets_cooperative_context() {
// Test that cooperative context is reset when encountering an eager evaluation boundary.
use arrow::datatypes::Schema;
+ use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::{
- DisplayAs, DisplayFormatType, Partitioning, PlanProperties,
+ DisplayAs, DisplayFormatType, Partitioning, PhysicalExpr, PlanProperties,
SendableRecordBatchStream,
execution_plan::{Boundedness, EmissionType},
};
@@ -351,6 +352,13 @@ mod tests {
) -> Result<SendableRecordBatchStream> {
internal_err!("DummyExec does not support execution")
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
// Build a plan similar to the original test:
diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rs
index 75721951f8..b2212c94bc 100644
--- a/datafusion/physical-optimizer/src/output_requirements.rs
+++ b/datafusion/physical-optimizer/src/output_requirements.rs
@@ -27,7 +27,9 @@ use std::sync::Arc;
use crate::PhysicalOptimizerRule;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::{Result, Statistics};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::Distribution;
@@ -296,6 +298,36 @@ impl ExecutionPlan for OutputRequirementExec {
fn fetch(&self) -> Option<usize> {
self.fetch
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Visit expressions in order_requirement
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(order_reqs) = &self.order_requirement {
+ let lexes = match order_reqs {
+ OrderingRequirements::Hard(alternatives) => alternatives,
+ OrderingRequirements::Soft(alternatives) => alternatives,
+ };
+ for lex in lexes {
+ for sort_expr in lex {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ }
+
+ // Visit expressions in dist_requirement if it's HashPartitioned
+ if let Distribution::HashPartitioned(exprs) = &self.dist_requirement {
+ for expr in exprs {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ }
+
+ Ok(tnr)
+ }
}
impl PhysicalOptimizerRule for OutputRequirements {
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index 8599993851..561e9d1d05 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -45,6 +45,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_schema::FieldRef;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraint, Constraints, Result, ScalarValue, assert_eq_or_internal_err, not_impl_err,
};
@@ -1382,6 +1383,36 @@ impl ExecutionPlan for AggregateExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to group by expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for expr in self.group_by.input_exprs() {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+
+ // Apply to aggregate expressions
+ for aggr in self.aggr_expr.iter() {
+ for expr in aggr.expressions() {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ }
+
+ // Apply to filter expressions (FILTER WHERE clauses)
+ for filter in self.filter_expr.iter().flatten() {
+ tnr = tnr.visit_sibling(|| f(filter.as_ref()))?;
+ }
+
+ // Apply to dynamic filter expression if present
+ if let Some(dyn_filter) = &self.dynamic_filter {
+ tnr = tnr.visit_sibling(|| f(dyn_filter.filter.as_ref()))?;
+ }
+
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
@@ -2479,6 +2510,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index eca31ea0e1..4aa78055da 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -31,9 +31,11 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::instant::Instant;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::PhysicalExpr;
use futures::StreamExt;
@@ -143,6 +145,13 @@ impl ExecutionPlan for AnalyzeExec {
vec![Distribution::UnspecifiedDistribution]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rs
index 72741f4314..abfe870f52 100644
--- a/datafusion/physical-plan/src/async_func.rs
+++ b/datafusion/physical-plan/src/async_func.rs
@@ -24,7 +24,8 @@ use crate::{
};
use arrow::array::RecordBatch;
use arrow_schema::{Fields, Schema, SchemaRef};
-use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
+use datafusion_common::tree_node::TreeNodeRecursion;
+use datafusion_common::tree_node::{Transformed, TreeNode};
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::ScalarFunctionExpr;
@@ -169,6 +170,13 @@ impl ExecutionPlan for AsyncFuncExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rs
index a59d062929..8477f39a9b 100644
--- a/datafusion/physical-plan/src/buffer.rs
+++ b/datafusion/physical-plan/src/buffer.rs
@@ -31,6 +31,7 @@ use crate::{
};
use arrow::array::RecordBatch;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, Statistics, internal_err, plan_err};
use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
@@ -172,6 +173,13 @@ impl ExecutionPlan for BufferExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs
index 663b0b51ea..d26d1f9bea 100644
--- a/datafusion/physical-plan/src/coalesce_batches.rs
+++ b/datafusion/physical-plan/src/coalesce_batches.rs
@@ -33,6 +33,7 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
@@ -187,6 +188,13 @@ impl ExecutionPlan for CoalesceBatchesExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index 39906d3680..3dffdd6a99 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -35,6 +35,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_proper
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
@@ -158,6 +159,13 @@ impl ExecutionPlan for CoalescePartitionsExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index 5f0040b3dd..c7a5432994 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -71,6 +71,7 @@
//! that report [`SchedulingType::NonCooperative`] in their [plan properties](ExecutionPlan::properties).
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_physical_expr::PhysicalExpr;
#[cfg(datafusion_coop = "tokio_fallback")]
use futures::Future;
@@ -281,6 +282,13 @@ impl ExecutionPlan for CooperativeExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs
index 44148f2d0e..c69ac03178 100644
--- a/datafusion/physical-plan/src/display.rs
+++ b/datafusion/physical-plan/src/display.rs
@@ -1120,8 +1120,11 @@ mod tests {
use std::fmt::Write;
use std::sync::Arc;
- use datafusion_common::{Result, Statistics, internal_datafusion_err};
+ use datafusion_common::{
+ Result, Statistics, internal_datafusion_err, tree_node::TreeNodeRecursion,
+ };
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+ use datafusion_physical_expr::PhysicalExpr;
use crate::{DisplayAs, ExecutionPlan, PlanProperties};
@@ -1168,6 +1171,13 @@ mod tests {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_: usize,
diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs
index e4d4da4e88..db3f5161a9 100644
--- a/datafusion/physical-plan/src/empty.rs
+++ b/datafusion/physical-plan/src/empty.rs
@@ -30,9 +30,10 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use crate::execution_plan::SchedulingType;
use log::trace;
@@ -123,6 +124,13 @@ impl ExecutionPlan for EmptyExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index 681a1345d8..0a9348b68a 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -27,7 +27,9 @@ pub use crate::stream::EmptyRecordBatchStream;
use arrow_schema::Schema;
pub use datafusion_common::hash_utils;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
pub use datafusion_common::utils::project_schema;
pub use datafusion_common::{ColumnStatistics, Statistics, internal_err};
pub use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};
@@ -205,6 +207,80 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync {
/// joins).
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>>;
+ /// Apply a closure `f` to each expression (non-recursively) in the current
+ /// physical plan node. This does not include expressions in any children.
+ ///
+ /// The closure `f` is applied to expressions in the order they appear in the plan.
+ /// The closure can return `TreeNodeRecursion::Continue` to continue visiting,
+ /// `TreeNodeRecursion::Stop` to stop visiting immediately, or `TreeNodeRecursion::Jump`
+ /// to skip any remaining expressions (though typically all expressions are visited).
+ ///
+ /// The expressions visited do not necessarily represent or even contribute
+ /// to the output schema of this node. For example, `FilterExec` visits the
+ /// filter predicate even though the output of a Filter has the same columns
+ /// as the input.
+ ///
+ /// # Example Usage
+ /// ```
+ /// # use std::sync::Arc;
+ /// # use datafusion_physical_plan::ExecutionPlan;
+ /// # use datafusion_common::tree_node::TreeNodeRecursion;
+ /// # fn example(plan: Arc<dyn ExecutionPlan>) -> datafusion_common::Result<()> {
+ /// // Count the number of expressions
+ /// let mut count = 0;
+ /// plan.apply_expressions(&mut |_expr| {
+ /// count += 1;
+ /// Ok(TreeNodeRecursion::Continue)
+ /// })?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Implementation Examples
+ ///
+ /// ## Node with no expressions (e.g., EmptyExec, MemoryExec)
+ /// ```ignore
+ /// fn apply_expressions(
+ /// &self,
+ /// _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ /// ) -> Result<TreeNodeRecursion> {
+ /// Ok(TreeNodeRecursion::Continue)
+ /// }
+ /// ```
+ ///
+ /// ## Node with a single expression (e.g., FilterExec)
+ /// ```ignore
+ /// fn apply_expressions(
+ /// &self,
+ /// f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ /// ) -> Result<TreeNodeRecursion> {
+ /// f(self.predicate.as_ref())
+ /// }
+ /// ```
+ ///
+ /// ## Node with multiple expressions (e.g., ProjectionExec, JoinExec)
+ ///
+ /// Use [`TreeNodeRecursion::visit_sibling`] when iterating over multiple
+ /// expressions. This correctly propagates [`TreeNodeRecursion::Stop`]: if
+ /// `f` returns `Stop` for an earlier expression, `visit_sibling` short-circuits
+ /// and skips the remaining ones.
+ /// ```ignore
+ /// fn apply_expressions(
+ /// &self,
+ /// f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ /// ) -> Result<TreeNodeRecursion> {
+ /// let mut tnr = TreeNodeRecursion::Continue;
+ /// for expr in &self.expressions {
+ /// tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ /// }
+ /// Ok(tnr)
+ /// }
+ /// ```
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
+
/// Returns a new `ExecutionPlan` where all existing children were replaced
/// by the `children`, in order
fn with_new_children(
@@ -1549,6 +1625,13 @@ mod tests {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
@@ -1605,6 +1688,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -1625,6 +1715,110 @@ mod tests {
}
}
+ /// A test node that holds a fixed list of expressions, used to test
+ /// `apply_expressions` behavior.
+ #[derive(Debug)]
+ struct MultiExprExec {
+ exprs: Vec<Arc<dyn PhysicalExpr>>,
+ }
+
+ impl DisplayAs for MultiExprExec {
+ fn fmt_as(
+ &self,
+ _t: DisplayFormatType,
+ _f: &mut std::fmt::Formatter,
+ ) -> std::fmt::Result {
+ unimplemented!()
+ }
+ }
+
+ impl ExecutionPlan for MultiExprExec {
+ fn name(&self) -> &'static str {
+ "MultiExprExec"
+ }
+
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+
+ fn properties(&self) -> &std::sync::Arc<PlanProperties> {
+ unimplemented!()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![]
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unimplemented!()
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for expr in &self.exprs {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unimplemented!()
+ }
+
+ fn partition_statistics(&self, _partition: Option<usize>) -> Result<Statistics> {
+ unimplemented!()
+ }
+ }
+
+ /// Returns a simple literal `Arc<dyn PhysicalExpr>` for use in tests.
+ fn lit_expr(val: i64) -> Arc<dyn PhysicalExpr> {
+ use datafusion_physical_expr::expressions::Literal;
+ Arc::new(Literal::new(datafusion_common::ScalarValue::Int64(Some(
+ val,
+ ))))
+ }
+
+ /// `apply_expressions` visits all expressions when `f` always returns `Continue`.
+ #[test]
+ fn test_apply_expressions_continue_visits_all() -> Result<()> {
+ let plan = MultiExprExec {
+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],
+ };
+ let mut visited = 0usize;
+ plan.apply_expressions(&mut |_expr| {
+ visited += 1;
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ assert_eq!(visited, 3);
+ Ok(())
+ }
+
+ #[test]
+ fn test_apply_expressions_stop_halts_early() -> Result<()> {
+ let plan = MultiExprExec {
+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],
+ };
+ let mut visited = 0usize;
+ let tnr = plan.apply_expressions(&mut |_expr| {
+ visited += 1;
+ Ok(TreeNodeRecursion::Stop)
+ })?;
+ // Only the first expression is visited; the rest are skipped.
+ assert_eq!(visited, 1);
+ assert_eq!(tnr, TreeNodeRecursion::Stop);
+ Ok(())
+ }
+
#[test]
fn test_execution_plan_name() {
let schema1 = Arc::new(Schema::empty());
diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs
index bf21b04846..fa684f3483 100644
--- a/datafusion/physical-plan/src/explain.rs
+++ b/datafusion/physical-plan/src/explain.rs
@@ -27,9 +27,10 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::display::StringifiedPlan;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use log::trace;
@@ -121,6 +122,13 @@ impl ExecutionPlan for ExplainExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index ecea4e6ebe..4622439b98 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -51,6 +51,7 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, Result, ScalarValue, internal_err, plan_err, project_schema,
};
@@ -506,6 +507,13 @@ impl ExecutionPlan for FilterExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ f(self.predicate.as_ref())
+ }
+
fn maintains_input_order(&self) -> Vec<bool> {
// Tell optimizer this operator doesn't reorder its input
vec![true]
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index 342cb7e70a..bdad207503 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -41,11 +41,13 @@ use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::compute::concat_batches;
use arrow::datatypes::{Fields, Schema, SchemaRef};
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinType, Result, ScalarValue, assert_eq_or_internal_err, internal_err,
};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::equivalence::join_equivalence_properties;
use async_trait::async_trait;
@@ -285,6 +287,14 @@ impl ExecutionPlan for CrossJoinExec {
Some(self.metrics.clone_inner())
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // CrossJoin has no join conditions or expressions
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index b6ce472bbe..b5e8d3c1e3 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -69,6 +69,7 @@ use arrow::record_batch::RecordBatch;
use arrow::util::bit_util;
use arrow_schema::{DataType, Schema};
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::memory::estimate_memory_size;
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,
@@ -1231,6 +1232,30 @@ impl ExecutionPlan for HashJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to join key expressions from both sides
+ let mut tnr = TreeNodeRecursion::Continue;
+ for (left, right) in &self.on {
+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;
+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;
+ }
+
+ // Apply to join filter expression if present
+ if let Some(filter) = &self.filter {
+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;
+ }
+
+ // Apply to dynamic filter expression if present
+ if let Some(df) = &self.dynamic_filter {
+ tnr = tnr.visit_sibling(|| f(df.filter.as_ref()))?;
+ }
+
+ Ok(tnr)
+ }
+
/// Creates a new HashJoinExec with different children while preserving configuration.
///
/// This method is called during query optimization when the optimizer creates new
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 4fb7dabf67..1ba6ee23ca 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -61,6 +61,7 @@ use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_schema::DataType;
use datafusion_common::cast::as_boolean_array;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinSide, Result, ScalarValue, Statistics, arrow_err, assert_eq_or_internal_err,
internal_datafusion_err, internal_err, project_schema, unwrap_or_internal_err,
@@ -557,6 +558,17 @@ impl ExecutionPlan for NestedLoopJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to join filter expressions if present
+ if let Some(filter) = &self.filter {
+ f(filter.expression().as_ref())?;
+ }
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
index abb6e34aa2..fb1c4b1605 100644
--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
@@ -23,6 +23,7 @@ use arrow::{
};
use arrow_schema::{SchemaRef, SortOptions};
use datafusion_common::not_impl_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{JoinSide, Result, internal_err};
use datafusion_execution::{
SendableRecordBatchStream,
@@ -511,6 +512,14 @@ impl ExecutionPlan for PiecewiseMergeJoinExec {
vec![&self.buffered, &self.streamed]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to the two expressions being compared in the range predicate
+ f(self.on.0.as_ref())?.visit_sibling(|| f(self.on.1.as_ref()))
+ }
+
fn required_input_distribution(&self) -> Vec<Distribution> {
vec![
Distribution::SinglePartition,
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index b34e811f91..7a01888c28 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -44,6 +44,7 @@ use crate::{
use arrow::compute::SortOptions;
use arrow::datatypes::SchemaRef;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err, internal_err,
plan_err,
@@ -450,6 +451,23 @@ impl ExecutionPlan for SortMergeJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to join keys from both sides
+ let mut tnr = TreeNodeRecursion::Continue;
+ for (left, right) in &self.on {
+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;
+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;
+ }
+ // Apply to join filter expressions if present
+ if let Some(filter) = &self.filter {
+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index 29917dd2cc..eeb160ae51 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -66,6 +66,7 @@ use arrow::compute::concat_batches;
use arrow::datatypes::{ArrowNativeType, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::hash_utils::create_hashes;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::bisect;
use datafusion_common::{
HashSet, JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err,
@@ -464,6 +465,23 @@ impl ExecutionPlan for SymmetricHashJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn crate::PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to join keys from both sides
+ let mut tnr = TreeNodeRecursion::Continue;
+ for (left, right) in &self.on {
+ tnr = tnr.visit_sibling(|| f(left.as_ref()))?;
+ tnr = tnr.visit_sibling(|| f(right.as_ref()))?;
+ }
+ // Apply to join filter expressions if present
+ if let Some(filter) = &self.filter {
+ tnr = tnr.visit_sibling(|| f(filter.expression().as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index a78e5c067f..63a722ef26 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -35,10 +35,11 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::LexOrdering;
+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};
use futures::stream::{Stream, StreamExt};
use log::trace;
@@ -178,6 +179,20 @@ impl ExecutionPlan for GlobalLimitExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to required ordering expressions if present
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = &self.required_ordering {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -353,6 +368,20 @@ impl ExecutionPlan for LocalLimitExec {
vec![true]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to required ordering expressions if present
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = &self.required_ordering {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs
index 90fd3f24cf..d607f2b440 100644
--- a/datafusion/physical-plan/src/memory.rs
+++ b/datafusion/physical-plan/src/memory.rs
@@ -32,10 +32,11 @@ use crate::{
use arrow::array::RecordBatch;
use arrow::datatypes::SchemaRef;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, assert_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryReservation;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use futures::Stream;
@@ -315,6 +316,13 @@ impl ExecutionPlan for LazyMemoryExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs
index 5dbd7b3032..68fe62f168 100644
--- a/datafusion/physical-plan/src/placeholder_row.rs
+++ b/datafusion/physical-plan/src/placeholder_row.rs
@@ -30,9 +30,11 @@ use crate::{
use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::PhysicalExpr;
use log::trace;
@@ -140,6 +142,13 @@ impl ExecutionPlan for PlaceholderRowExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index db3a71fc70..a3d940df13 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -318,6 +318,17 @@ impl ExecutionPlan for ProjectionExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for proj_expr in self.projector.projection().as_ref().iter() {
+ tnr = tnr.visit_sibling(|| f(proj_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs
index 995aa4822a..049aa9563d 100644
--- a/datafusion/physical-plan/src/recursive_query.rs
+++ b/datafusion/physical-plan/src/recursive_query.rs
@@ -36,10 +36,12 @@ use arrow::array::{BooleanArray, BooleanBuilder};
use arrow::compute::filter_record_batch;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use futures::{Stream, StreamExt, ready};
@@ -151,6 +153,13 @@ impl ExecutionPlan for RecursiveQueryExec {
vec![&self.static_term, &self.recursive_term]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
// TODO: control these hints and see whether we can
// infer some from the child plans (static/recursive terms).
fn maintains_input_order(&self) -> Vec<bool> {
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index da4329e2cc..081f10d482 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -49,6 +49,7 @@ use arrow::compute::take_arrays;
use arrow::datatypes::{SchemaRef, UInt32Type};
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::transpose;
use datafusion_common::{
ColumnStatistics, DataFusionError, HashMap, assert_or_internal_err,
@@ -914,6 +915,21 @@ impl ExecutionPlan for RepartitionExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to hash partition expressions if this is a hash repartition
+ if let Partitioning::Hash(exprs, _) = self.partitioning() {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for expr in exprs {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ return Ok(tnr);
+ }
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs
index 0dbb75f2ef..6df3babb8e 100644
--- a/datafusion/physical-plan/src/sorts/partial_sort.rs
+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs
@@ -69,9 +69,10 @@ use arrow::compute::concat_batches;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::evaluate_partition_ranges;
use datafusion_execution::{RecordBatchStream, TaskContext};
-use datafusion_physical_expr::LexOrdering;
+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};
use futures::{Stream, StreamExt, ready};
use log::trace;
@@ -287,6 +288,17 @@ impl ExecutionPlan for PartialSortExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for sort_expr in &self.expr {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index b3ea548d53..6447b3c310 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -58,6 +58,7 @@ use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray};
use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
use arrow::datatypes::SchemaRef;
use datafusion_common::config::SpillCompression;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
unwrap_or_internal_err,
@@ -1194,6 +1195,25 @@ impl ExecutionPlan for SortExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to sort expressions
+ let mut tnr = TreeNodeRecursion::Continue;
+ for sort_expr in &self.expr {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+
+ // Apply to dynamic filter expression if present (when fetch is Some, TopK mode)
+ if let Some(filter) = &self.filter {
+ let filter_guard = filter.read();
+ tnr = tnr.visit_sibling(|| f(filter_guard.expr().as_ref()))?;
+ }
+
+ Ok(tnr)
+ }
+
fn benefits_from_input_partitioning(&self) -> Vec<bool> {
vec![false]
}
@@ -1490,6 +1510,13 @@ mod tests {
Ok(self)
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 763b72a660..7cbc21f8cb 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -31,9 +31,11 @@ use crate::{
check_if_same_properties,
};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryConsumer;
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};
use crate::execution_plan::{EvaluationType, SchedulingType};
@@ -290,6 +292,17 @@ impl ExecutionPlan for SortPreservingMergeExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for sort_expr in &self.expr {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -1413,6 +1426,12 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs
index 1535482374..5a1206629a 100644
--- a/datafusion/physical-plan/src/streaming.rs
+++ b/datafusion/physical-plan/src/streaming.rs
@@ -34,8 +34,10 @@ use crate::stream::RecordBatchStreamAdapter;
use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};
use arrow::datatypes::{Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, internal_err, plan_err};
use datafusion_execution::TaskContext;
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};
use async_trait::async_trait;
@@ -248,6 +250,13 @@ impl ExecutionPlan for StreamingTableExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs
index 0e7b900eb6..499c10413a 100644
--- a/datafusion/physical-plan/src/test.rs
+++ b/datafusion/physical-plan/src/test.rs
@@ -36,6 +36,7 @@ use crate::{DisplayAs, DisplayFormatType, PlanProperties};
use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, Statistics, assert_or_internal_err, config::ConfigOptions, project_schema,
};
@@ -45,7 +46,9 @@ use datafusion_physical_expr::equivalence::{
};
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::utils::collect_columns;
-use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, Partitioning};
+use datafusion_physical_expr::{
+ EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr,
+};
use futures::{Future, FutureExt};
@@ -142,6 +145,20 @@ impl ExecutionPlan for TestMemoryExec {
Vec::new()
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to all sort information orderings
+ let mut tnr = TreeNodeRecursion::Continue;
+ for ordering in &self.sort_information {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rs
index d628fb819f..7b31855ce8 100644
--- a/datafusion/physical-plan/src/test/exec.rs
+++ b/datafusion/physical-plan/src/test/exec.rs
@@ -36,9 +36,10 @@ use std::{
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use futures::Stream;
use tokio::sync::Barrier;
@@ -200,6 +201,13 @@ impl ExecutionPlan for MockExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -437,6 +445,13 @@ impl ExecutionPlan for BarrierExec {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Returns a stream which yields data
fn execute(
&self,
@@ -573,6 +588,13 @@ impl ExecutionPlan for ErrorExec {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Returns a stream which yields data
fn execute(
&self,
@@ -656,6 +678,13 @@ impl ExecutionPlan for StatisticsExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -767,6 +796,13 @@ impl ExecutionPlan for BlockingExec {
internal_err!("Children cannot be replaced in {self:?}")
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
@@ -906,6 +942,13 @@ impl ExecutionPlan for PanicExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 9fc02e730d..5c4f821c98 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -50,6 +50,7 @@ use arrow::datatypes::{Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, assert_or_internal_err, exec_err, internal_datafusion_err,
};
@@ -273,6 +274,13 @@ impl ExecutionPlan for UnionExec {
self.inputs.iter().collect()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
@@ -589,6 +597,13 @@ impl ExecutionPlan for InterleaveExec {
vec![false; self.inputs().len()]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs
index 422a9dd0d3..72faeb83c5 100644
--- a/datafusion/physical-plan/src/unnest.rs
+++ b/datafusion/physical-plan/src/unnest.rs
@@ -42,6 +42,7 @@ use arrow::datatypes::{DataType, Int64Type, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_ord::cmp::lt;
use async_trait::async_trait;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraints, HashMap, HashSet, Result, UnnestOptions, exec_datafusion_err, exec_err,
internal_err,
@@ -240,6 +241,13 @@ impl ExecutionPlan for UnnestExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
index a31268b9c6..0a3d592776 100644
--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
@@ -48,6 +48,7 @@ use arrow::{
};
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::{
evaluate_partition_ranges, get_at_indices, get_row_at_idx,
};
@@ -323,6 +324,19 @@ impl ExecutionPlan for BoundedWindowAggExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for window_expr in &self.window_expr {
+ for expr in window_expr.expressions() {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {
let partition_bys = self.window_expr()[0].partition_by();
let order_keys = self.window_expr()[0].order_by();
diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs
index 0a146d51d6..01710d5b3c 100644
--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs
@@ -41,6 +41,7 @@ use arrow::datatypes::SchemaRef;
use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::{evaluate_partition_ranges, transpose};
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
@@ -225,6 +226,19 @@ impl ExecutionPlan for WindowAggExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for window_expr in &self.window_expr {
+ for expr in window_expr.expressions() {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+ }
+
fn maintains_input_order(&self) -> Vec<bool> {
vec![true]
}
diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs
index 4c7f77e0ff..4e8cfeea39 100644
--- a/datafusion/physical-plan/src/work_table.rs
+++ b/datafusion/physical-plan/src/work_table.rs
@@ -31,10 +31,11 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryReservation;
-use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
+use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
/// A vector of record batches with a memory reservation.
#[derive(Debug)]
@@ -189,6 +190,13 @@ impl ExecutionPlan for WorkTableExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md
index 50005a7527..70b6be3ae2 100644
--- a/docs/source/library-user-guide/custom-table-providers.md
+++ b/docs/source/library-user-guide/custom-table-providers.md
@@ -52,11 +52,12 @@ use std::any::Any;
use std::sync::{Arc, Mutex};
use std::collections::{BTreeMap, HashMap};
use datafusion::common::Result;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::physical_plan::expressions::PhysicalSortExpr;
use datafusion::physical_plan::{
ExecutionPlan, SendableRecordBatchStream, DisplayAs, DisplayFormatType,
- Statistics, PlanProperties
+ Statistics, PlanProperties, PhysicalExpr
};
use datafusion::execution::context::TaskContext;
use datafusion::arrow::array::{UInt64Builder, UInt8Builder};
@@ -153,6 +154,13 @@ impl ExecutionPlan for CustomExec {
None,
)?))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
```
@@ -277,12 +285,20 @@ The `scan` method of the `TableProvider` returns a `Result<Arc<dyn ExecutionPlan
# None,
# )?))
# }
+#
+# fn apply_expressions(
+# &self,
+# _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+# ) -> Result<TreeNodeRecursion> {
+# Ok(TreeNodeRecursion::Continue)
+# }
# }
use async_trait::async_trait;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::logical_expr::expr::Expr;
use datafusion::datasource::{TableProvider, TableType};
-use datafusion::physical_plan::project_schema;
+use datafusion::physical_plan::{project_schema, PhysicalExpr};
use datafusion::catalog::Session;
impl CustomExec {
@@ -469,12 +485,20 @@ This will allow you to use the custom table provider in DataFusion. For example,
# None,
# )?))
# }
+#
+# fn apply_expressions(
+# &self,
+# _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+# ) -> Result<TreeNodeRecursion> {
+# Ok(TreeNodeRecursion::Continue)
+# }
# }
# use async_trait::async_trait;
+# use datafusion::common::tree_node::TreeNodeRecursion;
# use datafusion::logical_expr::expr::Expr;
# use datafusion::datasource::{TableProvider, TableType};
-# use datafusion::physical_plan::project_schema;
+# use datafusion::physical_plan::{project_schema, PhysicalExpr};
# use datafusion::catalog::Session;
#
# impl CustomExec {
diff --git a/docs/source/library-user-guide/upgrading/54.0.0.md b/docs/source/library-user-guide/upgrading/54.0.0.md
new file mode 100644
index 0000000000..85cb8e9dd0
--- /dev/null
+++ b/docs/source/library-user-guide/upgrading/54.0.0.md
@@ -0,0 +1,93 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Upgrade Guides
+
+## DataFusion 54.0.0
+
+**Note:** DataFusion `54.0.0` has not been released yet. The information provided
+in this section pertains to features and changes that have already been merged
+to the main branch and are awaiting release in this version.
+
+### `ExecutionPlan::apply_expressions` is now a required method
+
+`apply_expressions` has been added as a **required** method on the `ExecutionPlan` trait (no default implementation). The same applies to the `FileSource` and `DataSource` traits. Any custom implementation of these traits must now implement `apply_expressions`.
+
+**Who is affected:**
+
+- Users who implement custom `ExecutionPlan` nodes
+- Users who implement custom `FileSource` or `DataSource` sources
+
+**Migration guide:**
+
+Add `apply_expressions` to your implementation. Call `f` on each top-level `PhysicalExpr` your node owns, using `visit_sibling` to correctly propagate `TreeNodeRecursion`:
+
+**Node with no expressions:**
+
+```rust,ignore
+fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+}
+```
+
+**Node with a single expression:**
+
+```rust,ignore
+fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+) -> Result<TreeNodeRecursion> {
+ f(self.predicate.as_ref())
+}
+```
+
+**Node with multiple expressions:**
+
+```rust,ignore
+fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for expr in &self.expressions {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ Ok(tnr)
+}
+```
+
+**Node whose only expressions are in `output_ordering()` (e.g. a synthetic test node with no owned expression fields):**
+
+```rust,ignore
+fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ if let Some(ordering) = self.cache.output_ordering() {
+ for sort_expr in ordering {
+ tnr = tnr.visit_sibling(|| f(sort_expr.expr.as_ref()))?;
+ }
+ }
+ Ok(tnr)
+}
+```
diff --git a/docs/source/library-user-guide/upgrading/index.rst b/docs/source/library-user-guide/upgrading/index.rst
index 16bb33b759..1ed5eca2a5 100644
--- a/docs/source/library-user-guide/upgrading/index.rst
+++ b/docs/source/library-user-guide/upgrading/index.rst
@@ -21,6 +21,7 @@ Upgrade Guides
.. toctree::
:maxdepth: 1
+ DataFusion 54.0.0 <54.0.0>
DataFusion 53.0.0 <53.0.0>
DataFusion 52.0.0 <52.0.0>
DataFusion 51.0.0 <51.0.0>