Skip to content

Apache DataFusion SQL Query Engine

Reapply "Add ExecutionPlan::apply_expressions() (apache#20337)" (apache#22437) (#24018)

Jayant Shrivastava2bfdd4aea181
diff --git a/datafusion-examples/examples/custom_data_source/custom_datasource.rs b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
index a2d7d76999..a5a38edf0b 100644
--- a/datafusion-examples/examples/custom_data_source/custom_datasource.rs
+++ b/datafusion-examples/examples/custom_data_source/custom_datasource.rs
@@ -27,6 +27,7 @@ use datafusion::arrow::array::{UInt8Builder, UInt64Builder};
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::common::assert_batches_eq;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::datasource::{TableProvider, TableType, provider_as_source};
use datafusion::error::Result;
use datafusion::execution::context::TaskContext;
@@ -314,4 +315,13 @@ impl ExecutionPlan for CustomExec {
None,
)?))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
index ca765774d1..6decb84b55 100644
--- a/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
+++ b/datafusion-examples/examples/execution_monitoring/memory_pool_execution_plan.rs
@@ -29,6 +29,7 @@
use arrow::array::record_batch;
use arrow::record_batch::RecordBatch;
use arrow_schema::SchemaRef;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::common::{exec_datafusion_err, internal_err};
use datafusion::datasource::{DefaultTableSource, memory::MemTable};
use datafusion::error::Result;
@@ -291,4 +292,13 @@ impl ExecutionPlan for BufferingExecutionPlan {
}),
)))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rs
index 6077a982c3..d5197fe61b 100644
--- a/datafusion-examples/examples/proto/composed_extension_codec.rs
+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs
@@ -37,6 +37,7 @@ use std::sync::Arc;
use datafusion::common::Result;
use datafusion::common::internal_err;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::execution::TaskContext;
use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
use datafusion::prelude::SessionContext;
@@ -124,6 +125,15 @@ impl ExecutionPlan for ParentExec {
) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
unreachable!()
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// A PhysicalExtensionCodec that can serialize and deserialize ParentExec
@@ -202,6 +212,15 @@ impl ExecutionPlan for ChildExec {
) -> Result<datafusion::physical_plan::SendableRecordBatchStream> {
unreachable!()
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// A PhysicalExtensionCodec that can serialize and deserialize ChildExec
diff --git a/datafusion-examples/examples/relation_planner/table_sample.rs b/datafusion-examples/examples/relation_planner/table_sample.rs
index c019e136cc..388175ee3a 100644
--- a/datafusion-examples/examples/relation_planner/table_sample.rs
+++ b/datafusion-examples/examples/relation_planner/table_sample.rs
@@ -118,7 +118,7 @@ use datafusion::{
};
use datafusion_common::{
DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
- plan_datafusion_err, plan_err,
+ plan_datafusion_err, plan_err, tree_node::TreeNodeRecursion,
};
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
use datafusion_expr::{
@@ -750,6 +750,15 @@ impl ExecutionPlan for SampleExec {
Ok(Arc::new(stats))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// Bernoulli sampler: includes each row with probability `(upper - lower)`.
diff --git a/datafusion/catalog/src/memory/table.rs b/datafusion/catalog/src/memory/table.rs
index 5d07133799..ef5669a3a1 100644
--- a/datafusion/catalog/src/memory/table.rs
+++ b/datafusion/catalog/src/memory/table.rs
@@ -31,6 +31,7 @@ use arrow::compute::{and, filter_record_batch};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::error::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Constraints, DFSchema, SchemaExt, not_impl_err, plan_err};
use datafusion_datasource::memory::{MemSink, MemorySourceConfig};
use datafusion_datasource::sink::DataSinkExec;
@@ -39,13 +40,13 @@ use datafusion_expr::dml::InsertOp;
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
use datafusion_expr::{Expr, SortExpr, TableType};
use datafusion_physical_expr::{
- LexOrdering, PhysicalExpr, create_physical_expr, create_physical_sort_exprs,
+ LexOrdering, create_physical_expr, create_physical_sort_exprs,
};
use datafusion_physical_plan::repartition::RepartitionExec;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_physical_plan::{
- DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
- collect_partitioned,
+ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr,
+ PlanProperties, collect_partitioned,
};
use datafusion_session::Session;
@@ -597,4 +598,11 @@ impl ExecutionPlan for DmlResultExec {
stream,
)))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs
index 58f00551ac..1202f08a56 100644
--- a/datafusion/core/src/physical_planner.rs
+++ b/datafusion/core/src/physical_planner.rs
@@ -4950,6 +4950,13 @@ mod tests {
) -> Result<SendableRecordBatchStream> {
unimplemented!("NoOpExecutionPlan::execute");
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
struct ExpressionExtensionPlanner;
@@ -5117,6 +5124,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for OkExtensionNode {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
@@ -5163,6 +5176,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for InvariantFailsExtensionNode {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
@@ -5287,6 +5306,12 @@ digraph {
) -> Result<SendableRecordBatchStream> {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl DisplayAs for ExecutableInvariantFails {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
diff --git a/datafusion/core/tests/custom_sources_cases/mod.rs b/datafusion/core/tests/custom_sources_cases/mod.rs
index c70722cb2f..43f4be05b2 100644
--- a/datafusion/core/tests/custom_sources_cases/mod.rs
+++ b/datafusion/core/tests/custom_sources_cases/mod.rs
@@ -38,6 +38,7 @@ use datafusion_catalog::Session;
use datafusion_common::cast::as_primitive_array;
use datafusion_common::project_schema;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::PlanProperties;
use datafusion_physical_plan::StatisticsArgs;
@@ -209,6 +210,15 @@ impl ExecutionPlan for CustomExecutionPlan {
.collect(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[async_trait]
diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
index 18695accd0..7437bbc543 100644
--- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
+++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs
@@ -35,6 +35,7 @@ use datafusion::prelude::*;
use datafusion::scalar::ScalarValue;
use datafusion_catalog::Session;
use datafusion_common::cast::as_primitive_array;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, internal_err, not_impl_err};
use datafusion_expr::expr::{BinaryExpr, Cast};
use datafusion_functions_aggregate::expr_fn::count;
@@ -148,6 +149,15 @@ impl ExecutionPlan for CustomPlan {
})),
)))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[derive(Clone, Debug)]
diff --git a/datafusion/core/tests/custom_sources_cases/statistics.rs b/datafusion/core/tests/custom_sources_cases/statistics.rs
index d289b5c348..c14ca685b2 100644
--- a/datafusion/core/tests/custom_sources_cases/statistics.rs
+++ b/datafusion/core/tests/custom_sources_cases/statistics.rs
@@ -33,6 +33,7 @@ use datafusion::{
scalar::ScalarValue,
};
use datafusion_catalog::Session;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{project_schema, stats::Precision};
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -185,6 +186,15 @@ impl ExecutionPlan for StatisticsValidation {
Ok(Arc::new(self.stats.clone()))
}
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
fn init_ctx(stats: Statistics, schema: Schema) -> Result<SessionContext> {
diff --git a/datafusion/core/tests/fuzz_cases/once_exec.rs b/datafusion/core/tests/fuzz_cases/once_exec.rs
index 9b57141061..638cbe4c9d 100644
--- a/datafusion/core/tests/fuzz_cases/once_exec.rs
+++ b/datafusion/core/tests/fuzz_cases/once_exec.rs
@@ -17,6 +17,7 @@
use arrow_schema::SchemaRef;
use datafusion_common::internal_datafusion_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
@@ -105,4 +106,13 @@ impl ExecutionPlan for OnceExec {
stream.ok_or_else(|| internal_datafusion_err!("Stream already consumed"))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> datafusion_common::Result<TreeNodeRecursion>,
+ ) -> datafusion_common::Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
index ac7e7a75a2..e0b152d1f0 100644
--- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
+++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs
@@ -40,7 +40,9 @@ use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::ScalarValue;
use datafusion_common::config::CsvOptions;
use datafusion_common::error::Result;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_datasource::file_groups::FileGroup;
use datafusion_datasource::file_scan_config::FileScanConfigBuilder;
use datafusion_expr::{JoinType, Operator};
@@ -203,6 +205,13 @@ impl ExecutionPlan for SortRequiredExec {
)))
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
@@ -290,6 +299,13 @@ impl ExecutionPlan for SinglePartitionMaintainsOrderExec {
Ok(Arc::new(Self::new(child)))
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
diff --git a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
index c04ccd2f3c..83fabcdff8 100644
--- a/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
+++ b/datafusion/core/tests/physical_optimizer/ensure_requirements.rs
@@ -24,7 +24,7 @@
use insta::assert_snapshot;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{TransformedResult, TreeNode};
+use datafusion_common::tree_node::{TransformedResult, TreeNode, TreeNodeRecursion};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
use datafusion_physical_optimizer::ensure_requirements::enforce_sorting::{
@@ -130,6 +130,12 @@ impl ExecutionPlan for MockMultiPartitionExec {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
_children: Vec<Arc<dyn ExecutionPlan>>,
@@ -993,6 +999,12 @@ impl ExecutionPlan for MockReqExec {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn input_distribution_requirements(
&self,
) -> datafusion_physical_plan::InputDistributionRequirements {
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
index f2b1f66c28..a26761107a 100644
--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs
@@ -50,7 +50,9 @@ use datafusion_functions_aggregate::{
min_max::{max_udaf, min_udaf},
};
use datafusion_physical_expr::{
- LexOrdering, PhysicalSortExpr, expressions::col, utils::conjunction,
+ LexOrdering, PhysicalSortExpr,
+ expressions::{DynamicFilterPhysicalExpr, col},
+ utils::conjunction,
};
use datafusion_physical_expr::{
Partitioning, ScalarFunctionExpr, aggregate::AggregateExprBuilder,
@@ -2919,23 +2921,30 @@ async fn test_hashjoin_hash_table_pushdown_collect_left() {
);
}
-// Not portable to sqllogictest: asserts on the dynamic filter's Arc ownership.
-// The observable behavior (probe-side scan
-// receiving the dynamic filter when the data source supports it) is
-// already covered by the simpler CollectLeft port in push_down_filter_parquet.slt;
-// the with_support(false) branch has no SQL analog (parquet always supports
-// pushdown).
-#[expect(
- deprecated,
- reason = "the borrowed getter avoids adding a producer Arc that is_used would count"
-)]
-#[tokio::test]
-async fn test_hashjoin_dynamic_filter_pushdown_is_used() {
- use datafusion_common::JoinType;
- use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode};
+// Not portable to sqllogictest: verifies whether the optimized probe-side plan
+// retains the HashJoinExec's dynamic filter expression. The with_support(false)
+// branch has no SQL analog because parquet supports filter pushdown.
+#[test]
+fn test_hashjoin_dynamic_filter_pushdown_is_used() {
+ fn contains_expression_id(plan: &Arc<dyn ExecutionPlan>, expression_id: u64) -> bool {
+ let mut found = false;
+ plan.apply(|node| {
+ node.apply_expressions(&mut |root| {
+ root.apply(|expr| {
+ if expr.expression_id() == Some(expression_id) {
+ found = true;
+ Ok(TreeNodeRecursion::Stop)
+ } else {
+ Ok(TreeNodeRecursion::Continue)
+ }
+ })
+ })
+ })
+ .unwrap();
+ found
+ }
- // Test both cases: probe side with and without filter pushdown support
- for (probe_supports_pushdown, expected_is_used) in [(false, false), (true, true)] {
+ for (probe_supports_pushdown, expected_consumer) in [(false, false), (true, true)] {
let build_side_schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Utf8, false),
Field::new("b", DataType::Utf8, false),
@@ -2988,31 +2997,26 @@ async fn test_hashjoin_dynamic_filter_pushdown_is_used() {
.unwrap(),
) as Arc<dyn ExecutionPlan>;
- // Apply filter pushdown optimization
let mut config = ConfigOptions::default();
config.execution.parquet.pushdown_filters = true;
config.optimizer.enable_dynamic_filter_pushdown = true;
let plan = FilterPushdown::new_post_optimization()
.optimize(plan, &config)
.unwrap();
-
- // Get the HashJoinExec to check the dynamic filter
let hash_join = plan
.downcast_ref::<HashJoinExec>()
.expect("Plan should be HashJoinExec");
+ let dynamic_filters = hash_join.dynamic_expressions_produced();
+ let expression_id = dynamic_filters
+ .first()
+ .expect("Dynamic filter should be created")
+ .expression_id()
+ .expect("Dynamic filters always have an expression ID");
- // Verify that a dynamic filter was created
- let dynamic_filter = hash_join
- .dynamic_filter_expr()
- .expect("Dynamic filter should be created");
-
- // Verify that is_used() returns the expected value based on probe side support.
- // When probe_supports_pushdown=false: no consumer holds a reference (is_used=false)
- // When probe_supports_pushdown=true: probe side holds a reference (is_used=true)
assert_eq!(
- dynamic_filter.is_used(),
- expected_is_used,
- "is_used() should return {expected_is_used} when probe side support is {probe_supports_pushdown}"
+ contains_expression_id(hash_join.right(), expression_id),
+ expected_consumer,
+ "probe consumer should be {expected_consumer} when pushdown support is {probe_supports_pushdown}"
);
}
}
@@ -3110,6 +3114,106 @@ async fn test_filter_with_projection_pushdown() {
assert_batches_eq!(expected, &result);
}
+/// Test that ExecutionPlan::apply_expressions() can discover dynamic filters across the plan tree.
+///
+/// Not portable to sqllogictest: asserts by walking the plan tree with
+/// `apply_expressions` + `downcast_ref::<DynamicFilterPhysicalExpr>` and
+/// counting nodes. Neither API is observable from SQL.
+#[tokio::test]
+async fn test_discover_dynamic_filters_via_expressions_api() {
+ fn count_dynamic_filters(plan: &Arc<dyn ExecutionPlan>) -> usize {
+ let mut count = 0;
+
+ // Check expressions from this node using apply_expressions
+ let _ = plan.apply_expressions(&mut |expr| {
+ if let Some(_df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {
+ count += 1;
+ }
+ Ok(TreeNodeRecursion::Continue)
+ });
+
+ // Recursively visit children
+ for child in plan.children() {
+ count += count_dynamic_filters(child);
+ }
+
+ count
+ }
+
+ // Create build side (left)
+ let build_batches =
+ vec![record_batch!(("a", Utf8, ["foo", "bar"]), ("b", Int32, [1, 2])).unwrap()];
+ let build_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("b", DataType::Int32, false),
+ ]));
+ let build_scan = TestScanBuilder::new(build_schema.clone())
+ .with_support(true)
+ .with_batches(build_batches)
+ .build();
+
+ // Create probe side (right)
+ let probe_batches = vec![
+ record_batch!(
+ ("a", Utf8, ["foo", "bar", "baz", "qux"]),
+ ("c", Float64, [1.0, 2.0, 3.0, 4.0])
+ )
+ .unwrap(),
+ ];
+ let probe_schema = Arc::new(Schema::new(vec![
+ Field::new("a", DataType::Utf8, false),
+ Field::new("c", DataType::Float64, false),
+ ]));
+ let probe_scan = TestScanBuilder::new(probe_schema.clone())
+ .with_support(true)
+ .with_batches(probe_batches)
+ .build();
+
+ // Create HashJoinExec
+ let plan = Arc::new(
+ HashJoinExec::try_new(
+ build_scan,
+ probe_scan,
+ vec![(
+ col("a", &build_schema).unwrap(),
+ col("a", &probe_schema).unwrap(),
+ )],
+ None,
+ &JoinType::Inner,
+ None,
+ PartitionMode::CollectLeft,
+ datafusion_common::NullEquality::NullEqualsNothing,
+ false,
+ )
+ .unwrap(),
+ ) as Arc<dyn ExecutionPlan>;
+
+ // Before optimization: no dynamic filters
+ let count_before = count_dynamic_filters(&plan);
+ assert_eq!(
+ count_before, 0,
+ "Before optimization, should have no dynamic filters"
+ );
+
+ // Apply filter pushdown optimization (this creates dynamic filters)
+ let mut config = ConfigOptions::default();
+ config.optimizer.enable_dynamic_filter_pushdown = true;
+ config.execution.parquet.pushdown_filters = true;
+ let optimized_plan = FilterPushdown::new_post_optimization()
+ .optimize(plan, &config)
+ .unwrap();
+
+ // After optimization: should discover dynamic filters
+ // We expect 2 dynamic filters:
+ // 1. In the HashJoinExec (producer)
+ // 2. In the DataSourceExec (consumer, pushed down to the probe side)
+ let count_after = count_dynamic_filters(&optimized_plan);
+ assert_eq!(
+ count_after, 2,
+ "After optimization, should discover exactly 2 dynamic filters (1 in HashJoinExec, 1 in DataSourceExec), found {count_after}"
+ );
+}
+
#[test]
fn test_discover_dynamic_expression_producers() {
fn producer_count(plan: &Arc<dyn ExecutionPlan>) -> usize {
diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs
index 3827e6e98b..c7e3799842 100644
--- a/datafusion/core/tests/physical_optimizer/join_selection.rs
+++ b/datafusion/core/tests/physical_optimizer/join_selection.rs
@@ -25,6 +25,7 @@ use std::{
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{ColumnStatistics, JoinType, ScalarValue, stats::Precision};
use datafusion_common::{JoinSide, NullEquality};
use datafusion_common::{Result, Statistics};
@@ -1125,6 +1126,13 @@ impl ExecutionPlan for UnboundedExec {
batch: self.batch.clone(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[derive(Eq, PartialEq, Debug)]
@@ -1230,6 +1238,13 @@ impl ExecutionPlan for StatisticsExec {
self.stats.clone()
}))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[test]
diff --git a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
index 2ffd1899b3..4f8b9ad42b 100644
--- a/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/pushdown_utils.rs
@@ -18,6 +18,7 @@
use arrow::datatypes::SchemaRef;
use arrow::{array::RecordBatch, compute::concat_batches};
use datafusion::{datasource::object_store::ObjectStoreUrl, physical_plan::PhysicalExpr};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, config::ConfigOptions, internal_err};
use datafusion_datasource::{
PartitionedFile, file::FileSource, file_scan_config::FileScanConfig,
@@ -234,6 +235,21 @@ impl FileSource for TestSource {
fn table_schema(&self) -> &datafusion_datasource::TableSchema {
&self.table_schema
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.predicate.iter().chain(
+ self.projection
+ .iter()
+ .flatten()
+ .map(|proj_expr| &proj_expr.expr),
+ ),
+ f,
+ )
+ }
}
#[derive(Debug, Clone)]
@@ -549,4 +565,11 @@ impl ExecutionPlan for TestNode {
Ok(res)
}
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots([&self.predicate], f)
+ }
}
diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs
index 3235ea25fd..833077fe49 100644
--- a/datafusion/core/tests/physical_optimizer/test_utils.rs
+++ b/datafusion/core/tests/physical_optimizer/test_utils.rs
@@ -30,7 +30,9 @@ use datafusion::datasource::physical_plan::ParquetSource;
use datafusion::datasource::source::{DataSource, DataSourceExec};
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::utils::expr::COUNT_STAR_EXPANSION;
use datafusion_common::{
ColumnStatistics, JoinType, NullEquality, Result, Statistics, internal_err,
@@ -543,6 +545,13 @@ impl ExecutionPlan for RequirementsTestExec {
) -> Result<SendableRecordBatchStream> {
unimplemented!("Test exec does not support execution")
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// A [`PlanContext`] object is susceptible to being left in an inconsistent state after
@@ -1091,6 +1100,13 @@ impl ExecutionPlan for TestScan {
})
}
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// Helper function to create a TestScan with ordering
@@ -1107,6 +1123,13 @@ struct InexactMemorySource {
}
impl DataSource for InexactMemorySource {
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ self.inner.apply_expressions(f)
+ }
+
fn open(
&self,
partition: usize,
diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rs
index f3d3f70bdf..c61fe018aa 100644
--- a/datafusion/core/tests/user_defined/insert_operation.rs
+++ b/datafusion/core/tests/user_defined/insert_operation.rs
@@ -25,6 +25,7 @@ use datafusion::{
};
use datafusion_catalog::{Session, TableProvider};
use datafusion_common::config::Dialect;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_expr::{Expr, TableType, dml::InsertOp};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use datafusion_physical_plan::execution_plan::SchedulingType;
@@ -176,6 +177,15 @@ impl ExecutionPlan for TestInsertExec {
) -> Result<datafusion_execution::SendableRecordBatchStream> {
unimplemented!("TestInsertExec is a stub for testing.")
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
fn make_count_schema() -> SchemaRef {
diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rs
index 99363ba500..2b042b613d 100644
--- a/datafusion/core/tests/user_defined/user_defined_plan.rs
+++ b/datafusion/core/tests/user_defined/user_defined_plan.rs
@@ -91,7 +91,9 @@ use datafusion::{
prelude::{SessionConfig, SessionContext},
};
use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::{ScalarValue, assert_eq_or_internal_err, assert_or_internal_err};
use datafusion_expr::{FetchType, InvariantLevel, Projection, SortExpr};
use datafusion_optimizer::AnalyzerRule;
@@ -749,6 +751,15 @@ impl ExecutionPlan for TopKExec {
state: BTreeMap::new(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion::physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
// A very specialized TopK implementation
diff --git a/datafusion/datasource-arrow/src/source.rs b/datafusion/datasource-arrow/src/source.rs
index dbdc1b1cf0..bf92faa9a1 100644
--- a/datafusion/datasource-arrow/src/source.rs
+++ b/datafusion/datasource-arrow/src/source.rs
@@ -40,6 +40,7 @@ use arrow::buffer::Buffer;
use arrow::ipc::reader::{FileDecoder, FileReader, StreamReader};
use datafusion_common::error::Result;
use datafusion_common::exec_datafusion_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::PartitionedFile;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -393,6 +394,21 @@ impl FileSource for ArrowSource {
Some(&self.projection.source)
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.projection
+ .source
+ .iter()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
/// Emit an `ArrowScan` node wrapping the shared base config.
///
/// Decoding defaults to the IPC file format because protobuf does not
diff --git a/datafusion/datasource-avro/src/source.rs b/datafusion/datasource-avro/src/source.rs
index cef85c58df..3956a7318d 100644
--- a/datafusion/datasource-avro/src/source.rs
+++ b/datafusion/datasource-avro/src/source.rs
@@ -22,6 +22,7 @@ use std::sync::Arc;
use arrow::datatypes::{Schema, SchemaRef};
use arrow_avro::reader::{Reader, ReaderBuilder};
use datafusion_common::error::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -169,6 +170,21 @@ impl FileSource for AvroSource {
false
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.projection
+ .source
+ .iter()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
/// Emit an `AvroScan` node wrapping the shared base config.
#[cfg(feature = "proto")]
fn try_to_proto(
diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs
index f498a3c5b7..d5fc6288ea 100644
--- a/datafusion/datasource-csv/src/source.rs
+++ b/datafusion/datasource-csv/src/source.rs
@@ -33,6 +33,7 @@ use datafusion_datasource::{
use arrow::csv;
use datafusion_common::config::CsvOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, exec_datafusion_err};
use datafusion_common_runtime::JoinSet;
use datafusion_datasource::file::FileSource;
@@ -309,6 +310,21 @@ impl FileSource for CsvSource {
}
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.projection
+ .source
+ .iter()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
/// Emit a `CsvScan` node wrapping the shared base config and CSV options.
#[cfg(feature = "proto")]
fn try_to_proto(
diff --git a/datafusion/datasource-json/src/source.rs b/datafusion/datasource-json/src/source.rs
index b7c2e5a45c..c6d420bafb 100644
--- a/datafusion/datasource-json/src/source.rs
+++ b/datafusion/datasource-json/src/source.rs
@@ -27,6 +27,7 @@ use crate::utils::{ChannelReader, JsonArrayToNdjsonReader};
use datafusion_common::error::{DataFusionError, Result};
use datafusion_common::exec_datafusion_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common_runtime::{JoinSet, SpawnedTask};
use datafusion_datasource::boundary_stream::AlignedBoundaryStream;
use datafusion_datasource::decoder::{DecoderDeserializer, deserialize_stream};
@@ -232,6 +233,21 @@ impl FileSource for JsonSource {
"json"
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.projection
+ .source
+ .iter()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
/// Emit a `JsonScan` node wrapping the shared base config.
#[cfg(feature = "proto")]
fn try_to_proto(
diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs
index b11436812c..097b4563af 100644
--- a/datafusion/datasource-parquet/src/source.rs
+++ b/datafusion/datasource-parquet/src/source.rs
@@ -40,6 +40,7 @@ use arrow::array::timezone::Tz;
use arrow::datatypes::TimeUnit;
use datafusion_common::DataFusionError;
use datafusion_common::config::TableParquetOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_datasource::TableSchema;
use datafusion_datasource::file::FileSource;
use datafusion_datasource::file_scan_config::FileScanConfig;
@@ -1058,6 +1059,20 @@ impl FileSource for ParquetSource {
})
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn PhysicalExpr>,
+ ) -> datafusion_common::Result<TreeNodeRecursion>,
+ ) -> datafusion_common::Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.predicate
+ .iter()
+ .chain(self.projection.iter().map(|proj_expr| &proj_expr.expr)),
+ f,
+ )
+ }
+
/// Emit a `ParquetScan` node wrapping the shared base config plus the
/// Parquet-specific pushdown predicate and `TableParquetOptions`.
#[cfg(feature = "proto")]
diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs
index 691bb314b7..f1a94f2e12 100644
--- a/datafusion/datasource/src/file.rs
+++ b/datafusion/datasource/src/file.rs
@@ -29,6 +29,7 @@ use crate::morsel::{FileOpenerMorselizer, Morselizer};
#[expect(deprecated)]
use crate::schema_adapter::SchemaAdapterFactory;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, not_impl_err};
use datafusion_physical_expr::projection::ProjectionExprs;
use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalExpr};
@@ -352,6 +353,23 @@ pub trait FileSource: Any + Send + Sync {
None
}
+ /// Apply a function to all physical expressions used by this file source.
+ ///
+ /// This includes:
+ /// - Filter predicates (which may contain dynamic filters)
+ /// - Projection expressions
+ ///
+ /// The function `f` should be called once per expression unless the function returns
+ /// [`TreeNodeRecursion::Stop`] to stop iteration.
+ ///
+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.
+ ///
+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
+
/// Serialize this file source into a full [`PhysicalPlanNode`] (a
/// `DataSourceExec` wrapping the `FileScanConfig`), if it knows how.
///
diff --git a/datafusion/datasource/src/file_scan_config/mod.rs b/datafusion/datasource/src/file_scan_config/mod.rs
index 766da2f8f7..91dcd5b76f 100644
--- a/datafusion/datasource/src/file_scan_config/mod.rs
+++ b/datafusion/datasource/src/file_scan_config/mod.rs
@@ -37,6 +37,7 @@ use crate::{
use arrow::datatypes::Fields;
use arrow::datatypes::{DataType, Schema, SchemaRef};
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraints, Result, ScalarValue, Statistics, internal_datafusion_err, internal_err,
};
@@ -89,7 +90,9 @@ use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};
/// # use arrow::datatypes::{Field, Fields, DataType, Schema, SchemaRef};
/// # use object_store::ObjectStore;
/// # use datafusion_common::Result;
+/// # use datafusion_common::tree_node::TreeNodeRecursion;
/// # use datafusion_datasource::file::FileSource;
+/// # use datafusion_physical_plan::PhysicalExpr;
/// # use datafusion_datasource::file_groups::FileGroup;
/// # use datafusion_datasource::PartitionedFile;
/// # use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
@@ -119,6 +122,7 @@ use std::{fmt::Debug, fmt::Formatter, fmt::Result as FmtResult, sync::Arc};
/// # fn file_type(&self) -> &str { "parquet" }
/// # // Note that this implementation drops the projection on the floor, it is not complete!
/// # fn try_pushdown_projection(&self, projection: &ProjectionExprs) -> Result<Option<Arc<dyn FileSource>>> { Ok(Some(Arc::new(self.clone()) as Arc<dyn FileSource>)) }
+/// # fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }
/// # }
/// # impl ParquetSource {
/// # fn new(table_schema: impl Into<TableSchema>) -> Self { Self {table_schema: table_schema.into()} }
@@ -1162,6 +1166,14 @@ impl DataSource for FileScanConfig {
Some(Arc::new(new_config))
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Delegate to the file source
+ self.file_source.apply_expressions(f)
+ }
+
/// Create any shared state that should be passed between sibling streams
/// during one execution.
///
@@ -1585,9 +1597,12 @@ mod tests {
use arrow::datatypes::Field;
use datafusion_common::ColumnStatistics;
use datafusion_common::stats::Precision;
+ use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_batches_eq, internal_err};
use datafusion_execution::TaskContext;
use datafusion_expr::SortExpr;
+ use datafusion_physical_expr::PhysicalExpr;
+
#[cfg(feature = "proto")]
use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF};
use datafusion_physical_expr::create_physical_sort_expr;
@@ -1656,6 +1671,13 @@ mod tests {
inner: Arc::new(self.clone()) as Arc<dyn FileSource>,
})
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[cfg(feature = "proto")]
@@ -1702,6 +1724,13 @@ mod tests {
"proto-hook-test"
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn try_to_proto(
&self,
_base: &FileScanConfig,
@@ -3023,6 +3052,13 @@ mod tests {
inner: Arc::new(self.clone()) as Arc<dyn FileSource>,
})
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
#[test]
diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs
index 15ea2600f1..2370ed87a2 100644
--- a/datafusion/datasource/src/memory.rs
+++ b/datafusion/datasource/src/memory.rs
@@ -28,6 +28,7 @@ use crate::source::{DataSource, DataSourceExec};
use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::datatypes::{Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, ScalarValue, assert_or_internal_err, plan_err, project_schema,
};
@@ -257,6 +258,13 @@ impl DataSource for MemorySourceConfig {
.transpose()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Serialize this `MemorySourceConfig` as a `MemoryScanExecNode` wrapped
/// in a [`PhysicalPlanNode`]. Byte-compatible with the former central
/// `MemoryScan` arm in `datafusion-proto`.
diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rs
index 25b559f780..b89cf5d356 100644
--- a/datafusion/datasource/src/sink.rs
+++ b/datafusion/datasource/src/sink.rs
@@ -24,9 +24,10 @@ use std::sync::Arc;
use arrow::array::{ArrayRef, RecordBatch, UInt64Array};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::{Distribution, EquivalenceProperties};
+use datafusion_physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr};
use datafusion_physical_expr_common::sort_expr::{LexRequirement, OrderingRequirements};
use datafusion_physical_plan::metrics::MetricsSet;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
@@ -315,6 +316,13 @@ impl ExecutionPlan for DataSinkExec {
)))
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Execute the plan and return a stream of `RecordBatch`es for
/// the specified partition.
fn execute(
diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs
index 1fd5f865c4..929fed02b3 100644
--- a/datafusion/datasource/src/source.rs
+++ b/datafusion/datasource/src/source.rs
@@ -40,6 +40,7 @@ use itertools::Itertools;
use crate::file::FileSource;
use crate::file_scan_config::FileScanConfig;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Constraints, Result, Statistics};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
@@ -225,6 +226,22 @@ pub trait DataSource: Any + Send + Sync + Debug {
None
}
+ /// Apply a closure to each expression used by this data source.
+ ///
+ /// This includes filter predicates (which may contain dynamic filters) and any
+ /// other expressions used during data scanning.
+ ///
+ /// The function `f` should be called once per expression unless the function returns
+ /// [`TreeNodeRecursion::Stop`] to stop iteration.
+ ///
+ /// See [`ExecutionPlan::apply_expressions`] for more details and implementation examples.
+ ///
+ /// [`ExecutionPlan::apply_expressions`]: datafusion_physical_plan::ExecutionPlan::apply_expressions
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
+
/// Injects arbitrary run-time state into this DataSource, returning a new instance
/// that incorporates that state *if* it is relevant to the concrete DataSource implementation.
///
@@ -383,6 +400,14 @@ impl ExecutionPlan for DataSourceExec {
Vec::new()
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Delegate to the underlying data source
+ self.data_source.apply_expressions(f)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rs
index 5bef6d44e1..20dfae5b3a 100644
--- a/datafusion/datasource/src/test_util.rs
+++ b/datafusion/datasource/src/test_util.rs
@@ -22,7 +22,7 @@ use crate::{
use std::sync::Arc;
use arrow::datatypes::Schema;
-use datafusion_common::Result;
+use datafusion_common::{Result, tree_node::TreeNodeRecursion};
use datafusion_physical_expr::{PhysicalExpr, expressions::Column};
use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet;
use object_store::ObjectStore;
@@ -125,6 +125,13 @@ impl FileSource for MockSource {
) -> Option<&datafusion_physical_plan::projection::ProjectionExprs> {
Some(&self.projection.source)
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
/// Create a column expression
diff --git a/datafusion/ffi/src/execution_plan.rs b/datafusion/ffi/src/execution_plan.rs
index 5191b2fc03..a0dd5e6cb6 100644
--- a/datafusion/ffi/src/execution_plan.rs
+++ b/datafusion/ffi/src/execution_plan.rs
@@ -20,6 +20,7 @@ use std::pin::Pin;
use std::sync::Arc;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, Statistics};
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
use datafusion_physical_expr_common::metrics::MetricsSet;
@@ -51,6 +52,10 @@ pub struct FFI_ExecutionPlan {
/// Return a vector of children plans
pub children: unsafe extern "C" fn(plan: &Self) -> SVec<FFI_ExecutionPlan>,
+ /// Return the physical expression roots owned by this plan node.
+ pub apply_expressions:
+ unsafe extern "C" fn(plan: &Self) -> FFI_Result<SVec<FFI_PhysicalExpr>>,
+
/// Return the dynamic expressions produced by this plan node.
pub dynamic_expressions_produced:
unsafe extern "C" fn(plan: &Self) -> SVec<FFI_PhysicalExpr>,
@@ -96,6 +101,9 @@ pub struct FFI_ExecutionPlan {
/// Release the memory of the private data when it is no longer being used.
pub release: unsafe extern "C" fn(arg: &mut Self),
+ /// Return the major DataFusion version number of this provider.
+ pub version: unsafe extern "C" fn() -> u64,
+
/// Internal data. This is only to be accessed by the provider of the plan.
/// A [`ForeignExecutionPlan`] should never attempt to access this data.
pub private_data: *mut c_void,
@@ -143,6 +151,17 @@ unsafe extern "C" fn children_fn_wrapper(
.collect()
}
+unsafe extern "C" fn apply_expressions_fn_wrapper(
+ plan: &FFI_ExecutionPlan,
+) -> FFI_Result<SVec<FFI_PhysicalExpr>> {
+ let mut expressions = SVec::new();
+ let result = plan.inner().apply_expressions(&mut |expr| {
+ expressions.push(FFI_PhysicalExpr::from(Arc::clone(expr)));
+ Ok(TreeNodeRecursion::Continue)
+ });
+ sresult!(result.map(|_| expressions))
+}
+
unsafe extern "C" fn dynamic_expressions_produced_fn_wrapper(
plan: &FFI_ExecutionPlan,
) -> SVec<FFI_PhysicalExpr> {
@@ -321,6 +340,7 @@ impl FFI_ExecutionPlan {
Self {
properties: properties_fn_wrapper,
children: children_fn_wrapper,
+ apply_expressions: apply_expressions_fn_wrapper,
dynamic_expressions_produced: dynamic_expressions_produced_fn_wrapper,
with_new_children: with_new_children_fn_wrapper,
name: name_fn_wrapper,
@@ -330,6 +350,7 @@ impl FFI_ExecutionPlan {
partition_statistics: partition_statistics_fn_wrapper,
clone: clone_fn_wrapper,
release: release_fn_wrapper,
+ version: crate::version,
private_data: Box::into_raw(private_data) as *mut c_void,
library_marker_id: crate::get_library_marker_id,
}
@@ -458,6 +479,24 @@ impl ExecutionPlan for ForeignExecutionPlan {
}
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let expressions =
+ df_result!(unsafe { (self.plan.apply_expressions)(&self.plan) })?;
+ datafusion_physical_plan::apply_expression_roots(
+ expressions.iter().map(|expression| {
+ let expression: Arc<dyn datafusion_physical_plan::PhysicalExpr> =
+ expression.into();
+ expression
+ }),
+ f,
+ )
+ }
+
fn dynamic_expressions_produced(
&self,
) -> Vec<Arc<dyn datafusion_physical_plan::PhysicalExpr>> {
@@ -509,6 +548,7 @@ pub mod tests {
pub struct EmptyExec {
props: Arc<PlanProperties>,
children: Vec<Arc<dyn ExecutionPlan>>,
+ expressions: Vec<Arc<dyn PhysicalExpr>>,
dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,
metrics: Option<MetricsSet>,
statistics: Option<Statistics>,
@@ -524,6 +564,7 @@ pub mod tests {
Boundedness::Bounded,
)),
children: Vec::default(),
+ expressions: Vec::default(),
dynamic_expressions: Vec::default(),
metrics: None,
statistics: None,
@@ -540,6 +581,14 @@ pub mod tests {
self
}
+ pub fn with_expressions(
+ mut self,
+ expressions: Vec<Arc<dyn PhysicalExpr>>,
+ ) -> Self {
+ self.expressions = expressions;
+ self
+ }
+
pub fn with_dynamic_expressions(
mut self,
dynamic_expressions: Vec<Arc<dyn PhysicalExpr>>,
@@ -579,6 +628,7 @@ pub mod tests {
Ok(Arc::new(EmptyExec {
props: Arc::clone(&self.props),
children,
+ expressions: self.expressions.clone(),
dynamic_expressions: self.dynamic_expressions.clone(),
metrics: self.metrics.clone(),
statistics: self.statistics.clone(),
@@ -610,6 +660,13 @@ pub mod tests {
Statistics::new_unknown(self.props.eq_properties.schema())
})))
}
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(&self.expressions, f)
+ }
}
pub(crate) fn create_dynamic_filter() -> Arc<DynamicFilterPhysicalExpr> {
@@ -645,6 +702,35 @@ pub mod tests {
Ok(())
}
+ #[test]
+ fn test_ffi_execution_plan_apply_expressions() -> Result<()> {
+ let schema = Arc::new(arrow::datatypes::Schema::empty());
+ let dynamic_filter = create_dynamic_filter();
+ let expected_id = dynamic_filter
+ .expression_id()
+ .expect("dynamic filters always have an expression ID");
+ let expression: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;
+ let original_plan =
+ Arc::new(EmptyExec::new(schema).with_expressions(vec![expression]));
+
+ let mut ffi_plan = FFI_ExecutionPlan::new(original_plan, None);
+ ffi_plan.library_marker_id = crate::mock_foreign_marker_id;
+ let foreign_plan: Arc<dyn ExecutionPlan> = (&ffi_plan).try_into()?;
+
+ let mut retained = None;
+ foreign_plan.apply_expressions(&mut |expr| {
+ retained = Some(Arc::clone(expr));
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ drop(foreign_plan);
+
+ assert_eq!(
+ retained.and_then(|expr| expr.expression_id()),
+ Some(expected_id)
+ );
+ Ok(())
+ }
+
#[test]
fn test_ffi_execution_plan_dynamic_expressions_produced() -> Result<()> {
let schema = Arc::new(arrow::datatypes::Schema::empty());
diff --git a/datafusion/ffi/src/physical_expr/mod.rs b/datafusion/ffi/src/physical_expr/mod.rs
index debe0bd7bf..8e6676b6e4 100644
--- a/datafusion/ffi/src/physical_expr/mod.rs
+++ b/datafusion/ffi/src/physical_expr/mod.rs
@@ -726,7 +726,7 @@ impl PhysicalExpr for ForeignPhysicalExpr {
}
fn expression_id(&self) -> Option<u64> {
- unsafe { (self.expr.expression_id)(&self.expr).into() }
+ unsafe { (self.expr.expression_id)(&self.expr) }.into()
}
}
@@ -762,7 +762,9 @@ mod tests {
use datafusion_expr::interval_arithmetic::Interval;
#[expect(deprecated)]
use datafusion_expr::statistics::Distribution;
- use datafusion_physical_expr::expressions::{Column, NegativeExpr, NotExpr};
+ use datafusion_physical_expr::expressions::{
+ Column, DynamicFilterPhysicalExpr, NegativeExpr, NotExpr, lit,
+ };
use datafusion_physical_expr_common::physical_expr::{PhysicalExpr, fmt_sql};
use crate::physical_expr::FFI_PhysicalExpr;
@@ -777,6 +779,21 @@ mod tests {
(original, foreign_expr)
}
+ #[test]
+ fn ffi_physical_expr_expression_id() {
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], lit(true)));
+ let expected_id = dynamic_filter
+ .expression_id()
+ .expect("dynamic filters always have an expression ID");
+ let expression: Arc<dyn PhysicalExpr> =
+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter);
+ let mut ffi_expr = FFI_PhysicalExpr::from(expression);
+ ffi_expr.library_marker_id = crate::mock_foreign_marker_id;
+
+ let foreign_expr: Arc<dyn PhysicalExpr> = (&ffi_expr).into();
+ assert_eq!(foreign_expr.expression_id(), Some(expected_id));
+ }
+
fn test_record_batch() -> RecordBatch {
record_batch!(("a", Int32, [1, 2, 3])).unwrap()
}
diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rs
index 9821c3e501..b9f353e89b 100644
--- a/datafusion/ffi/src/tests/async_provider.rs
+++ b/datafusion/ffi/src/tests/async_provider.rs
@@ -32,6 +32,7 @@ use arrow::array::RecordBatch;
use arrow::datatypes::Schema;
use async_trait::async_trait;
use datafusion_catalog::{MemoryCatalogProvider, TableProvider};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, exec_err};
use datafusion_execution::RecordBatchStream;
use datafusion_expr::Expr;
@@ -227,6 +228,15 @@ impl ExecutionPlan for AsyncTestExecutionPlan {
batch_receiver: self.batch_receiver.resubscribe(),
}))
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_plan::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl datafusion_physical_plan::DisplayAs for AsyncTestExecutionPlan {
diff --git a/datafusion/ffi/src/tests/mod.rs b/datafusion/ffi/src/tests/mod.rs
index ad7e069546..74310d9c28 100644
--- a/datafusion/ffi/src/tests/mod.rs
+++ b/datafusion/ffi/src/tests/mod.rs
@@ -113,6 +113,8 @@ pub struct ForeignLibraryModule {
pub create_empty_exec: extern "C" fn() -> FFI_ExecutionPlan,
+ pub create_exec_with_expressions: extern "C" fn() -> FFI_ExecutionPlan,
+
pub create_exec_with_dynamic_expressions: extern "C" fn() -> FFI_ExecutionPlan,
pub create_exec_with_statistics: extern "C" fn() -> FFI_ExecutionPlan,
@@ -180,6 +182,13 @@ pub(crate) extern "C" fn create_empty_exec() -> FFI_ExecutionPlan {
FFI_ExecutionPlan::new(plan, None)
}
+pub(crate) extern "C" fn create_exec_with_expressions() -> FFI_ExecutionPlan {
+ let schema = Arc::new(Schema::empty());
+ let expression: Arc<dyn PhysicalExpr> = create_dynamic_filter();
+ let plan = Arc::new(EmptyExec::new(schema).with_expressions(vec![expression]));
+ FFI_ExecutionPlan::new(plan, None)
+}
+
pub(crate) extern "C" fn create_exec_with_dynamic_expressions() -> FFI_ExecutionPlan {
let schema = Arc::new(Schema::empty());
let expression: Arc<dyn PhysicalExpr> = create_dynamic_filter();
@@ -286,6 +295,7 @@ pub extern "C" fn datafusion_ffi_get_module() -> ForeignLibraryModule {
create_rank_udwf: create_ffi_rank_func,
create_extension_options: config::create_extension_options,
create_empty_exec,
+ create_exec_with_expressions,
create_exec_with_dynamic_expressions,
create_exec_with_statistics,
create_table_with_statistics,
diff --git a/datafusion/ffi/tests/ffi_execution_plan.rs b/datafusion/ffi/tests/ffi_execution_plan.rs
index a26ba9a1a6..946dca61e1 100644
--- a/datafusion/ffi/tests/ffi_execution_plan.rs
+++ b/datafusion/ffi/tests/ffi_execution_plan.rs
@@ -21,6 +21,7 @@ mod tests {
use arrow::datatypes::Schema;
use arrow_schema::DataType;
use datafusion_common::DataFusionError;
+ use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_ffi::execution_plan::FFI_ExecutionPlan;
use datafusion_ffi::execution_plan::ForeignExecutionPlan;
use datafusion_ffi::execution_plan::{ExecutionPlanPrivateData, tests::EmptyExec};
@@ -64,6 +65,30 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_ffi_execution_plan_expressions_cross_library() -> Result<(), DataFusionError>
+ {
+ let module = get_module()?;
+ let plan = (module.create_exec_with_expressions)();
+ let plan: Arc<dyn ExecutionPlan> = (&plan).try_into()?;
+ assert!(plan.is::<ForeignExecutionPlan>());
+
+ let mut retained = None;
+ plan.apply_expressions(&mut |expr| {
+ retained = Some(Arc::clone(expr));
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ drop(plan);
+
+ assert!(
+ retained
+ .as_ref()
+ .and_then(|expr| expr.expression_id())
+ .is_some()
+ );
+ Ok(())
+ }
+
#[test]
fn test_ffi_execution_plan_dynamic_expressions_cross_library()
-> Result<(), DataFusionError> {
diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
index 0fd0ad93bf..eb3d457de8 100644
--- a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
@@ -434,6 +434,10 @@ impl DynamicFilterPhysicalExpr {
/// We check both Arc counts to handle two cases:
/// - Transformed filters (via `with_new_children`) share the inner Arc (inner count > 1)
/// - Direct clones (via `Arc::clone`) increment the outer count (outer count > 1)
+ #[deprecated(
+ since = "55.0.0",
+ note = "Traverse ExecutionPlan::apply_expressions and compare PhysicalExpr::expression_id instead"
+ )]
pub fn is_used(self: &Arc<Self>) -> bool {
// Strong count > 1 means at least one consumer is holding a reference beyond the producer.
Arc::strong_count(self) > 1 || Arc::strong_count(&self.inner) > 1
@@ -1091,6 +1095,10 @@ mod test {
}
#[test]
+ #[expect(
+ deprecated,
+ reason = "covers the deprecated API during its retention period"
+ )]
fn test_is_used() {
let filter = Arc::new(DynamicFilterPhysicalExpr::new(
vec![],
diff --git a/datafusion/physical-optimizer/src/ensure_coop.rs b/datafusion/physical-optimizer/src/ensure_coop.rs
index e7aacb2321..10da9e4e75 100644
--- a/datafusion/physical-optimizer/src/ensure_coop.rs
+++ b/datafusion/physical-optimizer/src/ensure_coop.rs
@@ -264,10 +264,11 @@ mod tests {
// Test that cooperative context is reset when encountering an eager evaluation boundary.
use arrow::datatypes::Schema;
use datafusion_common::internal_err;
+ use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
use datafusion_physical_plan::{
- DisplayAs, DisplayFormatType, Partitioning, PlanProperties,
+ DisplayAs, DisplayFormatType, Partitioning, PhysicalExpr, PlanProperties,
SendableRecordBatchStream,
execution_plan::{Boundedness, EmissionType},
};
@@ -345,6 +346,13 @@ mod tests {
) -> Result<SendableRecordBatchStream> {
internal_err!("DummyExec does not support execution")
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
// Build a plan similar to the original test:
diff --git a/datafusion/physical-optimizer/src/output_requirements.rs b/datafusion/physical-optimizer/src/output_requirements.rs
index b9d0d06da1..fc8bf490b9 100644
--- a/datafusion/physical-optimizer/src/output_requirements.rs
+++ b/datafusion/physical-optimizer/src/output_requirements.rs
@@ -27,7 +27,9 @@ use std::sync::Arc;
use crate::PhysicalOptimizerRule;
use datafusion_common::config::ConfigOptions;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
use datafusion_common::{Result, Statistics, internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::Distribution;
@@ -324,6 +326,15 @@ impl ExecutionPlan for OutputRequirementExec {
fn fetch(&self) -> Option<usize> {
self.fetch
}
+
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn datafusion_physical_expr_common::physical_expr::PhysicalExpr>,
+ ) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
}
impl PhysicalOptimizerRule for OutputRequirements {
diff --git a/datafusion/physical-plan/benches/compute_statistics.rs b/datafusion/physical-plan/benches/compute_statistics.rs
index 56a518c952..93c95ea4ba 100644
--- a/datafusion/physical-plan/benches/compute_statistics.rs
+++ b/datafusion/physical-plan/benches/compute_statistics.rs
@@ -33,6 +33,7 @@ use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use datafusion_common::ScalarValue;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, Statistics};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
@@ -97,6 +98,13 @@ impl ExecutionPlan for BenchLeaf {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index bae95d368e..fdf6ce323b 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -156,7 +156,9 @@ use crate::aggregates::{
partial_reduce_stream::PartialReduceHashAggregateStream,
single_stream::SingleHashAggregateStream,
};
-use crate::execution_plan::{CardinalityEffect, EmissionType};
+use crate::execution_plan::{
+ CardinalityEffect, EmissionType, plan_contains_expression_id,
+};
use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
FilterPushdownPropagation,
@@ -176,6 +178,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_schema::FieldRef;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
ColumnStatistics, Constraint, Constraints, Result, ScalarValue,
assert_eq_or_internal_err, internal_err, not_impl_err,
@@ -2028,6 +2031,33 @@ impl ExecutionPlan for AggregateExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let group_by = self.group_by.input_exprs();
+ let aggregates = self.aggr_expr.iter().flat_map(|aggr| {
+ let expressions = aggr.all_expressions();
+ expressions
+ .args
+ .into_iter()
+ .chain(expressions.order_by_exprs)
+ });
+ let filters = self.filter_expr.iter().flatten().cloned();
+ let dynamic_filter = self.dynamic_filter.iter().map(|dynamic_filter| {
+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter.filter)
+ as Arc<dyn PhysicalExpr>
+ });
+ crate::apply_expression_roots(
+ group_by
+ .into_iter()
+ .chain(aggregates)
+ .chain(filters)
+ .chain(dynamic_filter),
+ f,
+ )
+ }
+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
self.dynamic_filter
.iter()
@@ -2175,27 +2205,12 @@ impl ExecutionPlan for AggregateExec {
if phase == FilterPushdownPhase::Post
&& let Some(dyn_filter) = &self.dynamic_filter
{
- // let child_accepts_dyn_filter = child_pushdown_result
- // .self_filters
- // .first()
- // .map(|filters| {
- // assert_eq_or_internal_err!(
- // filters.len(),
- // 1,
- // "Aggregate only pushdown one self dynamic filter"
- // );
- // let filter = filters.get(0).unwrap(); // Asserted above
- // Ok(matches!(filter.discriminant, PushedDown::Yes))
- // })
- // .unwrap_or_else(|| internal_err!("The length of self filters equals to the number of child of this ExecutionPlan, so it must be 1"))?;
-
- // HACK: The above snippet should be used, however, now the child reply
- // `PushDown::No` can indicate they're not able to push down row-level
- // filter, but still keep the filter for statistics pruning.
- // So here, we try to use ref count to determine if the dynamic filter
- // has actually be pushed down.
- // Issue: <https://github.com/apache/datafusion/issues/18856>
- let child_accepts_dyn_filter = Arc::strong_count(dyn_filter) > 1;
+ let child_accepts_dyn_filter = dyn_filter
+ .filter
+ .expression_id()
+ .map(|id| plan_contains_expression_id(&self.input, id))
+ .transpose()?
+ .unwrap_or(false);
if !child_accepts_dyn_filter {
// Child can't consume the self dynamic filter, so disable it by setting
@@ -2589,6 +2604,8 @@ impl AggregateExec {
})?;
aggregate.with_dynamic_filter_expr(dynamic_filter)?
} else {
+ let mut aggregate = aggregate;
+ aggregate.dynamic_filter = None;
aggregate
};
@@ -3092,6 +3109,7 @@ mod tests {
use crate::empty::EmptyExec;
use crate::execution_plan::Boundedness;
use crate::expressions::col;
+ use crate::filter::FilterExecBuilder;
use crate::metrics::MetricValue;
use crate::statistics::{StatisticsArgs, StatisticsContext};
use crate::test::TestMemoryExec;
@@ -3127,7 +3145,7 @@ mod tests {
use datafusion_physical_expr::Partitioning;
use datafusion_physical_expr::PhysicalSortExpr;
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
- use datafusion_physical_expr::expressions::Literal;
+ use datafusion_physical_expr::expressions::{Literal, NotExpr};
use crate::projection::ProjectionExec;
use crate::repartition::RepartitionExec;
@@ -3644,6 +3662,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -7812,6 +7837,38 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_plan_contains_expression_id_recurses_plans_and_expressions() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let empty: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![col("a", &schema)?],
+ lit(true),
+ ));
+ let expression_id = dynamic_filter
+ .expression_id()
+ .expect("dynamic filters always have an expression ID");
+
+ assert!(!plan_contains_expression_id(&empty, expression_id)?);
+
+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> =
+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter);
+ let predicate: Arc<dyn PhysicalExpr> =
+ Arc::new(NotExpr::new(dynamic_filter_expr));
+ let filter: Arc<dyn ExecutionPlan> =
+ Arc::new(FilterExecBuilder::new(predicate, empty).build()?);
+ let projection: Arc<dyn ExecutionPlan> = Arc::new(ProjectionExec::try_new(
+ [ProjectionExpr::new_from_expression(
+ col("a", &schema)?,
+ &schema,
+ )?],
+ filter,
+ )?);
+
+ assert!(plan_contains_expression_id(&projection, expression_id)?);
+ Ok(())
+ }
+
/// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the aggregate does not support dynamic filtering
#[test]
fn test_with_dynamic_filter_error_unsupported() -> Result<()> {
diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs
index f2c3489c73..9a69518953 100644
--- a/datafusion/physical-plan/src/analyze.rs
+++ b/datafusion/physical-plan/src/analyze.rs
@@ -32,11 +32,13 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::format::ExplainFormat;
use datafusion_common::instant::Instant;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, Result, assert_eq_or_internal_err, internal_err,
};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::PhysicalExpr;
use futures::StreamExt;
@@ -219,6 +221,13 @@ impl ExecutionPlan for AnalyzeExec {
])
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/async_func.rs b/datafusion/physical-plan/src/async_func.rs
index d89c2cc6bf..91531ec35c 100644
--- a/datafusion/physical-plan/src/async_func.rs
+++ b/datafusion/physical-plan/src/async_func.rs
@@ -157,6 +157,19 @@ impl ExecutionPlan for AsyncFuncExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots(
+ self.async_exprs
+ .iter()
+ .cloned()
+ .map(|expr| expr as Arc<dyn PhysicalExpr>),
+ f,
+ )
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/buffer.rs b/datafusion/physical-plan/src/buffer.rs
index 3be331a1ee..a1c3c7ea01 100644
--- a/datafusion/physical-plan/src/buffer.rs
+++ b/datafusion/physical-plan/src/buffer.rs
@@ -32,6 +32,7 @@ use crate::{
};
use arrow::array::RecordBatch;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, Statistics, internal_err, plan_err};
use datafusion_common_runtime::SpawnedTask;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
@@ -159,6 +160,13 @@ impl ExecutionPlan for BufferExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs
index c5b9176777..511e5d793b 100644
--- a/datafusion/physical-plan/src/coalesce_batches.rs
+++ b/datafusion/physical-plan/src/coalesce_batches.rs
@@ -34,6 +34,7 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
@@ -173,6 +174,13 @@ impl ExecutionPlan for CoalesceBatchesExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs
index f9694e0d16..5cd7a707c2 100644
--- a/datafusion/physical-plan/src/coalesce_partitions.rs
+++ b/datafusion/physical-plan/src/coalesce_partitions.rs
@@ -35,6 +35,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning, check_if_same_proper
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::PhysicalExpr;
@@ -143,6 +144,13 @@ impl ExecutionPlan for CoalescePartitionsExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs
index a5b57f546b..dc7d98891e 100644
--- a/datafusion/physical-plan/src/coop.rs
+++ b/datafusion/physical-plan/src/coop.rs
@@ -71,6 +71,7 @@
//! that report [`SchedulingType::NonCooperative`] in their [plan properties](ExecutionPlan::properties).
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_physical_expr::PhysicalExpr;
#[cfg(datafusion_coop = "tokio_fallback")]
use futures::Future;
@@ -267,6 +268,13 @@ impl ExecutionPlan for CooperativeExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs
index 34493a5f51..2370e3e6ce 100644
--- a/datafusion/physical-plan/src/display.rs
+++ b/datafusion/physical-plan/src/display.rs
@@ -1504,8 +1504,11 @@ mod tests {
use std::fmt::Write;
use std::sync::Arc;
- use datafusion_common::{Result, Statistics, internal_datafusion_err};
+ use datafusion_common::{
+ Result, Statistics, internal_datafusion_err, tree_node::TreeNodeRecursion,
+ };
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+ use datafusion_physical_expr::PhysicalExpr;
use crate::statistics::StatisticsArgs;
use crate::{DisplayAs, ExecutionPlan, PlanProperties};
@@ -1549,6 +1552,13 @@ mod tests {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_: usize,
@@ -1706,6 +1716,17 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![&self.inner]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(
+ &Arc<dyn PhysicalExpr>,
+ ) -> Result<
+ datafusion_common::tree_node::TreeNodeRecursion,
+ >,
+ ) -> Result<datafusion_common::tree_node::TreeNodeRecursion>
+ {
+ Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs
index 3bd38bf238..bd91ec742d 100644
--- a/datafusion/physical-plan/src/empty.rs
+++ b/datafusion/physical-plan/src/empty.rs
@@ -29,9 +29,10 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{ColumnStatistics, Result, ScalarValue, assert_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use crate::execution_plan::SchedulingType;
use crate::statistics::StatisticsArgs;
@@ -119,6 +120,13 @@ impl ExecutionPlan for EmptyExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs
index bc4ece5adf..763ec2f7df 100644
--- a/datafusion/physical-plan/src/execution_plan.rs
+++ b/datafusion/physical-plan/src/execution_plan.rs
@@ -28,7 +28,9 @@ pub use crate::stream::EmptyRecordBatchStream;
use arrow_schema::Schema;
pub use datafusion_common::hash_utils;
-use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
+use datafusion_common::tree_node::{
+ Transformed, TransformedResult, TreeNode, TreeNodeRecursion,
+};
pub use datafusion_common::utils::project_schema;
pub use datafusion_common::{ColumnStatistics, Statistics, internal_err};
pub use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};
@@ -39,6 +41,7 @@ pub use datafusion_physical_expr::{
};
use std::any::Any;
+use std::borrow::Borrow;
use std::collections::HashSet;
use std::fmt::Debug;
use std::sync::{Arc, LazyLock};
@@ -264,6 +267,70 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
/// joins).
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>>;
+ /// Apply a closure `f` to each root expression that this node owns and uses
+ /// during execution, either by evaluating it or updating it dynamically.
+ ///
+ /// An expression must not be visited solely because it describes an input or
+ /// output property, such as cached ordering, partitioning, or equivalence
+ /// metadata. However, these may be traversed indirectly. For example,
+ /// `RepartitionExec` visits the partitioning expressions it evaluates and
+ /// `SortExec` visits the sort expressions it evaluates to order rows.
+ ///
+ /// This method is shallow: it must not visit expression children or expressions
+ /// owned by child execution plans.
+ ///
+ /// Similarly to other [`TreeNode`] APIs, the closure can return
+ /// [`TreeNodeRecursion::Stop`] to stop iteration, otherwise iteration
+ /// should continue. Note that [`TreeNodeRecursion::Continue`] and
+ /// [`TreeNodeRecursion::Jump`] are equivalent because this method is not
+ /// recursive.
+ ///
+ ///
+ /// # Example Usage
+ /// ```
+ /// # use std::sync::Arc;
+ /// # use datafusion_physical_plan::ExecutionPlan;
+ /// # use datafusion_common::tree_node::TreeNodeRecursion;
+ /// # fn example(plan: Arc<dyn ExecutionPlan>) -> datafusion_common::Result<()> {
+ /// // Count the number of expressions
+ /// let mut count = 0;
+ /// plan.apply_expressions(&mut |_expr| {
+ /// count += 1;
+ /// Ok(TreeNodeRecursion::Continue)
+ /// })?;
+ /// # Ok(())
+ /// # }
+ /// ```
+ ///
+ /// # Implementation Examples
+ ///
+ /// ## Node with expressions (e.g., FilterExec, ProjectionExec)
+ ///
+ /// Use [`apply_expression_roots`] to implement this method. It abstracts away the
+ /// [`TreeNodeRecursion`] iteration from implementors.
+ /// ```ignore
+ /// fn apply_expressions(
+ /// &self,
+ /// f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ /// ) -> Result<TreeNodeRecursion> {
+ /// apply_expression_roots([&self.predicate], f)
+ /// }
+ /// ```
+ ///
+ /// ## Node with no expressions (e.g., EmptyExec, MemoryExec)
+ /// ```ignore
+ /// fn apply_expressions(
+ /// &self,
+ /// _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ /// ) -> Result<TreeNodeRecursion> {
+ /// Ok(TreeNodeRecursion::Continue)
+ /// }
+ /// ```
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion>;
+
/// Returns a new `ExecutionPlan` where all existing children were replaced
/// by the `children`, in order
fn with_new_children(
@@ -869,6 +936,58 @@ pub trait ExecutionPlan: Any + Debug + DisplayAs + Send + Sync {
}
}
+/// Applies `f` to a shallow sequence of physical expression roots.
+///
+/// [`TreeNodeRecursion::Stop`] stops iteration and is returned immediately.
+/// [`TreeNodeRecursion::Jump`] is normalized to [`TreeNodeRecursion::Continue`]
+/// because this function does not visit expression children.
+pub fn apply_expression_roots<I>(
+ roots: I,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+) -> Result<TreeNodeRecursion>
+where
+ I: IntoIterator,
+ I::Item: Borrow<Arc<dyn PhysicalExpr>>,
+{
+ for root in roots {
+ match f(root.borrow())? {
+ TreeNodeRecursion::Stop => return Ok(TreeNodeRecursion::Stop),
+ TreeNodeRecursion::Continue | TreeNodeRecursion::Jump => {}
+ }
+ }
+ Ok(TreeNodeRecursion::Continue)
+}
+
+/// Returns whether `plan` contains a physical expression with `expression_id`.
+///
+/// This traverses both the execution plan and the children of each expression root
+/// reported by [`ExecutionPlan::apply_expressions`].
+pub(crate) fn plan_contains_expression_id(
+ plan: &Arc<dyn ExecutionPlan>,
+ expression_id: u64,
+) -> Result<bool> {
+ let mut found = false;
+ plan.apply(|node| {
+ node.apply_expressions(&mut |root| {
+ root.apply(|expr| {
+ if expr.expression_id() == Some(expression_id) {
+ found = true;
+ Ok(TreeNodeRecursion::Stop)
+ } else {
+ Ok(TreeNodeRecursion::Continue)
+ }
+ })
+ })?;
+
+ Ok(if found {
+ TreeNodeRecursion::Stop
+ } else {
+ TreeNodeRecursion::Continue
+ })
+ })?;
+ Ok(found)
+}
+
impl dyn ExecutionPlan {
/// Returns `true` if the plan is of type `T`.
///
@@ -1824,6 +1943,13 @@ mod tests {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
self.dynamic_expressions.iter().map(Arc::clone).collect()
}
@@ -1910,6 +2036,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -1960,6 +2093,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ self.0.apply_expressions(f)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -2019,6 +2159,12 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -2082,6 +2228,12 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -2173,6 +2325,12 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -2314,6 +2472,171 @@ mod tests {
Ok(())
}
+ /// A test node that holds a fixed list of expressions, used to test
+ /// `apply_expressions` behavior.
+ #[derive(Debug)]
+ struct MultiExprExec {
+ exprs: Vec<Arc<dyn PhysicalExpr>>,
+ children: Vec<Arc<dyn ExecutionPlan>>,
+ }
+
+ impl DisplayAs for MultiExprExec {
+ fn fmt_as(
+ &self,
+ _t: DisplayFormatType,
+ _f: &mut std::fmt::Formatter,
+ ) -> std::fmt::Result {
+ unimplemented!()
+ }
+ }
+
+ impl ExecutionPlan for MultiExprExec {
+ fn name(&self) -> &'static str {
+ "MultiExprExec"
+ }
+
+ fn properties(&self) -> &Arc<PlanProperties> {
+ unimplemented!()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ self.children.iter().collect()
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unimplemented!()
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ apply_expression_roots(&self.exprs, f)
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unimplemented!()
+ }
+
+ fn partition_statistics(
+ &self,
+ _partition: Option<usize>,
+ ) -> Result<Arc<Statistics>> {
+ unimplemented!()
+ }
+ }
+
+ /// Returns a simple literal `Arc<dyn PhysicalExpr>` for use in tests.
+ fn lit_expr(val: i64) -> Arc<dyn PhysicalExpr> {
+ use datafusion_physical_expr::expressions::Literal;
+ Arc::new(Literal::new(datafusion_common::ScalarValue::Int64(Some(
+ val,
+ ))))
+ }
+
+ /// `apply_expressions` visits all expressions when `f` always returns `Continue`.
+ #[test]
+ fn test_apply_expressions_continue_visits_all() -> Result<()> {
+ let plan = MultiExprExec {
+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],
+ children: vec![],
+ };
+ let mut visited = 0usize;
+ plan.apply_expressions(&mut |_expr| {
+ visited += 1;
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ assert_eq!(visited, 3);
+ Ok(())
+ }
+
+ #[test]
+ fn test_apply_expressions_stop_halts_early() -> Result<()> {
+ let plan = MultiExprExec {
+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],
+ children: vec![],
+ };
+ let mut visited = 0usize;
+ let tnr = plan.apply_expressions(&mut |_expr| {
+ visited += 1;
+ Ok(TreeNodeRecursion::Stop)
+ })?;
+ // Only the first expression is visited; the rest are skipped.
+ assert_eq!(visited, 1);
+ assert_eq!(tnr, TreeNodeRecursion::Stop);
+ Ok(())
+ }
+
+ #[test]
+ fn test_apply_expressions_jump_visits_next_root() -> Result<()> {
+ let plan = MultiExprExec {
+ exprs: vec![lit_expr(1), lit_expr(2), lit_expr(3)],
+ children: vec![],
+ };
+ let mut visited = 0usize;
+ let tnr = plan.apply_expressions(&mut |_expr| {
+ visited += 1;
+ Ok(TreeNodeRecursion::Jump)
+ })?;
+ assert_eq!(visited, 3);
+ assert_eq!(tnr, TreeNodeRecursion::Continue);
+ Ok(())
+ }
+
+ #[test]
+ fn test_apply_expressions_does_not_recurse() -> Result<()> {
+ use datafusion_physical_expr::expressions::NegativeExpr;
+
+ let child: Arc<dyn ExecutionPlan> = Arc::new(MultiExprExec {
+ exprs: vec![lit_expr(2)],
+ children: vec![],
+ });
+ let nested: Arc<dyn PhysicalExpr> = Arc::new(NegativeExpr::new(lit_expr(1)));
+ let plan = MultiExprExec {
+ exprs: vec![nested],
+ children: vec![child],
+ };
+
+ let mut visited = 0;
+ plan.apply_expressions(&mut |expr| {
+ visited += 1;
+ assert!(expr.is::<NegativeExpr>());
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ assert_eq!(visited, 1);
+ Ok(())
+ }
+
+ #[test]
+ fn test_apply_expressions_callback_can_retain_arc() -> Result<()> {
+ let expected = lit_expr(1);
+ let plan = MultiExprExec {
+ exprs: vec![Arc::clone(&expected)],
+ children: vec![],
+ };
+ let mut retained = None;
+ plan.apply_expressions(&mut |expr| {
+ retained = Some(Arc::clone(expr));
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ drop(plan);
+
+ assert!(Arc::ptr_eq(
+ &expected,
+ retained
+ .as_ref()
+ .expect("callback should retain expression")
+ ));
+ Ok(())
+ }
+
#[test]
fn test_execution_plan_name() {
let schema1 = Arc::new(Schema::empty());
diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs
index a270a003eb..72ceae81f3 100644
--- a/datafusion/physical-plan/src/explain.rs
+++ b/datafusion/physical-plan/src/explain.rs
@@ -26,9 +26,10 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning};
use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch};
use datafusion_common::display::StringifiedPlan;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use log::trace;
@@ -116,6 +117,13 @@ impl ExecutionPlan for ExplainExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs
index 50c8246b37..65abfa259d 100644
--- a/datafusion/physical-plan/src/filter.rs
+++ b/datafusion/physical-plan/src/filter.rs
@@ -55,6 +55,7 @@ use arrow::record_batch::RecordBatch;
use datafusion_common::cast::as_boolean_array;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, Result, ScalarValue, internal_err, plan_err, project_schema,
};
@@ -539,6 +540,13 @@ impl ExecutionPlan for FilterExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots([&self.predicate], f)
+ }
+
fn maintains_input_order(&self) -> Vec<bool> {
// Tell optimizer this operator doesn't reorder its input
vec![true]
diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs
index 16155aaafd..524887bf03 100644
--- a/datafusion/physical-plan/src/joins/cross_join.rs
+++ b/datafusion/physical-plan/src/joins/cross_join.rs
@@ -43,11 +43,13 @@ use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::compute::concat_batches;
use arrow::datatypes::{Fields, Schema, SchemaRef};
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinType, Result, ScalarValue, assert_eq_or_internal_err, internal_err,
};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::equivalence::join_equivalence_properties;
use async_trait::async_trait;
@@ -266,6 +268,14 @@ impl ExecutionPlan for CrossJoinExec {
Some(self.metrics.clone_inner())
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // CrossJoin has no join conditions or expressions
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index e26df9a4ae..4e68d871b8 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -25,7 +25,7 @@ use std::vec;
use crate::ExecutionPlanProperties;
use crate::execution_plan::{
EmissionType, boundedness_from_children, has_same_children_properties,
- stub_properties,
+ plan_contains_expression_id, stub_properties,
};
use crate::filter_pushdown::{
ChildFilterDescription, ChildPushdownResult, FilterDescription, FilterPushdownPhase,
@@ -73,6 +73,7 @@ use arrow::record_batch::RecordBatch;
use arrow::util::bit_util;
use arrow_schema::{DataType, Schema};
use datafusion_common::config::ConfigOptions;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::memory::{RecordBatchMemoryCounter, estimate_memory_size};
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, assert_or_internal_err, internal_err,
@@ -1325,6 +1326,25 @@ impl ExecutionPlan for HashJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let join_keys = self
+ .on
+ .iter()
+ .flat_map(|(left, right)| [Arc::clone(left), Arc::clone(right)]);
+ let filter = self
+ .filter
+ .iter()
+ .map(|filter| Arc::clone(filter.expression()));
+ let dynamic_filter = self.dynamic_filter.iter().map(|dynamic_filter| {
+ Arc::<DynamicFilterPhysicalExpr>::clone(&dynamic_filter.filter)
+ as Arc<dyn PhysicalExpr>
+ });
+ crate::apply_expression_roots(join_keys.chain(filter).chain(dynamic_filter), f)
+ }
+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
self.dynamic_filter
.iter()
@@ -1377,18 +1397,20 @@ impl ExecutionPlan for HashJoinExec {
consider using CoalescePartitionsExec or the EnforceDistribution rule"
);
- // Only enable dynamic filter pushdown if:
- // - The session config enables dynamic filter pushdown
- // - A dynamic filter exists
- // - At least one consumer is holding a reference to it, this avoids expensive filter
- // computation when disabled or when no consumer will use it.
- let enable_dynamic_filter_pushdown = self
+ // Only compute a dynamic filter when the probe subtree contains a consumer.
+ // Searching from `self` would always find the producer expression owned by this join.
+ let enable_dynamic_filter_pushdown = if self
.allow_join_dynamic_filter_pushdown(context.session_config().options())
- && self
- .dynamic_filter
+ {
+ self.dynamic_filter
.as_ref()
- .map(|df| df.filter.is_used())
- .unwrap_or(false);
+ .and_then(|df| df.filter.expression_id())
+ .map(|id| plan_contains_expression_id(&self.right, id))
+ .transpose()?
+ .unwrap_or(false)
+ } else {
+ false
+ };
let join_metrics = BuildProbeJoinMetrics::new(partition, &self.metrics);
@@ -2411,6 +2433,7 @@ mod tests {
use crate::coalesce_partitions::CoalescePartitionsExec;
use crate::execution_plan::Boundedness;
+ use crate::filter::FilterExecBuilder;
use crate::joins::hash_join::stream::lookup_join_hashmap;
use crate::test::{TestMemoryExec, assert_join_metrics};
use crate::{
@@ -2478,6 +2501,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -2626,7 +2656,12 @@ mod tests {
Arc::new(Column::new_with_schema("b1", &right_schema).unwrap()) as _,
)];
let right: Arc<dyn ExecutionPlan> = Arc::new(
- MockExec::new(vec![Ok(right_batch), err], right_schema).with_use_task(false),
+ MockExec::new(vec![Ok(right_batch), err], right_schema)
+ .with_use_task(false)
+ // The planted error must only surface if the probe side is
+ // polled, not when a parent node computes statistics during
+ // planning.
+ .with_unknown_statistics(),
);
(left, right, on)
@@ -2706,6 +2741,8 @@ mod tests {
mode: PartitionMode,
) -> Result<(HashJoinExec, Arc<DynamicFilterPhysicalExpr>)> {
let dynamic_filter = HashJoinExec::create_dynamic_filter(&on);
+ let consumer: Arc<dyn PhysicalExpr> = Arc::clone(&dynamic_filter) as _;
+ let right = Arc::new(FilterExecBuilder::new(consumer, right).build()?);
let mut join = HashJoinExec::try_new(
left,
right,
diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs
index 515dcc2931..7069a8b448 100644
--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs
+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs
@@ -61,6 +61,7 @@ use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_schema::DataType;
use datafusion_common::cast::as_boolean_array;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinSide, NullEquality, Result, ScalarValue, Statistics, arrow_err,
assert_eq_or_internal_err, internal_datafusion_err, internal_err, project_schema,
@@ -562,6 +563,17 @@ impl ExecutionPlan for NestedLoopJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to join filter expressions if present
+ crate::apply_expression_roots(
+ self.filter.iter().map(|filter| filter.expression()),
+ f,
+ )
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
index 5ec564295e..b60ec1c784 100644
--- a/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/piecewise_merge_join/exec.rs
@@ -23,6 +23,7 @@ use arrow::{
};
use arrow_schema::{SchemaRef, SortOptions};
use datafusion_common::not_impl_err;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{JoinSide, Result, internal_err};
use datafusion_execution::{
SendableRecordBatchStream,
@@ -482,6 +483,14 @@ impl ExecutionPlan for PiecewiseMergeJoinExec {
vec![&self.buffered, &self.streamed]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ // Apply to the two expressions being compared in the range predicate
+ crate::apply_expression_roots([&self.on.0, &self.on.1], f)
+ }
+
fn required_input_distribution(&self) -> Vec<Distribution> {
self.input_distribution_requirements().into_per_child()
}
diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
index 1abcd9d6c7..55a4b2136c 100644
--- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs
@@ -47,6 +47,7 @@ use crate::{
use arrow::compute::SortOptions;
use arrow::datatypes::SchemaRef;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err, internal_err,
plan_err,
@@ -439,6 +440,15 @@ impl ExecutionPlan for SortMergeJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let join_keys = self.on.iter().flat_map(|(left, right)| [left, right]);
+ let filter = self.filter.iter().map(|filter| filter.expression());
+ crate::apply_expression_roots(join_keys.chain(filter), f)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
index eb358b10b4..86f721b711 100644
--- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
+++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs
@@ -66,6 +66,7 @@ use arrow::compute::concat_batches;
use arrow::datatypes::{ArrowNativeType, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::hash_utils::create_hashes;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::bisect;
use datafusion_common::{
HashSet, JoinSide, JoinType, NullEquality, Result, assert_eq_or_internal_err,
@@ -452,6 +453,15 @@ impl ExecutionPlan for SymmetricHashJoinExec {
vec![&self.left, &self.right]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn crate::PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let join_keys = self.on.iter().flat_map(|(left, right)| [left, right]);
+ let filter = self.filter.iter().map(|filter| filter.expression());
+ crate::apply_expression_roots(join_keys.chain(filter), f)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs
index 8cba650b79..6e1df1f840 100644
--- a/datafusion/physical-plan/src/lib.rs
+++ b/datafusion/physical-plan/src/lib.rs
@@ -45,9 +45,9 @@ pub use crate::distribution_requirements::{
ChildSatisfactionOptions, InputDistributionRequirements,
};
pub use crate::execution_plan::{
- ExecutionPlan, ExecutionPlanProperties, PlanProperties, collect, collect_partitioned,
- displayable, execute_input_stream, execute_stream, execute_stream_partitioned,
- get_plan_string, with_new_children_if_necessary,
+ ExecutionPlan, ExecutionPlanProperties, PlanProperties, apply_expression_roots,
+ collect, collect_partitioned, displayable, execute_input_stream, execute_stream,
+ execute_stream_partitioned, get_plan_string, with_new_children_if_necessary,
};
pub use crate::metrics::Metric;
pub use crate::ordering::InputOrderMode;
diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs
index ddce680fc1..5bc5bc48f1 100644
--- a/datafusion/physical-plan/src/limit.rs
+++ b/datafusion/physical-plan/src/limit.rs
@@ -35,10 +35,11 @@ use crate::{
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::LexOrdering;
+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};
use futures::stream::{Stream, StreamExt};
use log::trace;
@@ -167,6 +168,13 @@ impl ExecutionPlan for GlobalLimitExec {
vec![false]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -398,6 +406,13 @@ impl ExecutionPlan for LocalLimitExec {
vec![true]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
@@ -648,7 +663,6 @@ mod tests {
use arrow::array::RecordBatchOptions;
use arrow::datatypes::Schema;
use datafusion_common::stats::Precision;
- use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::expressions::col;
#[tokio::test]
diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs
index ad54905f47..eb141b8c70 100644
--- a/datafusion/physical-plan/src/memory.rs
+++ b/datafusion/physical-plan/src/memory.rs
@@ -32,10 +32,11 @@ use crate::{
use arrow::array::RecordBatch;
use arrow::datatypes::SchemaRef;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, assert_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryReservation;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use futures::Stream;
@@ -311,6 +312,13 @@ impl ExecutionPlan for LazyMemoryExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/operator_statistics/mod.rs b/datafusion/physical-plan/src/operator_statistics/mod.rs
index 142768fcf4..ec54201e7b 100644
--- a/datafusion/physical-plan/src/operator_statistics/mod.rs
+++ b/datafusion/physical-plan/src/operator_statistics/mod.rs
@@ -1038,6 +1038,7 @@ mod tests {
use std::fmt;
use crate::execution_plan::{Boundedness, EmissionType};
+ use datafusion_common::tree_node::TreeNodeRecursion;
fn make_schema() -> Arc<Schema> {
Arc::new(Schema::new(vec![
@@ -1117,6 +1118,13 @@ mod tests {
&self.cache
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
@@ -1219,6 +1227,13 @@ mod tests {
self.input.properties()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs
index 5d71058269..de07529bab 100644
--- a/datafusion/physical-plan/src/placeholder_row.rs
+++ b/datafusion/physical-plan/src/placeholder_row.rs
@@ -29,9 +29,11 @@ use crate::{
use arrow::array::{ArrayRef, NullArray, RecordBatch, RecordBatchOptions};
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_or_internal_err};
use datafusion_execution::TaskContext;
use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::PhysicalExpr;
use crate::statistics::StatisticsArgs;
use log::trace;
@@ -136,6 +138,13 @@ impl ExecutionPlan for PlaceholderRowExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs
index fac837b09f..32d444fba2 100644
--- a/datafusion/physical-plan/src/projection.rs
+++ b/datafusion/physical-plan/src/projection.rs
@@ -330,6 +330,20 @@ impl ExecutionPlan for ProjectionExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots(
+ self.projector
+ .projection()
+ .as_ref()
+ .iter()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs
index 00df227cb8..4c6f0493ad 100644
--- a/datafusion/physical-plan/src/recursive_query.rs
+++ b/datafusion/physical-plan/src/recursive_query.rs
@@ -37,12 +37,14 @@ use arrow::array::{BooleanArray, BooleanBuilder};
use arrow::compute::filter_record_batch;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
use datafusion_common::{
Result, exec_datafusion_err, internal_datafusion_err, not_impl_err,
};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
use futures::{Stream, StreamExt, ready};
@@ -153,6 +155,13 @@ impl ExecutionPlan for RecursiveQueryExec {
vec![&self.static_term, &self.recursive_term]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
// TODO: control these hints and see whether we can
// infer some from the child plans (static/recursive terms).
fn maintains_input_order(&self) -> Vec<bool> {
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 873f35fd6a..2863524f16 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -53,6 +53,7 @@ use arrow::datatypes::{SchemaRef, UInt32Type};
use arrow_schema::SortOptions;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::{compare_rows, extract_row_at_idx_to_buf, transpose};
use datafusion_common::{
ColumnStatistics, DataFusionError, HashMap, ScalarValue, SplitPoint,
@@ -1337,6 +1338,20 @@ impl ExecutionPlan for RepartitionExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ match self.partitioning() {
+ Partitioning::Hash(exprs, _) => crate::apply_expression_roots(exprs, f),
+ Partitioning::Range(range) => crate::apply_expression_roots(
+ range.ordering().iter().map(|sort_expr| &sort_expr.expr),
+ f,
+ ),
+ _ => Ok(TreeNodeRecursion::Continue),
+ }
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -2934,6 +2949,13 @@ mod tests {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -4168,6 +4190,13 @@ mod test {
)?);
let exec = RepartitionExec::try_new(source, partitioning)?;
+ let mut expressions = vec![];
+ exec.apply_expressions(&mut |expr| {
+ expressions.push(expr.to_string());
+ Ok(TreeNodeRecursion::Continue)
+ })?;
+ assert_eq!(expressions, ["c0@0"]);
+
// Range partition count is fixed by split points, so repartitioned()
// cannot change it to an arbitrary target.
let result = exec.repartitioned(10, &Default::default())?;
diff --git a/datafusion/physical-plan/src/scalar_subquery.rs b/datafusion/physical-plan/src/scalar_subquery.rs
index 73acb2ab13..ee3c2e5d07 100644
--- a/datafusion/physical-plan/src/scalar_subquery.rs
+++ b/datafusion/physical-plan/src/scalar_subquery.rs
@@ -27,9 +27,11 @@
use std::fmt;
use std::sync::Arc;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, ScalarValue, Statistics, exec_err, internal_err};
use datafusion_execution::TaskContext;
use datafusion_expr::physical_planning_context::{ScalarSubqueryResults, SubqueryIndex};
+use datafusion_physical_expr::PhysicalExpr;
use crate::execution_plan::{CardinalityEffect, ExecutionPlan, PlanProperties};
use crate::joins::utils::{OnceAsync, OnceFut};
@@ -224,6 +226,13 @@ impl ExecutionPlan for ScalarSubqueryExec {
)))
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn maintains_input_order(&self) -> Vec<bool> {
// Only the main input (first child); subquery children don't contribute
// to ordering.
@@ -453,6 +462,13 @@ mod tests {
)))
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
partition: usize,
diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs
index 44e2b20adc..5f15f8b6cb 100644
--- a/datafusion/physical-plan/src/sorts/partial_sort.rs
+++ b/datafusion/physical-plan/src/sorts/partial_sort.rs
@@ -70,9 +70,10 @@ use arrow::compute::concat_batches;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
use datafusion_common::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::evaluate_partition_ranges;
use datafusion_execution::{RecordBatchStream, TaskContext};
-use datafusion_physical_expr::LexOrdering;
+use datafusion_physical_expr::{LexOrdering, PhysicalExpr};
use futures::{Stream, StreamExt, ready};
use log::trace;
@@ -374,6 +375,16 @@ impl ExecutionPlan for PartialSortExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots(
+ self.expr.iter().map(|sort_expr| &sort_expr.expr),
+ f,
+ )
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
index 730440a429..78dd9b9696 100644
--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs
+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
@@ -37,6 +37,7 @@ use std::sync::Arc;
use arrow::datatypes::SchemaRef;
use arrow::row::SortField;
use datafusion_common::Result;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_execution::TaskContext;
use datafusion_execution::runtime_env::RuntimeEnv;
use datafusion_physical_expr::PhysicalExpr;
@@ -379,6 +380,16 @@ impl ExecutionPlan for PartitionedTopKExec {
)?))
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots(
+ self.expr.iter().map(|sort_expr| &sort_expr.expr),
+ f,
+ )
+ }
+
fn execute(
&self,
partition: usize,
diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs
index b1b6a84fd9..5c6b86acc5 100644
--- a/datafusion/physical-plan/src/sorts/sort.rs
+++ b/datafusion/physical-plan/src/sorts/sort.rs
@@ -60,6 +60,7 @@ use arrow::array::{RecordBatch, RecordBatchOptions};
use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays};
use arrow::datatypes::SchemaRef;
use datafusion_common::config::SpillCompression;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, Result, assert_or_internal_err, internal_datafusion_err,
unwrap_or_internal_err,
@@ -1278,6 +1279,23 @@ impl ExecutionPlan for SortExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let dynamic_filter = self
+ .filter
+ .as_ref()
+ .map(|filter| filter.read().expr() as Arc<dyn PhysicalExpr>);
+ crate::apply_expression_roots(
+ self.expr
+ .iter()
+ .map(|sort_expr| &sort_expr.expr)
+ .chain(dynamic_filter.iter()),
+ f,
+ )
+ }
+
fn dynamic_expressions_produced(&self) -> Vec<Arc<dyn PhysicalExpr>> {
self.filter
.iter()
@@ -1770,6 +1788,13 @@ mod tests {
Ok(self)
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
index 2add3e1eb8..ac6f5d18cd 100644
--- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
+++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs
@@ -31,9 +31,11 @@ use crate::{
check_if_same_properties,
};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryConsumer;
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, OrderingRequirements};
use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
@@ -281,6 +283,16 @@ impl ExecutionPlan for SortPreservingMergeExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ crate::apply_expression_roots(
+ self.expr.iter().map(|sort_expr| &sort_expr.expr),
+ f,
+ )
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
@@ -1573,6 +1585,12 @@ mod tests {
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs
index 61a9b9cc6d..a82f8d9441 100644
--- a/datafusion/physical-plan/src/streaming.rs
+++ b/datafusion/physical-plan/src/streaming.rs
@@ -33,8 +33,10 @@ use crate::stream::RecordBatchStreamAdapter;
use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream};
use arrow::datatypes::{Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, internal_err, plan_err};
use datafusion_execution::TaskContext;
+use datafusion_physical_expr::PhysicalExpr;
use datafusion_physical_expr::projection::ProjectionMapping;
use datafusion_physical_expr::{EquivalenceProperties, LexOrdering};
@@ -271,6 +273,13 @@ impl ExecutionPlan for StreamingTableExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs
index e8c775a786..68e6ff7eca 100644
--- a/datafusion/physical-plan/src/test.rs
+++ b/datafusion/physical-plan/src/test.rs
@@ -36,6 +36,7 @@ use crate::{DisplayAs, DisplayFormatType, PlanProperties};
use arrow::array::{Array, ArrayRef, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, Statistics, assert_or_internal_err, config::ConfigOptions, project_schema,
};
@@ -45,7 +46,9 @@ use datafusion_physical_expr::equivalence::{
};
use datafusion_physical_expr::expressions::Column;
use datafusion_physical_expr::utils::collect_columns;
-use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, Partitioning};
+use datafusion_physical_expr::{
+ EquivalenceProperties, LexOrdering, Partitioning, PhysicalExpr,
+};
use futures::{Future, FutureExt};
@@ -138,6 +141,13 @@ impl ExecutionPlan for TestMemoryExec {
Vec::new()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/test/exec.rs b/datafusion/physical-plan/src/test/exec.rs
index b92008c6b2..9a3f05a6e0 100644
--- a/datafusion/physical-plan/src/test/exec.rs
+++ b/datafusion/physical-plan/src/test/exec.rs
@@ -35,9 +35,10 @@ use std::{
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{DataFusionError, Result, internal_err};
use datafusion_execution::TaskContext;
-use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_expr::{EquivalenceProperties, PhysicalExpr};
use futures::Stream;
use tokio::sync::Barrier;
@@ -124,6 +125,9 @@ pub struct MockExec {
/// if true (the default), sends data using a separate task to ensure the
/// batches are not available without this stream yielding first
use_task: bool,
+ /// if true, report unknown statistics instead of deriving them from
+ /// `data` (which propagates any planted errors at planning time)
+ unknown_statistics: bool,
cache: Arc<PlanProperties>,
}
@@ -141,6 +145,7 @@ impl MockExec {
data,
schema,
use_task: true,
+ unknown_statistics: false,
cache: Arc::new(cache),
}
}
@@ -153,6 +158,17 @@ impl MockExec {
self
}
+ /// Report unknown statistics rather than computing them from `data`.
+ ///
+ /// By default statistics are derived from `data`, which propagates any
+ /// planted errors when statistics are requested during planning (for
+ /// example when a parent node computes its properties). Use this when a
+ /// planted error should only surface at execution time.
+ pub fn with_unknown_statistics(mut self) -> Self {
+ self.unknown_statistics = true;
+ self
+ }
+
/// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc.
fn compute_properties(schema: SchemaRef) -> PlanProperties {
PlanProperties::new(
@@ -195,6 +211,13 @@ impl ExecutionPlan for MockExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -248,13 +271,14 @@ impl ExecutionPlan for MockExec {
}
}
- // Panics if one of the batches is an error
+ // Errors if one of the batches is an error, unless
+ // `with_unknown_statistics` was used
fn statistics_from_inputs(
&self,
_input_stats: &[Arc<Statistics>],
args: &StatisticsArgs,
) -> Result<Arc<Statistics>> {
- if args.partition().is_some() {
+ if self.unknown_statistics || args.partition().is_some() {
return Ok(Arc::new(Statistics::new_unknown(&self.schema)));
}
let data: Result<Vec<_>> = self
@@ -432,6 +456,13 @@ impl ExecutionPlan for BarrierExec {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Returns a stream which yields data
fn execute(
&self,
@@ -568,6 +599,13 @@ impl ExecutionPlan for ErrorExec {
unimplemented!()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
/// Returns a stream which yields data
fn execute(
&self,
@@ -647,6 +685,13 @@ impl ExecutionPlan for StatisticsExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
@@ -758,6 +803,13 @@ impl ExecutionPlan for BlockingExec {
internal_err!("Children cannot be replaced in {self:?}")
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn execute(
&self,
_partition: usize,
@@ -893,6 +945,13 @@ impl ExecutionPlan for PanicExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs
index 8d77556509..88d2628ae2 100644
--- a/datafusion/physical-plan/src/union.rs
+++ b/datafusion/physical-plan/src/union.rs
@@ -51,6 +51,7 @@ use arrow::datatypes::{Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use datafusion_common::config::ConfigOptions;
use datafusion_common::stats::NdvFallback;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Result, assert_or_internal_err, exec_err, internal_datafusion_err,
};
@@ -330,6 +331,13 @@ impl ExecutionPlan for UnionExec {
self.inputs.iter().collect()
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
@@ -703,6 +711,13 @@ impl ExecutionPlan for InterleaveExec {
vec![false; self.inputs().len()]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs
index ababdd36a9..1877f668c5 100644
--- a/datafusion/physical-plan/src/unnest.rs
+++ b/datafusion/physical-plan/src/unnest.rs
@@ -44,6 +44,7 @@ use arrow::datatypes::{DataType, Int64Type, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use arrow_ord::cmp::lt;
use async_trait::async_trait;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
Constraints, HashMap, HashSet, Result, UnnestOptions, exec_datafusion_err, exec_err,
internal_err,
@@ -227,6 +228,13 @@ impl ExecutionPlan for UnnestExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
mut children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
index 2ca7187faf..e9a0d9f47c 100644
--- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs
@@ -50,6 +50,7 @@ use arrow::{
};
use datafusion_common::hash_utils::create_hashes;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::{
evaluate_partition_ranges, get_at_indices, get_row_at_idx,
};
@@ -312,6 +313,21 @@ impl ExecutionPlan for BoundedWindowAggExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let expressions = self.window_expr.iter().flat_map(|window_expr| {
+ let expressions = window_expr.all_expressions();
+ expressions
+ .args
+ .into_iter()
+ .chain(expressions.partition_by_exprs)
+ .chain(expressions.order_by_exprs)
+ });
+ crate::apply_expression_roots(expressions, f)
+ }
+
fn required_input_ordering(&self) -> Vec<Option<OrderingRequirements>> {
let partition_bys = self.window_expr()[0].partition_by();
let order_keys = self.window_expr()[0].order_by();
diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs
index 025f642f90..bbf9a14fd5 100644
--- a/datafusion/physical-plan/src/windows/window_agg_exec.rs
+++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs
@@ -45,6 +45,7 @@ use arrow::datatypes::SchemaRef;
use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::utils::{evaluate_partition_ranges, transpose};
use datafusion_common::{Result, assert_eq_or_internal_err};
use datafusion_execution::TaskContext;
@@ -214,6 +215,21 @@ impl ExecutionPlan for WindowAggExec {
vec![&self.input]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let expressions = self.window_expr.iter().flat_map(|window_expr| {
+ let expressions = window_expr.all_expressions();
+ expressions
+ .args
+ .into_iter()
+ .chain(expressions.partition_by_exprs)
+ .chain(expressions.order_by_exprs)
+ });
+ crate::apply_expression_roots(expressions, f)
+ }
+
fn maintains_input_order(&self) -> Vec<bool> {
vec![true]
}
diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs
index c92face1e5..83cd0a15a6 100644
--- a/datafusion/physical-plan/src/work_table.rs
+++ b/datafusion/physical-plan/src/work_table.rs
@@ -32,10 +32,11 @@ use crate::{
use crate::statistics::StatisticsArgs;
use arrow::datatypes::SchemaRef;
use arrow::record_batch::RecordBatch;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{Result, assert_eq_or_internal_err, internal_datafusion_err};
use datafusion_execution::TaskContext;
use datafusion_execution::memory_pool::MemoryReservation;
-use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
+use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
/// A vector of record batches with a memory reservation.
#[derive(Debug)]
@@ -186,6 +187,13 @@ impl ExecutionPlan for WorkTableExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc<Self>,
_: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 1252668f57..de684857f4 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -171,6 +171,22 @@ mod file_scan_config_serde {
"serde-test"
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(
+ &Arc<dyn PhysicalExpr>,
+ )
+ -> Result<datafusion_common::tree_node::TreeNodeRecursion>,
+ ) -> Result<datafusion_common::tree_node::TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(
+ self.projection
+ .iter()
+ .flatten()
+ .map(|proj_expr| &proj_expr.expr),
+ f,
+ )
+ }
+
fn try_pushdown_projection(
&self,
projection: &FileProjectionExprs,
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index 99ba5cf0ab..19249cf97b 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -108,6 +108,7 @@ use datafusion_common::file_options::json_writer::JsonWriterOptions;
use datafusion_common::format::ExplainFormat;
use datafusion_common::parsers::CompressionTypeVariant;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, JoinSide, NullEquality, Result, UnnestOptions, exec_datafusion_err,
internal_datafusion_err, internal_err, not_impl_err,
@@ -323,6 +324,13 @@ impl ExecutionPlan for DowncastDelegatingExec {
self.inner.children()
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ self.inner.apply_expressions(f)
+ }
+
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn ExecutionPlan>>,
@@ -4934,6 +4942,61 @@ fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {
Ok(())
}
+#[test]
+fn test_aggregate_without_dynamic_filter_roundtrip() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Partial,
+ PhysicalGroupBy::new_single(vec![]),
+ vec![
+ AggregateExprBuilder::new(
+ datafusion::functions_aggregate::min_max::min_udaf(),
+ vec![col_a],
+ )
+ .schema(Arc::clone(&schema))
+ .alias("min_a")
+ .build()
+ .map(Arc::new)?,
+ ],
+ vec![None],
+ child,
+ Arc::clone(&schema),
+ )?) as Arc<dyn ExecutionPlan>;
+
+ let mut config = ConfigOptions::default();
+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;
+ let optimizer = FilterPushdown::new_post_optimization();
+ let plan = optimizer.optimize(aggregate, &config)?;
+ assert!(
+ plan.downcast_ref::<AggregateExec>()
+ .expect("Should be AggregateExec")
+ .dynamic_expressions_produced()
+ .is_empty()
+ );
+
+ let ctx = SessionContext::new();
+ let codec = DefaultPhysicalExtensionCodec {};
+ let converter = DefaultPhysicalProtoConverter {};
+ let bytes = physical_plan_to_bytes_with_proto_converter(plan, &codec, &converter)?;
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &converter,
+ )?;
+
+ assert!(
+ deserialized
+ .downcast_ref::<AggregateExec>()
+ .expect("Should be AggregateExec")
+ .dynamic_expressions_produced()
+ .is_empty()
+ );
+ Ok(())
+}
+
/// Test that plan containing a SortExec with dynamic filter pushdown
/// can be serialized and deserialized while preserving references to the dynamic filter.
#[test]
@@ -5054,6 +5117,13 @@ impl ExecutionPlan for CustomExecWithExprs {
vec![&self.child]
}
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ datafusion_physical_plan::apply_expression_roots(&self.exprs, f)
+ }
+
fn with_new_children(
self: Arc<Self>,
_children: Vec<Arc<dyn ExecutionPlan>>,
diff --git a/docs/source/library-user-guide/custom-table-providers.md b/docs/source/library-user-guide/custom-table-providers.md
index 540782e3e8..c6a316aa74 100644
--- a/docs/source/library-user-guide/custom-table-providers.md
+++ b/docs/source/library-user-guide/custom-table-providers.md
@@ -766,6 +766,7 @@ impl DatePartitionedTable {
# fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { vec![] }
# fn with_new_children(self: Arc<Self>, _: Vec<Arc<dyn ExecutionPlan>>) -> Result<Arc<dyn ExecutionPlan>> { Ok(self) }
# fn execute(&self, _: usize, _: Arc<TaskContext>) -> Result<SendableRecordBatchStream> { todo!() }
+# fn apply_expressions(&self, _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>) -> Result<TreeNodeRecursion> { Ok(TreeNodeRecursion::Continue) }
# }
```
@@ -909,6 +910,13 @@ impl ExecutionPlan for CountingExec {
batch_stream,
)))
}
+
+# fn apply_expressions(
+# &self,
+# _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
+# ) -> Result<TreeNodeRecursion> {
+# Ok(TreeNodeRecursion::Continue)
+# }
}
```
diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md
index 2811fb4df2..73eb3dfd86 100644
--- a/docs/source/library-user-guide/upgrading/55.0.0.md
+++ b/docs/source/library-user-guide/upgrading/55.0.0.md
@@ -986,6 +986,11 @@ let plan = deserialize_bytes(&proto_bytes)?;
See [PR #23827](https://github.com/apache/datafusion/pull/23827) for details.
+### `ExecutionPlan::apply_expressions` is now a required method
+
+`apply_expressions` has been added as a **required** method on the `ExecutionPlan`, `FileSource`, and `DataSource` traits. Any custom implementation of
+these traits must now implement `apply_expressions`. See docs on `ExecutionPlan::apply_expressions` for migration details.
+
### `WindowExpr::evaluate_stateful` now takes a `WindowEvalContext`
`WindowExpr::evaluate_stateful` (and the provided