Apache DataFusion SQL Query Engine
chore: deprecate `with_dynamic_filter_expr` on Sort/Aggregate/HashJoin (#24610)
Adrian Garcia Badaracco2f592edca60a
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex a7f08ea83a..59d382302a 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -1119,7 +1119,23 @@ impl AggregateExec {/// Replace the dynamic filter expression. This method errors if the aggregate does not/// support dynamic filtering or if the filter expression is incompatible with this/// [`AggregateExec`].+ #[deprecated(+ since = "56.0.0",+ note = "unused by DataFusion; `AggregateExec` restores its dynamic filter in `AggregateExec::try_from_proto`, which sets the field directly. There is no replacement; please open an issue if you have a use case for it."+ )]pub fn with_dynamic_filter_expr(+ self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ self.set_dynamic_filter(filter)+ }++ /// Replace the dynamic filter expression, validating that it is compatible+ /// with this [`AggregateExec`].+ ///+ /// Only used to restore the filter when decoding a serialized plan: every+ /// other code path creates the filter in [`AggregateExec::try_new`].+ fn set_dynamic_filter(mut self,filter: Arc<DynamicFilterPhysicalExpr>,) -> Result<Self> {@@ -2695,7 +2711,7 @@ impl AggregateExec {"AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr")})?;- aggregate.with_dynamic_filter_expr(dynamic_filter)?+ aggregate.set_dynamic_filter(dynamic_filter)?} else {let mut aggregate = aggregate;aggregate.dynamic_filter = None;@@ -7998,7 +8014,7 @@ mod tests {}}- /// Test that [`AggregateExec::with_dynamic_filter_expr`] overrides the existing dynamic filter+ /// Test that [`AggregateExec::set_dynamic_filter`] overrides the existing dynamic filter#[test]fn test_with_dynamic_filter() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));@@ -8025,7 +8041,7 @@ mod tests {vec![col("a", &schema)?],lit(false),));- let agg = agg.with_dynamic_filter_expr(Arc::clone(&new_df))?;+ let agg = agg.set_dynamic_filter(Arc::clone(&new_df))?;let produced = agg.dynamic_expressions_produced();assert_eq!(produced.len(), 1);assert_eq!(produced[0].expression_id(), new_df.expression_id());@@ -8050,7 +8066,7 @@ mod tests {};// Hard to assert this because the filter is identical. No error means// the filter was accepted. That's a good enough assertion for now.- let _agg = agg.with_dynamic_filter_expr(remapped_df)?;+ let _agg = agg.set_dynamic_filter(remapped_df)?;Ok(())}@@ -8086,7 +8102,7 @@ mod tests {Ok(())}- /// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the aggregate does not support dynamic filtering+ /// Test that [`AggregateExec::set_dynamic_filter`] errors when the aggregate does not support dynamic filtering#[test]fn test_with_dynamic_filter_error_unsupported() -> Result<()> {let schema = Arc::new(Schema::new(vec![@@ -8115,11 +8131,11 @@ mod tests {vec![col("a", &schema)?],lit(true),));- assert!(agg.with_dynamic_filter_expr(df).is_err());+ assert!(agg.set_dynamic_filter(df).is_err());Ok(())}- /// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the column is not in the schema+ /// Test that [`AggregateExec::set_dynamic_filter`] errors when the column is not in the schema#[test]fn test_with_dynamic_filter_error_column_mismatch() -> Result<()> {let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));@@ -8143,7 +8159,7 @@ mod tests {vec![Arc::new(Column::new("bad", 99)) as _],lit(true),));- assert!(agg.with_dynamic_filter_expr(df).is_err());+ assert!(agg.set_dynamic_filter(df).is_err());Ok(())}}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex fb8077264a..df10a8a5fc 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -1008,7 +1008,25 @@ impl HashJoinExec {////// Validates that the filter's children reference valid columns in/// the probe (right) side's schema.+ #[deprecated(+ since = "56.0.0",+ note = "unused by DataFusion; `HashJoinExec` restores its dynamic filter in `HashJoinExec::try_from_proto`, which sets the field directly. There is no replacement; please open an issue if you have a use case for it."+ )]pub fn with_dynamic_filter_expr(+ self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ self.set_dynamic_filter(filter)+ }++ /// Set the dynamic filter on this hash join, resetting any internal state+ /// that depends on an existing one and validating that the filter's+ /// children reference valid columns in the probe (right) side's schema.+ ///+ /// Only used to restore the filter when decoding a serialized plan: every+ /// other code path installs the filter in+ /// [`ExecutionPlan::handle_child_pushdown_result`].+ fn set_dynamic_filter(mut self,filter: Arc<DynamicFilterPhysicalExpr>,) -> Result<Self> {@@ -2101,7 +2119,7 @@ impl HashJoinExec {"HashJoinExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr")})?;- hash_join = hash_join.with_dynamic_filter_expr(df)?;+ hash_join = hash_join.set_dynamic_filter(df)?;}Ok(Arc::new(hash_join))@@ -7851,7 +7869,7 @@ mod tests {vec![Arc::new(Column::new("b1", 1)) as _],lit(true),));- let join = join.with_dynamic_filter_expr(Arc::clone(&df))?;+ let join = join.set_dynamic_filter(Arc::clone(&df))?;let produced = join.dynamic_expressions_produced();assert_eq!(produced.len(), 1);@@ -7894,7 +7912,7 @@ mod tests {NullEquality::NullEqualsNothing,false,)?- .with_dynamic_filter_expr(dynamic_filter)?;+ .set_dynamic_filter(dynamic_filter)?;let err = join.swap_inputs(PartitionMode::CollectLeft).unwrap_err();assert_contains!(@@ -8143,7 +8161,7 @@ mod tests {vec![Arc::new(Column::new("bad", 99)) as _],lit(true),));- assert!(join.with_dynamic_filter_expr(df).is_err());+ assert!(join.set_dynamic_filter(df).is_err());Ok(())}}diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex c91df46351..9880c8b5e5 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -1113,7 +1113,24 @@ impl SortExec {////// Validates that the filter's children reference valid columns in/// the sort's input schema.+ #[deprecated(+ since = "56.0.0",+ note = "unused by DataFusion; `SortExec` restores its dynamic filter in `SortExec::try_from_proto`, which sets the field directly. There is no replacement; please open an issue if you have a use case for it."+ )]pub fn with_dynamic_filter_expr(+ self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ self.set_dynamic_filter(filter)+ }++ /// Replace the dynamic filter expression for this sort, resetting any+ /// internal state which depends on the previous one and validating that the+ /// filter's children reference valid columns in the sort's input schema.+ ///+ /// Only used to restore the filter when decoding a serialized plan: every+ /// other code path creates the filter in [`SortExec::with_fetch`].+ fn set_dynamic_filter(mut self,filter: Arc<DynamicFilterPhysicalExpr>,) -> Result<Self> {@@ -1726,7 +1743,7 @@ impl SortExec {"SortExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr")})?;- new_sort.with_dynamic_filter_expr(df)?+ new_sort.set_dynamic_filter(df)?} else {new_sort};@@ -3605,7 +3622,7 @@ mod tests {.expression_id().expect("DynamicFilterPhysicalExpr always has an expression_id");- // with_dynamic_filter replaces it with a new TopKDynamicFilters.+ // set_dynamic_filter replaces it with a new TopKDynamicFilters.let new_df = Arc::new(DynamicFilterPhysicalExpr::new(vec![Arc::new(Column::new("a", 0)) as _],lit(true),@@ -3613,7 +3630,7 @@ mod tests {let new_id = new_df.expression_id().expect("DynamicFilterPhysicalExpr always has an expression_id");- let sort = sort.with_dynamic_filter_expr(Arc::clone(&new_df))?;+ let sort = sort.set_dynamic_filter(Arc::clone(&new_df))?;let produced = sort.dynamic_expressions_produced();assert_eq!(produced.len(), 1);let restored_id = produced[0]@@ -3725,7 +3742,7 @@ mod tests {[PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into(),input,)- .with_dynamic_filter_expr(dynamic_filter)?+ .set_dynamic_filter(dynamic_filter)?.with_preserve_partitioning(true).with_fetch(Some(2));@@ -3774,7 +3791,7 @@ mod tests {vec![Arc::new(Column::new("bad", 99)) as _],lit(true),));- assert!(sort.with_dynamic_filter_expr(df).is_err());+ assert!(sort.set_dynamic_filter(df).is_err());Ok(())}