Apache DataFusion SQL Query Engine
proto: serialize dynamic filters on Sort, Aggregate, HashJoin plan nodes (#22011)
Jayant Shrivastava47655fd6c9ef
diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown.rsindex 4ff1fad8f5..f56b8c6d70 100644--- a/datafusion/core/tests/physical_optimizer/filter_pushdown.rs+++ b/datafusion/core/tests/physical_optimizer/filter_pushdown.rs@@ -2835,7 +2835,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_is_used() {// Verify that a dynamic filter was createdlet dynamic_filter = hash_join- .dynamic_filter_for_test()+ .dynamic_filter_expr().expect("Dynamic filter should be created");// Verify that is_used() returns the expected value based on probe side support.diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 212dca6cd5..7b3f7dcc76 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -73,7 +73,9 @@ pub type PhysicalExprRef = Arc<dyn PhysicalExpr>;/// [`create_physical_expr`]: https://docs.rs/datafusion/latest/datafusion/physical_expr/fn.create_physical_expr.html/// [`Column`]: https://docs.rs/datafusion/latest/datafusion/physical_expr/expressions/struct.Column.htmlpub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {- /// Get the data type of this expression, given the schema of the input+ /// Get the data type of this expression, given the schema of the input.+ /// Returns an error if the data type cannot be determined, ex. if the+ /// schema is missing a required field.fn data_type(&self, input_schema: &Schema) -> Result<DataType> {Ok(self.return_field(input_schema)?.data_type().to_owned())}diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rsindex 2db328377a..5b9de88216 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs@@ -86,7 +86,7 @@ pub struct DynamicFilterPhysicalExpr {/// **Warning:** exposed publicly solely so that proto (de)serialization in/// `datafusion-proto` can read and rebuild this state. Do not treat this type/// or its layout as a stable API.-#[derive(Clone)]+#[derive(Clone, Debug)]pub struct Inner {/// A unique identifier for the expression.pub expression_id: u64,@@ -100,24 +100,6 @@ pub struct Inner {pub is_complete: bool,}-// TODO: Include expression_id in Debug output.-//-// See https://github.com/apache/datafusion/issues/20418. Currently, plan nodes-// like `HashJoinExec`, `AggregateExec`, `SortExec` do not serialize their-// dynamic filter. They auto-create one on decode with a fresh `expression_id`,-// so a round-trip Debug comparison would diverge purely on the id even when-// the rest of the state is preserved. Excluding it from Debug keeps those-// roundtrip equality assertions meaningful until that work lands.-impl std::fmt::Debug for Inner {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("Inner")- .field("generation", &self.generation)- .field("expr", &self.expr)- .field("is_complete", &self.is_complete)- .finish()- }-}-impl Inner {fn new(expr: Arc<dyn PhysicalExpr>) -> Self {Self {diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rsindex 9953ce3f84..d1498e4a3e 100644--- a/datafusion/physical-plan/src/aggregates/mod.rs+++ b/datafusion/physical-plan/src/aggregates/mod.rs@@ -47,7 +47,8 @@ use arrow_schema::FieldRef;use datafusion_common::stats::Precision;use datafusion_common::tree_node::TreeNodeRecursion;use datafusion_common::{- Constraint, Constraints, Result, ScalarValue, assert_eq_or_internal_err, not_impl_err,+ Constraint, Constraints, Result, ScalarValue, assert_eq_or_internal_err,+ internal_err, not_impl_err,};use datafusion_execution::TaskContext;use datafusion_expr::{Accumulator, Aggregate};@@ -893,6 +894,47 @@ impl AggregateExec {&self.filter_expr}+ /// Returns the dynamic filter expression for this aggregate, if set.+ pub fn dynamic_filter_expr(&self) -> Option<&Arc<DynamicFilterPhysicalExpr>> {+ self.dynamic_filter.as_ref().map(|df| &df.filter)+ }++ /// Replace the dynamic filter expression. This method errors if the aggregate does not+ /// support dynamic filtering or if the filter expression is incompatible with this+ /// [`AggregateExec`].+ pub fn with_dynamic_filter_expr(+ mut self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ // If there is no dynamic filter state initialized via `try_new`, then+ // we can safely assume that the aggregate does not support dynamic filtering.+ let Some(dyn_filter) = self.dynamic_filter.as_ref() else {+ return internal_err!("Aggregate does not support dynamic filtering");+ };++ // Validate that the filter is compatible with the aggregation columns.+ let cols = self.cols_for_dynamic_filter(&dyn_filter.supported_accumulators_info);+ if cols.len() != filter.children().len() {+ return internal_err!(+ "Dynamic filter expression is incompatible with aggregate due to mismatched number of columns"+ );+ }+ for (col, child) in cols.iter().zip(filter.children()) {+ if !col.eq(child) {+ return internal_err!(+ "Dynamic filter expression is incompatible with aggregate due to mismatched column references {col} != {child}"+ );+ }+ }++ // Overwrite our filter+ self.dynamic_filter = Some(Arc::new(AggrDynFilter {+ filter,+ supported_accumulators_info: dyn_filter.supported_accumulators_info.clone(),+ }));+ Ok(self)+ }+/// Input planpub fn input(&self) -> &Arc<dyn ExecutionPlan> {&self.input@@ -1285,6 +1327,28 @@ impl AggregateExec {}}+ // Collect column references for the dynamic filter expression from the supported accumulators.+ fn cols_for_dynamic_filter(+ &self,+ supported_accumulators_info: &[PerAccumulatorDynFilter],+ ) -> Vec<Arc<dyn PhysicalExpr>> {+ let all_cols: Vec<Arc<dyn PhysicalExpr>> = supported_accumulators_info+ .iter()+ .filter_map(|info| {+ // This should always be true due to how the supported accumulators+ // are constructed. See `init_dynamic_filter` for more details.+ if let [arg] = &self.aggr_expr[info.aggr_index].expressions().as_slice()+ && arg.is::<Column>()+ {+ return Some(Arc::clone(arg));+ }+ None+ })+ .collect();+ debug_assert!(all_cols.len() == supported_accumulators_info.len());+ all_cols+ }+/// Calculate scaled byte size based on row count ratio./// Returns `Precision::Absent` if input statistics are insufficient./// Returns `Precision::Inexact` with the scaled value otherwise.@@ -2200,6 +2264,7 @@ mod tests {use crate::coalesce_partitions::CoalescePartitionsExec;use crate::common;use crate::common::collect;+ use crate::empty::EmptyExec;use crate::execution_plan::Boundedness;use crate::expressions::col;use crate::metrics::MetricValue;@@ -2225,6 +2290,7 @@ mod tests {use datafusion_functions_aggregate::count::count_udaf;use datafusion_functions_aggregate::first_last::{first_value_udaf, last_value_udaf};use datafusion_functions_aggregate::median::median_udaf;+ use datafusion_functions_aggregate::min_max::min_udaf;use datafusion_functions_aggregate::sum::sum_udaf;use datafusion_physical_expr::Partitioning;use datafusion_physical_expr::PhysicalSortExpr;@@ -3846,13 +3912,10 @@ mod tests {// Test with MIN for simple intermediate state (min) and AVG for multiple intermediate states (partial sum, partial count).let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(- AggregateExprBuilder::new(- datafusion_functions_aggregate::min_max::min_udaf(),- vec![col("b", &schema)?],- )- .schema(Arc::clone(&schema))- .alias("MIN(b)")- .build()?,+ AggregateExprBuilder::new(min_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("MIN(b)")+ .build()?,),Arc::new(AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])@@ -3991,13 +4054,10 @@ mod tests {// Test with MIN for simple intermediate state (min) and AVG for multiple intermediate states (partial sum, partial count).let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(- AggregateExprBuilder::new(- datafusion_functions_aggregate::min_max::min_udaf(),- vec![col("b", &schema)?],- )- .schema(Arc::clone(&schema))- .alias("MIN(b)")- .build()?,+ AggregateExprBuilder::new(min_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("MIN(b)")+ .build()?,),Arc::new(AggregateExprBuilder::new(avg_udaf(), vec![col("b", &schema)?])@@ -4945,4 +5005,118 @@ mod tests {Ok(())}++ /// Test that [`AggregateExec::with_dynamic_filter_expr`] overrides the existing dynamic filter+ #[test]+ fn test_with_dynamic_filter() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ // Partial min aggregate supports dynamic filtering+ let agg = AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![Arc::new(+ AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()?,+ )],+ vec![None],+ child,+ Arc::clone(&schema),+ )?;++ // Assertion 1: A filter with the same children can override the existing+ // dynamic filter.+ let new_df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![col("a", &schema)?],+ lit(false),+ ));+ let agg = agg.with_dynamic_filter_expr(Arc::clone(&new_df))?;++ // The aggregate's filter should now resolve to the new inner expression.+ let swapped = agg+ .dynamic_filter_expr()+ .expect("should still have dynamic filter")+ .current()?;+ assert_eq!(format!("{swapped}"), format!("{}", lit(false)));++ // Assertion 2: A filter that has been through `PhysicalExpr::with_new_children`+ // should still be accepted when the new children are equivalent to the originals.+ let new_df_as_pexpr: Arc<dyn PhysicalExpr> =+ Arc::<DynamicFilterPhysicalExpr>::clone(&new_df);+ let remapped_pexpr =+ new_df_as_pexpr.with_new_children(vec![col("a", &schema)?])?;+ let Ok(remapped_df) = (remapped_pexpr as Arc<dyn std::any::Any + Send + Sync>)+ .downcast::<DynamicFilterPhysicalExpr>()+ else {+ panic!("should be DynamicFilterPhysicalExpr after with_new_children");+ };+ // Hard to assert this because the filter is identical. No error means+ // the filter was accepted. That's a good enough assertion for now.+ let _agg = agg.with_dynamic_filter_expr(remapped_df)?;+ Ok(())+ }++ /// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the aggregate does not support dynamic filtering+ #[test]+ fn test_with_dynamic_filter_error_unsupported() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Int64, false),+ ]));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ // Final mode with a group-by does not support dynamic filters.+ let agg = AggregateExec::try_new(+ AggregateMode::Final,+ PhysicalGroupBy::new_single(vec![(col("a", &schema)?, "a".to_string())]),+ vec![Arc::new(+ AggregateExprBuilder::new(sum_udaf(), vec![col("b", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("sum_b")+ .build()?,+ )],+ vec![None],+ child,+ Arc::clone(&schema),+ )?;+ assert!(agg.dynamic_filter_expr().is_none());++ let df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![col("a", &schema)?],+ lit(true),+ ));+ assert!(agg.with_dynamic_filter_expr(df).is_err());+ Ok(())+ }++ /// Test that [`AggregateExec::with_dynamic_filter_expr`] errors when the column is not in the schema+ #[test]+ fn test_with_dynamic_filter_error_column_mismatch() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ let agg = AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![Arc::new(+ AggregateExprBuilder::new(min_udaf(), vec![col("a", &schema)?])+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()?,+ )],+ vec![None],+ child,+ Arc::clone(&schema),+ )?;++ let df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("bad", 99)) as _],+ lit(true),+ ));+ assert!(agg.with_dynamic_filter_expr(df).is_err());+ Ok(())+ }}diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 4ebbf7cb31..3cdd60d7ab 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -903,14 +903,34 @@ impl HashJoinExec {}/// Get the dynamic filter expression for testing purposes.- /// Returns `None` if no dynamic filter has been set.- ///- /// This method is intended for testing only and should not be used in production code.- #[doc(hidden)]- pub fn dynamic_filter_for_test(&self) -> Option<&Arc<DynamicFilterPhysicalExpr>> {+ /// Returns the dynamic filter expression for this hash join, if set.+ pub fn dynamic_filter_expr(&self) -> Option<&Arc<DynamicFilterPhysicalExpr>> {self.dynamic_filter.as_ref().map(|df| &df.filter)}+ /// Set the dynamic filter on this hash join.+ ///+ /// Resets any internal state that depends on any existing dynamic filter.+ ///+ /// Validates that the filter's children reference valid columns in+ /// the probe (right) side's schema.+ pub fn with_dynamic_filter_expr(+ mut self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ let probe_schema = self.right.schema();+ for child in filter.children() {+ child.data_type(&probe_schema)?;+ }+ self.dynamic_filter = Some(HashJoinExecDynamicFilter {+ filter,+ // Initialize with an empty accumulator which will be lazily populated+ // during execution.+ build_accumulator: OnceLock::new(),+ });+ Ok(self)+ }+/// Calculate order preservation flags for this hash join.fn maintains_input_order(join_type: JoinType) -> Vec<bool> {vec![@@ -6307,4 +6327,69 @@ mod tests {assert_eq!(lr_is_preserved(JoinType::RightAnti), (true, true));assert_eq!(lr_is_preserved(JoinType::RightMark), (false, true));}++ #[test]+ fn test_with_dynamic_filter() -> Result<()> {+ let (_, _, on) = build_schema_and_on()?;+ let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));+ let right = build_table(("a2", &vec![1]), ("b1", &vec![1]), ("c2", &vec![1]));++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?;+ assert!(join.dynamic_filter_expr().is_none());++ let df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("b1", 1)) as _],+ lit(true),+ ));+ let join = join.with_dynamic_filter_expr(Arc::clone(&df))?;++ let restored = join+ .dynamic_filter_expr()+ .expect("should have dynamic filter");+ assert_eq!(+ restored+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id"),+ df.expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id"),+ );+ Ok(())+ }++ #[test]+ fn test_with_dynamic_filter_rejects_invalid_columns() -> Result<()> {+ let (_, _, on) = build_schema_and_on()?;+ let left = build_table(("a1", &vec![1]), ("b1", &vec![1]), ("c1", &vec![1]));+ let right = build_table(("a2", &vec![1]), ("b1", &vec![1]), ("c2", &vec![1]));++ let join = HashJoinExec::try_new(+ left,+ right,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?;++ // Column index 99 is out of bounds for the right (probe) side schema.+ let df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("bad", 99)) as _],+ lit(true),+ ));+ assert!(join.with_dynamic_filter_expr(df).is_err());+ Ok(())+ }}diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 044580a86f..90d4b5ec12 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -978,6 +978,30 @@ impl SortExec {self.fetch}+ /// Returns the dynamic filter expression for this sort (TopK), if set.+ pub fn dynamic_filter_expr(&self) -> Option<Arc<DynamicFilterPhysicalExpr>> {+ self.filter.as_ref().map(|f| f.read().expr())+ }++ /// Replace the dynamic filter expression for this sort.+ ///+ ///+ /// Resets any internal state which may depend on the previous dynamic filter.+ ///+ /// Validates that the filter's children reference valid columns in+ /// the sort's input schema.+ pub fn with_dynamic_filter_expr(+ mut self,+ filter: Arc<DynamicFilterPhysicalExpr>,+ ) -> Result<Self> {+ let input_schema = self.input.schema();+ for child in filter.children() {+ child.data_type(&input_schema)?;+ }+ self.filter = Some(Arc::new(RwLock::new(TopKDynamicFilters::new(filter))));+ Ok(self)+ }+fn output_partitioning_helper(input: &Arc<dyn ExecutionPlan>,preserve_partitioning: bool,@@ -2725,6 +2749,71 @@ mod tests {Ok(())}+ #[test]+ fn test_with_dynamic_filter() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ let sort = SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::new(Column::new("a", 0)),+ options: SortOptions::default(),+ }])+ .unwrap(),+ child,+ )+ .with_fetch(Some(10));++ // SortExec with fetch creates a dynamic filter automatically.+ let original_id = sort+ .dynamic_filter_expr()+ .expect("should have dynamic filter with fetch")+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id");++ // with_dynamic_filter replaces it with a new TopKDynamicFilters.+ let new_df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0)) as _],+ lit(true),+ ));+ let new_id = new_df+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id");+ let sort = sort.with_dynamic_filter_expr(Arc::clone(&new_df))?;+ let restored_id = sort+ .dynamic_filter_expr()+ .expect("should still have dynamic filter")+ .expression_id()+ .expect("DynamicFilterPhysicalExpr always has an expression_id");+ assert_eq!(restored_id, new_id);+ assert_ne!(restored_id, original_id);+ Ok(())+ }++ #[test]+ fn test_with_dynamic_filter_rejects_invalid_columns() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let child = Arc::new(EmptyExec::new(Arc::clone(&schema)));++ let sort = SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::new(Column::new("a", 0)),+ options: SortOptions::default(),+ }])+ .unwrap(),+ child,+ )+ .with_fetch(Some(10));++ // Column index 99 is out of bounds for the input schema.+ let df = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("bad", 99)) as _],+ lit(true),+ ));+ assert!(sort.with_dynamic_filter_expr(df).is_err());+ Ok(())+ }+/// Verifies that `ExternalSorter::sort()` transfers the pre-reserved/// merge bytes to the merge stream via `take()`, rather than leaving/// them in the sorter (via `new_empty()`).diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 865887d41e..d34acf36c5 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -1188,6 +1188,8 @@ message HashJoinExecNode {JoinFilter filter = 8;repeated uint32 projection = 9;bool null_aware = 10;+ // Optional dynamic filter expression for pushing down to the probe side.+ PhysicalExprNode dynamic_filter = 11;}enum StreamPartitionMode {@@ -1319,6 +1321,8 @@ message AggregateExecNode {repeated MaybeFilter filter_expr = 10;AggLimit limit = 11;bool has_grouping_set = 12;+ // Optional dynamic filter expression for pushing down to the child.+ PhysicalExprNode dynamic_filter = 13;}message GlobalLimitExecNode {@@ -1340,6 +1344,8 @@ message SortExecNode {// Maximum number of highest/lowest rows to fetch; negative means no limitint64 fetch = 3;bool preserve_partitioning = 4;+ // Optional dynamic filter expression for TopK pushdown.+ PhysicalExprNode dynamic_filter = 5;}message SortPreservingMergeExecNode {diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex b8639afd04..f71fabbdac 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -154,6 +154,9 @@ impl serde::Serialize for AggregateExecNode {if self.has_grouping_set {len += 1;}+ if self.dynamic_filter.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.AggregateExecNode", len)?;if !self.group_expr.is_empty() {struct_ser.serialize_field("groupExpr", &self.group_expr)?;@@ -193,6 +196,9 @@ impl serde::Serialize for AggregateExecNode {if self.has_grouping_set {struct_ser.serialize_field("hasGroupingSet", &self.has_grouping_set)?;}+ if let Some(v) = self.dynamic_filter.as_ref() {+ struct_ser.serialize_field("dynamicFilter", v)?;+ }struct_ser.end()}}@@ -223,6 +229,8 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {"limit","has_grouping_set","hasGroupingSet",+ "dynamic_filter",+ "dynamicFilter",];#[allow(clippy::enum_variant_names)]@@ -239,6 +247,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {FilterExpr,Limit,HasGroupingSet,+ DynamicFilter,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -272,6 +281,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {"filterExpr" | "filter_expr" => Ok(GeneratedField::FilterExpr),"limit" => Ok(GeneratedField::Limit),"hasGroupingSet" | "has_grouping_set" => Ok(GeneratedField::HasGroupingSet),+ "dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -303,6 +313,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {let mut filter_expr__ = None;let mut limit__ = None;let mut has_grouping_set__ = None;+ let mut dynamic_filter__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::GroupExpr => {@@ -377,6 +388,12 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {}has_grouping_set__ = Some(map_.next_value()?);}+ GeneratedField::DynamicFilter => {+ if dynamic_filter__.is_some() {+ return Err(serde::de::Error::duplicate_field("dynamicFilter"));+ }+ dynamic_filter__ = map_.next_value()?;+ }}}Ok(AggregateExecNode {@@ -392,6 +409,7 @@ impl<'de> serde::Deserialize<'de> for AggregateExecNode {filter_expr: filter_expr__.unwrap_or_default(),limit: limit__,has_grouping_set: has_grouping_set__.unwrap_or_default(),+ dynamic_filter: dynamic_filter__,})}}@@ -8836,6 +8854,9 @@ impl serde::Serialize for HashJoinExecNode {if self.null_aware {len += 1;}+ if self.dynamic_filter.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.HashJoinExecNode", len)?;if let Some(v) = self.left.as_ref() {struct_ser.serialize_field("left", v)?;@@ -8870,6 +8891,9 @@ impl serde::Serialize for HashJoinExecNode {if self.null_aware {struct_ser.serialize_field("nullAware", &self.null_aware)?;}+ if let Some(v) = self.dynamic_filter.as_ref() {+ struct_ser.serialize_field("dynamicFilter", v)?;+ }struct_ser.end()}}@@ -8893,6 +8917,8 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {"projection","null_aware","nullAware",+ "dynamic_filter",+ "dynamicFilter",];#[allow(clippy::enum_variant_names)]@@ -8906,6 +8932,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {Filter,Projection,NullAware,+ DynamicFilter,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -8936,6 +8963,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {"filter" => Ok(GeneratedField::Filter),"projection" => Ok(GeneratedField::Projection),"nullAware" | "null_aware" => Ok(GeneratedField::NullAware),+ "dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -8964,6 +8992,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {let mut filter__ = None;let mut projection__ = None;let mut null_aware__ = None;+ let mut dynamic_filter__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Left => {@@ -9023,6 +9052,12 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {}null_aware__ = Some(map_.next_value()?);}+ GeneratedField::DynamicFilter => {+ if dynamic_filter__.is_some() {+ return Err(serde::de::Error::duplicate_field("dynamicFilter"));+ }+ dynamic_filter__ = map_.next_value()?;+ }}}Ok(HashJoinExecNode {@@ -9035,6 +9070,7 @@ impl<'de> serde::Deserialize<'de> for HashJoinExecNode {filter: filter__,projection: projection__.unwrap_or_default(),null_aware: null_aware__.unwrap_or_default(),+ dynamic_filter: dynamic_filter__,})}}@@ -22561,6 +22597,9 @@ impl serde::Serialize for SortExecNode {if self.preserve_partitioning {len += 1;}+ if self.dynamic_filter.is_some() {+ len += 1;+ }let mut struct_ser = serializer.serialize_struct("datafusion.SortExecNode", len)?;if let Some(v) = self.input.as_ref() {struct_ser.serialize_field("input", v)?;@@ -22576,6 +22615,9 @@ impl serde::Serialize for SortExecNode {if self.preserve_partitioning {struct_ser.serialize_field("preservePartitioning", &self.preserve_partitioning)?;}+ if let Some(v) = self.dynamic_filter.as_ref() {+ struct_ser.serialize_field("dynamicFilter", v)?;+ }struct_ser.end()}}@@ -22591,6 +22633,8 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {"fetch","preserve_partitioning","preservePartitioning",+ "dynamic_filter",+ "dynamicFilter",];#[allow(clippy::enum_variant_names)]@@ -22599,6 +22643,7 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {Expr,Fetch,PreservePartitioning,+ DynamicFilter,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -22624,6 +22669,7 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {"expr" => Ok(GeneratedField::Expr),"fetch" => Ok(GeneratedField::Fetch),"preservePartitioning" | "preserve_partitioning" => Ok(GeneratedField::PreservePartitioning),+ "dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -22647,6 +22693,7 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {let mut expr__ = None;let mut fetch__ = None;let mut preserve_partitioning__ = None;+ let mut dynamic_filter__ = None;while let Some(k) = map_.next_key()? {match k {GeneratedField::Input => {@@ -22675,6 +22722,12 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {}preserve_partitioning__ = Some(map_.next_value()?);}+ GeneratedField::DynamicFilter => {+ if dynamic_filter__.is_some() {+ return Err(serde::de::Error::duplicate_field("dynamicFilter"));+ }+ dynamic_filter__ = map_.next_value()?;+ }}}Ok(SortExecNode {@@ -22682,6 +22735,7 @@ impl<'de> serde::Deserialize<'de> for SortExecNode {expr: expr__.unwrap_or_default(),fetch: fetch__.unwrap_or_default(),preserve_partitioning: preserve_partitioning__.unwrap_or_default(),+ dynamic_filter: dynamic_filter__,})}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex b742e82ea2..24bea5cae9 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1774,6 +1774,9 @@ pub struct HashJoinExecNode {pub projection: ::prost::alloc::vec::Vec<u32>,#[prost(bool, tag = "10")]pub null_aware: bool,+ /// Optional dynamic filter expression for pushing down to the probe side.+ #[prost(message, optional, tag = "11")]+ pub dynamic_filter: ::core::option::Option<PhysicalExprNode>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct SymmetricHashJoinExecNode {@@ -1953,6 +1956,9 @@ pub struct AggregateExecNode {pub limit: ::core::option::Option<AggLimit>,#[prost(bool, tag = "12")]pub has_grouping_set: bool,+ /// Optional dynamic filter expression for pushing down to the child.+ #[prost(message, optional, tag = "13")]+ pub dynamic_filter: ::core::option::Option<PhysicalExprNode>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct GlobalLimitExecNode {@@ -1983,6 +1989,9 @@ pub struct SortExecNode {pub fetch: i64,#[prost(bool, tag = "4")]pub preserve_partitioning: bool,+ /// Optional dynamic filter expression for TopK pushdown.+ #[prost(message, optional, tag = "5")]+ pub dynamic_filter: ::core::option::Option<PhysicalExprNode>,}#[derive(Clone, PartialEq, ::prost::Message)]pub struct SortPreservingMergeExecNode {diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex 68a0b2a456..9a54891773 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -57,6 +57,7 @@ use datafusion_functions_table::generate_series::{};use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};use datafusion_physical_expr::async_scalar_function::AsyncFuncExpr;+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;use datafusion_physical_expr::{LexOrdering, LexRequirement, PhysicalExprRef};use datafusion_physical_plan::aggregates::{AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,@@ -1347,6 +1348,7 @@ impl protobuf::PhysicalPlanNode {}).collect::<Result<Vec<_>, _>>()?;+ let physical_schema_ref = Arc::clone(&physical_schema);let agg = AggregateExec::try_new(agg_mode,PhysicalGroupBy::new(group_expr, null_expr, groups, has_grouping_set),@@ -1367,6 +1369,24 @@ impl protobuf::PhysicalPlanNode {agg};+ let agg = if let Some(dynamic_filter_proto) = &hash_agg.dynamic_filter {+ let dynamic_filter_expr = proto_converter.proto_to_physical_expr(+ dynamic_filter_proto,+ physical_schema_ref.as_ref(),+ ctx,+ )?;+ let df = (dynamic_filter_expr as Arc<dyn Any + Send + Sync>)+ .downcast::<DynamicFilterPhysicalExpr>()+ .map_err(|_| {+ internal_datafusion_err!(+ "AggregateExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"+ )+ })?;+ agg.with_dynamic_filter_expr(df)?+ } else {+ agg+ };+Ok(Arc::new(agg))}@@ -1473,7 +1493,7 @@ impl protobuf::PhysicalPlanNode {} else {None};- Ok(Arc::new(HashJoinExec::try_new(+ let mut hash_join = HashJoinExec::try_new(left,right,on,@@ -1483,7 +1503,25 @@ impl protobuf::PhysicalPlanNode {partition_mode,null_equality.into(),hashjoin.null_aware,- )?))+ )?;++ if let Some(dynamic_filter_proto) = &hashjoin.dynamic_filter {+ let dynamic_filter_expr = proto_converter.proto_to_physical_expr(+ dynamic_filter_proto,+ right_schema.as_ref(),+ ctx,+ )?;+ let df = (dynamic_filter_expr as Arc<dyn Any + Send + Sync>)+ .downcast::<DynamicFilterPhysicalExpr>()+ .map_err(|_| {+ internal_datafusion_err!(+ "HashJoinExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"+ )+ })?;+ hash_join = hash_join.with_dynamic_filter_expr(df)?;+ }++ Ok(Arc::new(hash_join))}fn try_into_symmetric_hash_join_physical_plan(@@ -1721,6 +1759,24 @@ impl protobuf::PhysicalPlanNode {.with_fetch(fetch).with_preserve_partitioning(sort.preserve_partitioning);+ let new_sort = if let Some(dynamic_filter_proto) = &sort.dynamic_filter {+ let dynamic_filter_expr = proto_converter.proto_to_physical_expr(+ dynamic_filter_proto,+ new_sort.input().schema().as_ref(),+ ctx,+ )?;+ let df = (dynamic_filter_expr as Arc<dyn Any + Send + Sync>)+ .downcast::<DynamicFilterPhysicalExpr>()+ .map_err(|_| {+ internal_datafusion_err!(+ "SortExec dynamic_filter did not decode to a DynamicFilterPhysicalExpr"+ )+ })?;+ new_sort.with_dynamic_filter_expr(df)?+ } else {+ new_sort+ };+Ok(Arc::new(new_sort))}@@ -2527,6 +2583,15 @@ impl protobuf::PhysicalPlanNode {PartitionMode::Auto => protobuf::PartitionMode::Auto,};+ let dynamic_filter = exec+ .dynamic_filter_expr()+ .map(|df| {+ let df_expr: Arc<dyn PhysicalExpr> =+ Arc::clone(df) as Arc<dyn PhysicalExpr>;+ proto_converter.physical_expr_to_proto(&df_expr, codec)+ })+ .transpose()?;+Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::HashJoin(Box::new(protobuf::HashJoinExecNode {@@ -2541,6 +2606,7 @@ impl protobuf::PhysicalPlanNode {v.iter().map(|x| *x as u32).collect::<Vec<u32>>()}),null_aware: exec.null_aware,+ dynamic_filter,},))),})@@ -2870,6 +2936,14 @@ impl protobuf::PhysicalPlanNode {groups,limit,has_grouping_set: exec.group_expr().has_grouping_set(),+ dynamic_filter: exec+ .dynamic_filter_expr()+ .map(|df| {+ let df_expr: Arc<dyn PhysicalExpr> =+ Arc::clone(df) as Arc<dyn PhysicalExpr>;+ proto_converter.physical_expr_to_proto(&df_expr, codec)+ })+ .transpose()?,},))),})@@ -3163,6 +3237,14 @@ impl protobuf::PhysicalPlanNode {})}).collect::<Result<Vec<_>>>()?;+ let dynamic_filter = exec+ .dynamic_filter_expr()+ .map(|df| {+ let df_expr: Arc<dyn PhysicalExpr> = df as Arc<dyn PhysicalExpr>;+ proto_converter.physical_expr_to_proto(&df_expr, codec)+ })+ .transpose()?;+Ok(protobuf::PhysicalPlanNode {physical_plan_type: Some(PhysicalPlanType::Sort(Box::new(protobuf::SortExecNode {@@ -3173,6 +3255,7 @@ impl protobuf::PhysicalPlanNode {_ => -1,},preserve_partitioning: exec.preserve_partitioning(),+ dynamic_filter,},))),})diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex e21587cc62..0cb6068af3 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -54,6 +54,8 @@ use datafusion::physical_expr::window::{SlidingAggregateWindowExpr, StandardWinduse datafusion::physical_expr::{LexOrdering, PhysicalSortRequirement, ScalarFunctionExpr,};+use datafusion::physical_optimizer::PhysicalOptimizerRule;+use datafusion::physical_optimizer::filter_pushdown::FilterPushdown;use datafusion::physical_plan::aggregates::{AggregateExec, AggregateMode, LimitOptions, PhysicalGroupBy,};@@ -100,6 +102,7 @@ use datafusion_common::{internal_datafusion_err, internal_err, not_impl_err,};use datafusion_datasource::TableSchema;+use datafusion_datasource::file::FileSource;use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};use datafusion_expr::dml::InsertOp;use datafusion_expr::{@@ -2971,21 +2974,26 @@ fn assert_dynamic_filter_update_is_visible(Ok(())}+/// Extract the dynamic-filter predicate that was pushed down to the parquet+/// scan at the bottom of the plan tree.+fn parquet_source_predicate(child: &Arc<dyn ExecutionPlan>) -> Arc<dyn PhysicalExpr> {+ let data_source = child+ .downcast_ref::<DataSourceExec>()+ .expect("Child should be DataSourceExec");+ let (_, parquet_source) = data_source+ .downcast_to_file_source::<ParquetSource>()+ .expect("Should be ParquetSource");+ parquet_source+ .filter()+ .expect("ParquetSource should have a predicate after roundtrip")+}+/// Assert that two dynamic filters are equal both structurally (Debug output)/// and by identity (`expression_id`).-///fn assert_dynamic_filters_equal(expected: &Arc<dyn PhysicalExpr>,actual: &Arc<dyn PhysicalExpr>,) {- // TODO: Debug currently omits `expression_id` so the id has to be checked- // separately here. Once plan nodes like `SortExec` / `AggregateExec` /- // `HashJoinExec` serialize their own dynamic filter, Debug can include- // `expression_id`.- //- // See https://github.com/apache/datafusion/issues/20418- assert_eq!(expected.expression_id(), actual.expression_id());-// Structural.let expected_dbg = format!("{expected:?}");let actual_dbg = format!("{actual:?}");@@ -3564,6 +3572,86 @@ fn test_linearization_stops_at_different_op() -> Result<()> {Ok(())}+/// Create a DataSourceExec backed by a ParquetSource that accepts filter pushdown,+/// along with a ConfigOptions that enables all dynamic filter pushdown options.+fn datasource_for_dynamic_filter_pushdown(+ schema: &Arc<Schema>,+) -> (Arc<dyn ExecutionPlan>, ConfigOptions) {+ let mut parquet_options = TableParquetOptions::new();+ parquet_options.global.pushdown_filters = true;+ let source = Arc::new(+ ParquetSource::new(Arc::clone(schema))+ .with_table_parquet_options(parquet_options),+ );+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), source)+ .with_file(PartitionedFile::new("/path/to/file.parquet", 1024))+ .build();++ let mut config = ConfigOptions::default();+ config.execution.parquet.pushdown_filters = true;+ config.optimizer.enable_join_dynamic_filter_pushdown = true;+ config.optimizer.enable_aggregate_dynamic_filter_pushdown = true;+ config.optimizer.enable_topk_dynamic_filter_pushdown = true;++ (DataSourceExec::from_data_source(scan_config), config)+}++/// Test that plan containing a HashJoinExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.+#[test]+fn test_hash_join_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Int64, false)]));++ let left_child = Arc::new(EmptyExec::new(Arc::clone(&schema)));+ let (right_child, config) = datasource_for_dynamic_filter_pushdown(&schema);++ let on: Vec<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> = vec![(+ Arc::new(Column::new("col", 0)),+ Arc::new(Column::new("col", 0)),+ )];++ let hash_join = Arc::new(HashJoinExec::try_new(+ left_child,+ right_child,+ on,+ None,+ &JoinType::Inner,+ None,+ PartitionMode::CollectLeft,+ NullEquality::NullEqualsNothing,+ false,+ )?) as Arc<dyn ExecutionPlan>;++ // Run the optimizer rule for filter pushdown.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(hash_join, &config)?;++ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let deserialized = roundtrip_test_and_return(plan, &ctx, &codec, &converter)?;++ // Extract the deserialized HashJoinExec and its dynamic filter.+ let deserialized_join = deserialized+ .downcast_ref::<HashJoinExec>()+ .expect("Should be HashJoinExec");+ let deserialized_hash_join_df = deserialized_join+ .dynamic_filter_expr()+ .expect("HashJoinExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the probe side's ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_join.right());++ // The HashJoinExec's dynamic filter and the probe side's predicate should+ // refer to the same underlying expression.+ let plan_df: Arc<dyn PhysicalExpr> = deserialized_hash_join_df.clone();+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;++ Ok(())+}+/// returns a SessionContext with an empty `netflow` table registeredfn netflow_context() -> Result<SessionContext> {let ctx = SessionContext::new();@@ -3646,39 +3734,141 @@ async fn roundtrip_issue_18602_complex_filter_decode_recursion() -> Result<()> {roundtrip_test_sql_with_context(sql, &ctx).await}+/// Test that plan containing a AggregateExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.#[test]-fn roundtrip_filter_with_none_projection() -> Result<()> {- let schema = Arc::new(Schema::new(vec![- Field::new("a", DataType::Int32, false),- Field::new("b", DataType::Int32, false),- Field::new("c", DataType::Int32, false),- ]));- let predicate: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::new(Column::new("a", 0)),- Operator::Gt,- lit(ScalarValue::Int32(Some(0))),- ));- let input: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::clone(&schema)));+fn test_aggregate_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- // Case 1: None projection (return all columns)- roundtrip_test(Arc::new(FilterExec::try_new(- Arc::clone(&predicate),- Arc::clone(&input),- )?))?;+ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);- // Case 2: Some(vec![]) — explicitly empty projection- roundtrip_test(Arc::new(- FilterExecBuilder::new(Arc::clone(&predicate), Arc::clone(&input))- .apply_projection(Some(vec![]))?- .build()?,- ))?;+ let agg = Arc::new(AggregateExec::try_new(+ AggregateMode::Partial,+ PhysicalGroupBy::new_single(vec![]),+ vec![+ AggregateExprBuilder::new(+ datafusion::functions_aggregate::min_max::min_udaf(),+ vec![Arc::clone(&col_a)],+ )+ .schema(Arc::clone(&schema))+ .alias("min_a")+ .build()+ .map(Arc::new)?,+ ],+ vec![None],+ child,+ Arc::clone(&schema),+ )?) as Arc<dyn ExecutionPlan>;- // Case 3: Some(vec![2, 0]) — partial projection- roundtrip_test(Arc::new(- FilterExecBuilder::new(Arc::clone(&predicate), Arc::clone(&input))- .apply_projection(Some(vec![2, 0]))?- .build()?,- ))?;+ // Run the optimizer rule for filter pushdown.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(agg, &config)?;++ // Roundtrip with deduplication.+ //+ // Note: We don't use `roundtrip_test_and_return` here because there's a+ // pre-existing issue with PhysicalGroupBy serialization where empty groups+ // `[[]]` become `[]` after roundtrip. This behavior is unrelated to this test.+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&plan),+ &codec,+ &converter,+ )?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Extract the deserialized AggregateExec and its dynamic filter.+ let deserialized_agg = deserialized+ .downcast_ref::<AggregateExec>()+ .expect("Should be AggregateExec");+ let deserialized_agg_df = deserialized_agg+ .dynamic_filter_expr()+ .expect("AggregateExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the child ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_agg.input());++ // The AggregateExec's dynamic filter and the child's predicate should+ // refer to the same underlying expression.+ let plan_df: Arc<dyn PhysicalExpr> = deserialized_agg_df.clone();+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;++ Ok(())+}++/// Test that plan containing a SortExec with dynamic filter pushdown+/// can be serialized and deserialized while preserving references to the dynamic filter.+#[test]+fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let col_a: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));++ let (child, config) = datasource_for_dynamic_filter_pushdown(&schema);++ let sort = Arc::new(+ SortExec::new(+ LexOrdering::new(vec![PhysicalSortExpr {+ expr: Arc::clone(&col_a),+ options: SortOptions::default(),+ }])+ .unwrap(),+ child,+ )+ .with_fetch(Some(10)),+ ) as Arc<dyn ExecutionPlan>;++ // Verify the optimizer kept the dynamic filter on the SortExec.+ let optimizer = FilterPushdown::new_post_optimization();+ let plan = optimizer.optimize(sort, &config)?;++ // Roundtrip with deduplication.+ //+ // Note: We don't use `roundtrip_test_and_return` here because+ // `DeduplicatingDeserializer` rewrites cache hits via `with_new_children`,+ // which sets `remapped_children: Some(...)` on the second encounter of a+ // shared `DynamicFilterPhysicalExpr`. SortExec's `Debug` includes its+ // dynamic filter, so the original-vs-deserialized structural equality check+ // would fail purely on this artifact.+ let ctx = SessionContext::new();+ let codec = DefaultPhysicalExtensionCodec {};+ let converter = DeduplicatingProtoConverter {};+ let bytes = physical_plan_to_bytes_with_proto_converter(+ Arc::clone(&plan),+ &codec,+ &converter,+ )?;+ let deserialized = physical_plan_from_bytes_with_proto_converter(+ bytes.as_ref(),+ ctx.task_ctx().as_ref(),+ &codec,+ &converter,+ )?;++ // Extract the deserialized SortExec and its dynamic filter.+ let deserialized_sort = deserialized+ .downcast_ref::<SortExec>()+ .expect("Should be SortExec");+ let deserialized_sort_df = deserialized_sort+ .dynamic_filter_expr()+ .expect("SortExec should have a dynamic filter after roundtrip");++ // Extract the dynamic filter pushed down to the child ParquetSource.+ let deserialized_predicate = parquet_source_predicate(deserialized_sort.input());++ // The SortExec's dynamic filter and the child's predicate should+ // refer to the same underlying expression.+ let plan_df: Arc<dyn PhysicalExpr> = deserialized_sort_df;+ assert_dynamic_filters_equal(&plan_df, &deserialized_predicate);+ assert_dynamic_filter_update_is_visible(&plan_df, &deserialized_predicate)?;Ok(())}