Apache DataFusion SQL Query Engine
refactor: add `try_to_proto` / `try_from_proto` to `DynamicFilterPhysicalExpr` (#22452)
Anurag Tryambak Rautf911d529a57b
diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rsindex 9fe3feb586..0669913c32 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs@@ -23,7 +23,7 @@ use tokio::sync::watch;use crate::PhysicalExpr;use arrow::datatypes::{DataType, Schema};use datafusion_common::{- Result,+ Result, internal_datafusion_err,tree_node::{Transformed, TransformedResult, TreeNode},};use datafusion_expr::ColumnarValue;@@ -548,6 +548,111 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {fn expression_id(&self) -> Option<u64> {Some(self.inner.read().expression_id)}+ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;+ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;++ let children = self+ .children+ .iter()+ .map(|c| ctx.encode_child(c))+ .collect::<Result<Vec<_>>>()?;++ let remapped_children = match &self.remapped_children {+ Some(remapped) => remapped+ .iter()+ .map(|c| ctx.encode_child(c))+ .collect::<Result<Vec<_>>>()?,+ None => vec![],+ };++ let inner = self.inner.read().clone();+ let inner_expr = Box::new(ctx.encode_child(&inner.expr)?);++ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: Some(inner.expression_id),+ expr_type: Some(ExprType::DynamicFilter(Box::new(+ protobuf::PhysicalDynamicFilterNode {+ children,+ remapped_children,+ generation: inner.generation,+ inner_expr: Some(inner_expr),+ is_complete: inner.is_complete,+ },+ ))),+ }))+ }+}++#[cfg(feature = "proto")]+impl DynamicFilterPhysicalExpr {+ /// Reconstruct a [`DynamicFilterPhysicalExpr`] from a proto node.+ ///+ /// Called by the `ExprType::DynamicFilter` arm in `datafusion-proto`'s+ /// `parse_physical_expr_with_converter`. Follows the same+ /// `PhysicalExprDecodeCtx`-based pattern used by `Column`, `BinaryExpr`, etc.+ pub fn try_from_proto(+ proto: &datafusion_proto_models::protobuf::PhysicalExprNode,+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,+ ) -> Result<Arc<dyn PhysicalExpr>> {+ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;++ let ExprType::DynamicFilter(df) = proto.expr_type.as_ref().ok_or_else(|| {+ internal_datafusion_err!("Missing expr_type in PhysicalExprNode")+ })?+ else {+ return Err(internal_datafusion_err!("Expected DynamicFilter expr_type"));+ };++ // Decode original children+ let children = df+ .children+ .iter()+ .map(|c| ctx.decode(c))+ .collect::<Result<Vec<_>>>()?;++ // Decode remapped children (empty vec means None)+ let remapped_children = if df.remapped_children.is_empty() {+ None+ } else {+ Some(+ df.remapped_children+ .iter()+ .map(|c| ctx.decode(c))+ .collect::<Result<Vec<_>>>()?,+ )+ };++ // Decode the inner expression+ let inner_expr_proto = df.inner_expr.as_ref().ok_or_else(|| {+ internal_datafusion_err!("Missing inner_expr in PhysicalDynamicFilterNode")+ })?;+ let inner_expr = ctx.decode(inner_expr_proto)?;++ // Restore the expression_id from the outer PhysicalExprNode+ let expression_id = proto.expr_id.ok_or_else(|| {+ internal_datafusion_err!(+ "Missing expr_id in PhysicalExprNode for DynamicFilter"+ )+ })?;++ let inner = Inner {+ expression_id,+ generation: df.generation,+ expr: inner_expr,+ is_complete: df.is_complete,+ };++ Ok(Arc::new(Self::from_parts(+ children,+ remapped_children,+ inner,+ )))+ }}/// The result of polling a [`DynamicFilterSubscription`].diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 36751d8a61..7bd1a3dd66 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -64,9 +64,7 @@ use super::{use crate::convert::TryFromProto;use crate::protobuf::physical_expr_node::ExprType;use crate::{convert_required, convert_required_proto, protobuf};-use datafusion_physical_expr::expressions::{- DynamicFilterInner, DynamicFilterPhysicalExpr,-};+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;/// Parses a physical sort expression from a protobuf.///@@ -359,52 +357,8 @@ pub fn parse_physical_expr_with_converter(results.clone(),))}- ExprType::DynamicFilter(dynamic_filter) => {- let children = parse_physical_exprs(- &dynamic_filter.children,- ctx,- input_schema,- proto_converter,- )?;-- let remapped_children = if !dynamic_filter.remapped_children.is_empty() {- Some(parse_physical_exprs(- &dynamic_filter.remapped_children,- ctx,- input_schema,- proto_converter,- )?)- } else {- None- };-- let inner_expr = parse_required_physical_expr(- dynamic_filter.inner_expr.as_deref(),- ctx,- "inner_expr",- input_schema,- proto_converter,- )?;-- let expression_id = proto.expr_id.ok_or_else(|| {- proto_error(- "DynamicFilterPhysicalExpr requires PhysicalExprNode.expr_id \- to be set by the serializer",- )- })?;-- let base_filter: Arc<dyn PhysicalExpr> =- Arc::new(DynamicFilterPhysicalExpr::from_parts(- children,- remapped_children,- DynamicFilterInner {- expression_id,- generation: dynamic_filter.generation,- expr: inner_expr,- is_complete: dynamic_filter.is_complete,- },- ));- base_filter+ ExprType::DynamicFilter(_) => {+ DynamicFilterPhysicalExpr::try_from_proto(proto, &decode_ctx)?}ExprType::Extension(extension) => {let inputs: Vec<Arc<dyn PhysicalExpr>> = extension@@ -420,18 +374,6 @@ pub fn parse_physical_expr_with_converter(Ok(pexpr)}-fn parse_required_physical_expr(- expr: Option<&protobuf::PhysicalExprNode>,- ctx: &PhysicalPlanDecodeContext<'_>,- field: &str,- input_schema: &Schema,- proto_converter: &dyn PhysicalProtoConverterExtension,-) -> Result<Arc<dyn PhysicalExpr>> {- expr.map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx))- .transpose()?- .ok_or_else(|| internal_datafusion_err!("Missing required field {field:?}"))-}-pub fn parse_protobuf_hash_partitioning(partitioning: Option<&protobuf::PhysicalHashRepartition>,ctx: &PhysicalPlanDecodeContext<'_>,diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex d9315af431..7310c0928e 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -35,7 +35,6 @@ use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;-use datafusion_physical_plan::expressions::DynamicFilterPhysicalExpr;use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};use datafusion_physical_plan::{@@ -330,39 +329,6 @@ pub fn serialize_physical_expr_with_converter(},)),})- } else if let Some(df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {- let children = df- .original_children()- .iter()- .map(|child| proto_converter.physical_expr_to_proto(child, codec))- .collect::<Result<Vec<_>>>()?;-- let remapped_children = if let Some(remapped) = df.remapped_children() {- remapped- .iter()- .map(|child| proto_converter.physical_expr_to_proto(child, codec))- .collect::<Result<Vec<_>>>()?- } else {- vec![]- };-- // Atomic snapshot of inner state.- let inner = df.inner();- let inner_expr =- Box::new(proto_converter.physical_expr_to_proto(&inner.expr, codec)?);-- Ok(protobuf::PhysicalExprNode {- expr_id,- expr_type: Some(protobuf::physical_expr_node::ExprType::DynamicFilter(- Box::new(protobuf::PhysicalDynamicFilterNode {- children,- remapped_children,- generation: inner.generation,- inner_expr: Some(inner_expr),- is_complete: inner.is_complete,- }),- )),- })} else {let mut buf: Vec<u8> = vec![];match codec.try_encode_expr(value, &mut buf) {