Skip to content

Apache DataFusion SQL Query Engine

refactor: add `try_to_proto` / `try_from_proto` to `DynamicFilterPhysicalExpr` (#22452)

Anurag Tryambak Rautf911d529a57b
diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
index 9fe3feb586..0669913c32 100644
--- a/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
+++ b/datafusion/physical-expr/src/expressions/dynamic_filters/mod.rs
@@ -23,7 +23,7 @@ use tokio::sync::watch;
use crate::PhysicalExpr;
use arrow::datatypes::{DataType, Schema};
use datafusion_common::{
- Result,
+ Result, internal_datafusion_err,
tree_node::{Transformed, TransformedResult, TreeNode},
};
use datafusion_expr::ColumnarValue;
@@ -548,6 +548,111 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {
fn expression_id(&self) -> Option<u64> {
Some(self.inner.read().expression_id)
}
+ #[cfg(feature = "proto")]
+ fn try_to_proto(
+ &self,
+ ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,
+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {
+ use datafusion_proto_models::protobuf;
+ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
+
+ let children = self
+ .children
+ .iter()
+ .map(|c| ctx.encode_child(c))
+ .collect::<Result<Vec<_>>>()?;
+
+ let remapped_children = match &self.remapped_children {
+ Some(remapped) => remapped
+ .iter()
+ .map(|c| ctx.encode_child(c))
+ .collect::<Result<Vec<_>>>()?,
+ None => vec![],
+ };
+
+ let inner = self.inner.read().clone();
+ let inner_expr = Box::new(ctx.encode_child(&inner.expr)?);
+
+ Ok(Some(protobuf::PhysicalExprNode {
+ expr_id: Some(inner.expression_id),
+ expr_type: Some(ExprType::DynamicFilter(Box::new(
+ protobuf::PhysicalDynamicFilterNode {
+ children,
+ remapped_children,
+ generation: inner.generation,
+ inner_expr: Some(inner_expr),
+ is_complete: inner.is_complete,
+ },
+ ))),
+ }))
+ }
+}
+
+#[cfg(feature = "proto")]
+impl DynamicFilterPhysicalExpr {
+ /// Reconstruct a [`DynamicFilterPhysicalExpr`] from a proto node.
+ ///
+ /// Called by the `ExprType::DynamicFilter` arm in `datafusion-proto`'s
+ /// `parse_physical_expr_with_converter`. Follows the same
+ /// `PhysicalExprDecodeCtx`-based pattern used by `Column`, `BinaryExpr`, etc.
+ pub fn try_from_proto(
+ proto: &datafusion_proto_models::protobuf::PhysicalExprNode,
+ ctx: &datafusion_physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx<'_>,
+ ) -> Result<Arc<dyn PhysicalExpr>> {
+ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;
+
+ let ExprType::DynamicFilter(df) = proto.expr_type.as_ref().ok_or_else(|| {
+ internal_datafusion_err!("Missing expr_type in PhysicalExprNode")
+ })?
+ else {
+ return Err(internal_datafusion_err!("Expected DynamicFilter expr_type"));
+ };
+
+ // Decode original children
+ let children = df
+ .children
+ .iter()
+ .map(|c| ctx.decode(c))
+ .collect::<Result<Vec<_>>>()?;
+
+ // Decode remapped children (empty vec means None)
+ let remapped_children = if df.remapped_children.is_empty() {
+ None
+ } else {
+ Some(
+ df.remapped_children
+ .iter()
+ .map(|c| ctx.decode(c))
+ .collect::<Result<Vec<_>>>()?,
+ )
+ };
+
+ // Decode the inner expression
+ let inner_expr_proto = df.inner_expr.as_ref().ok_or_else(|| {
+ internal_datafusion_err!("Missing inner_expr in PhysicalDynamicFilterNode")
+ })?;
+ let inner_expr = ctx.decode(inner_expr_proto)?;
+
+ // Restore the expression_id from the outer PhysicalExprNode
+ let expression_id = proto.expr_id.ok_or_else(|| {
+ internal_datafusion_err!(
+ "Missing expr_id in PhysicalExprNode for DynamicFilter"
+ )
+ })?;
+
+ let inner = Inner {
+ expression_id,
+ generation: df.generation,
+ expr: inner_expr,
+ is_complete: df.is_complete,
+ };
+
+ Ok(Arc::new(Self::from_parts(
+ children,
+ remapped_children,
+ inner,
+ )))
+ }
}
/// The result of polling a [`DynamicFilterSubscription`].
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index 36751d8a61..7bd1a3dd66 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -64,9 +64,7 @@ use super::{
use crate::convert::TryFromProto;
use crate::protobuf::physical_expr_node::ExprType;
use crate::{convert_required, convert_required_proto, protobuf};
-use datafusion_physical_expr::expressions::{
- DynamicFilterInner, DynamicFilterPhysicalExpr,
-};
+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
/// Parses a physical sort expression from a protobuf.
///
@@ -359,52 +357,8 @@ pub fn parse_physical_expr_with_converter(
results.clone(),
))
}
- ExprType::DynamicFilter(dynamic_filter) => {
- let children = parse_physical_exprs(
- &dynamic_filter.children,
- ctx,
- input_schema,
- proto_converter,
- )?;
-
- let remapped_children = if !dynamic_filter.remapped_children.is_empty() {
- Some(parse_physical_exprs(
- &dynamic_filter.remapped_children,
- ctx,
- input_schema,
- proto_converter,
- )?)
- } else {
- None
- };
-
- let inner_expr = parse_required_physical_expr(
- dynamic_filter.inner_expr.as_deref(),
- ctx,
- "inner_expr",
- input_schema,
- proto_converter,
- )?;
-
- let expression_id = proto.expr_id.ok_or_else(|| {
- proto_error(
- "DynamicFilterPhysicalExpr requires PhysicalExprNode.expr_id \
- to be set by the serializer",
- )
- })?;
-
- let base_filter: Arc<dyn PhysicalExpr> =
- Arc::new(DynamicFilterPhysicalExpr::from_parts(
- children,
- remapped_children,
- DynamicFilterInner {
- expression_id,
- generation: dynamic_filter.generation,
- expr: inner_expr,
- is_complete: dynamic_filter.is_complete,
- },
- ));
- base_filter
+ ExprType::DynamicFilter(_) => {
+ DynamicFilterPhysicalExpr::try_from_proto(proto, &decode_ctx)?
}
ExprType::Extension(extension) => {
let inputs: Vec<Arc<dyn PhysicalExpr>> = extension
@@ -420,18 +374,6 @@ pub fn parse_physical_expr_with_converter(
Ok(pexpr)
}
-fn parse_required_physical_expr(
- expr: Option<&protobuf::PhysicalExprNode>,
- ctx: &PhysicalPlanDecodeContext<'_>,
- field: &str,
- input_schema: &Schema,
- proto_converter: &dyn PhysicalProtoConverterExtension,
-) -> Result<Arc<dyn PhysicalExpr>> {
- expr.map(|e| proto_converter.proto_to_physical_expr(e, input_schema, ctx))
- .transpose()?
- .ok_or_else(|| internal_datafusion_err!("Missing required field {field:?}"))
-}
-
pub fn parse_protobuf_hash_partitioning(
partitioning: Option<&protobuf::PhysicalHashRepartition>,
ctx: &PhysicalPlanDecodeContext<'_>,
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index d9315af431..7310c0928e 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -35,7 +35,6 @@ use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
-use datafusion_physical_plan::expressions::DynamicFilterPhysicalExpr;
use datafusion_physical_plan::udaf::AggregateFunctionExpr;
use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};
use datafusion_physical_plan::{
@@ -330,39 +329,6 @@ pub fn serialize_physical_expr_with_converter(
},
)),
})
- } else if let Some(df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {
- let children = df
- .original_children()
- .iter()
- .map(|child| proto_converter.physical_expr_to_proto(child, codec))
- .collect::<Result<Vec<_>>>()?;
-
- let remapped_children = if let Some(remapped) = df.remapped_children() {
- remapped
- .iter()
- .map(|child| proto_converter.physical_expr_to_proto(child, codec))
- .collect::<Result<Vec<_>>>()?
- } else {
- vec![]
- };
-
- // Atomic snapshot of inner state.
- let inner = df.inner();
- let inner_expr =
- Box::new(proto_converter.physical_expr_to_proto(&inner.expr, codec)?);
-
- Ok(protobuf::PhysicalExprNode {
- expr_id,
- expr_type: Some(protobuf::physical_expr_node::ExprType::DynamicFilter(
- Box::new(protobuf::PhysicalDynamicFilterNode {
- children,
- remapped_children,
- generation: inner.generation,
- inner_expr: Some(inner_expr),
- is_complete: inner.is_complete,
- }),
- )),
- })
} else {
let mut buf: Vec<u8> = vec![];
match codec.try_encode_expr(value, &mut buf) {