Skip to content

Apache DataFusion SQL Query Engine

proto: add proto converter reference to PhysicalExtensionCodec trait (#21055)

Jayant Shrivastava50d74a704d7f
diff --git a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
index d82bd2097c..a956348279 100644
--- a/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
+++ b/datafusion-examples/examples/custom_data_source/adapter_serialization.rs
@@ -275,6 +275,7 @@ impl PhysicalExtensionCodec for AdapterPreservingCodec {
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
// Try to parse as our extension payload
if let Ok(payload) = serde_json::from_slice::<ExtensionPayload>(buf)
@@ -303,6 +304,7 @@ impl PhysicalExtensionCodec for AdapterPreservingCodec {
&self,
_node: Arc<dyn ExecutionPlan>,
_buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
// We don't need this for the example - adapter wrapping happens in
// `execution_plan_to_proto` instead.
diff --git a/datafusion-examples/examples/proto/composed_extension_codec.rs b/datafusion-examples/examples/proto/composed_extension_codec.rs
index ae9503dd87..b76e0e70e9 100644
--- a/datafusion-examples/examples/proto/composed_extension_codec.rs
+++ b/datafusion-examples/examples/proto/composed_extension_codec.rs
@@ -43,6 +43,7 @@ use datafusion::physical_plan::{DisplayAs, ExecutionPlan};
use datafusion::prelude::SessionContext;
use datafusion_proto::physical_plan::{
AsExecutionPlan, ComposedPhysicalExtensionCodec, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
};
use datafusion_proto::protobuf;
@@ -145,6 +146,7 @@ impl PhysicalExtensionCodec for ParentPhysicalExtensionCodec {
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
if buf == "ParentExec".as_bytes() {
Ok(Arc::new(ParentExec {
@@ -155,7 +157,12 @@ impl PhysicalExtensionCodec for ParentPhysicalExtensionCodec {
}
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
if node.is::<ParentExec>() {
buf.extend_from_slice("ParentExec".as_bytes());
Ok(())
@@ -226,6 +233,7 @@ impl PhysicalExtensionCodec for ChildPhysicalExtensionCodec {
buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
if buf == "ChildExec".as_bytes() {
Ok(Arc::new(ChildExec {}))
@@ -234,7 +242,12 @@ impl PhysicalExtensionCodec for ChildPhysicalExtensionCodec {
}
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
if node.is::<ChildExec>() {
buf.extend_from_slice("ChildExec".as_bytes());
Ok(())
diff --git a/datafusion-examples/examples/proto/expression_deduplication.rs b/datafusion-examples/examples/proto/expression_deduplication.rs
index 26d246b2ef..ae4a76e79f 100644
--- a/datafusion-examples/examples/proto/expression_deduplication.rs
+++ b/datafusion-examples/examples/proto/expression_deduplication.rs
@@ -187,6 +187,7 @@ impl PhysicalExtensionCodec for CachingCodec {
_buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
datafusion::common::not_impl_err!("No custom extension nodes")
}
@@ -196,6 +197,7 @@ impl PhysicalExtensionCodec for CachingCodec {
&self,
_node: Arc<dyn ExecutionPlan>,
_buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
datafusion::common::not_impl_err!("No custom extension nodes")
}
diff --git a/datafusion/ffi/src/proto/physical_extension_codec.rs b/datafusion/ffi/src/proto/physical_extension_codec.rs
index 60d9d03dbd..9e64df82e3 100644
--- a/datafusion/ffi/src/proto/physical_extension_codec.rs
+++ b/datafusion/ffi/src/proto/physical_extension_codec.rs
@@ -25,7 +25,10 @@ use datafusion_expr::{
AggregateUDF, AggregateUDFImpl, ScalarUDF, ScalarUDFImpl, WindowUDF, WindowUDFImpl,
};
use datafusion_physical_plan::ExecutionPlan;
-use datafusion_proto::physical_plan::PhysicalExtensionCodec;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+};
use stabby::slice::Slice as SSlice;
use stabby::str::Str as SStr;
@@ -145,8 +148,12 @@ unsafe extern "C" fn try_decode_fn_wrapper(
.collect::<Result<Vec<_>>>();
let inputs = sresult_return!(inputs);
- let plan =
- sresult_return!(codec.try_decode(buf.as_ref(), &inputs, task_ctx.as_ref()));
+ let plan = sresult_return!(codec.try_decode(
+ buf.as_ref(),
+ &inputs,
+ task_ctx.as_ref(),
+ &DefaultPhysicalProtoConverter {},
+ ));
FFI_Result::Ok(FFI_ExecutionPlan::new(plan, runtime))
}
@@ -160,7 +167,11 @@ unsafe extern "C" fn try_encode_fn_wrapper(
let plan: Arc<dyn ExecutionPlan> = sresult_return!((&node).try_into());
let mut bytes = Vec::new();
- sresult_return!(codec.try_encode(plan, &mut bytes));
+ sresult_return!(codec.try_encode(
+ plan,
+ &mut bytes,
+ &DefaultPhysicalProtoConverter {}
+ ));
FFI_Result::Ok(bytes.into_iter().collect())
}
@@ -335,6 +346,7 @@ impl PhysicalExtensionCodec for ForeignPhysicalExtensionCodec {
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
let inputs = inputs
.iter()
@@ -348,7 +360,12 @@ impl PhysicalExtensionCodec for ForeignPhysicalExtensionCodec {
Ok(plan)
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
let plan = FFI_ExecutionPlan::new(node, None);
let bytes = df_result!(unsafe { (self.0.try_encode)(&self.0, plan) })?;
@@ -426,7 +443,10 @@ pub(crate) mod tests {
use datafusion_functions_aggregate::sum::Sum;
use datafusion_functions_window::rank::{Rank, RankType};
use datafusion_physical_plan::ExecutionPlan;
- use datafusion_proto::physical_plan::PhysicalExtensionCodec;
+ use datafusion_proto::physical_plan::{
+ DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
+ PhysicalProtoConverterExtension,
+ };
use crate::execution_plan::tests::EmptyExec;
use crate::proto::physical_extension_codec::FFI_PhysicalExtensionCodec;
@@ -449,6 +469,7 @@ pub(crate) mod tests {
buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
if buf[0] != Self::MAGIC_NUMBER {
return exec_err!(
@@ -467,6 +488,7 @@ pub(crate) mod tests {
&self,
node: Arc<dyn ExecutionPlan>,
buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
buf.push(Self::MAGIC_NUMBER);
@@ -587,10 +609,18 @@ pub(crate) mod tests {
let exec = create_test_exec();
let input_execs = [create_test_exec()];
let mut bytes = Vec::new();
- foreign_codec.try_encode(Arc::clone(&exec), &mut bytes)?;
-
- let returned_exec =
- foreign_codec.try_decode(&bytes, &input_execs, ctx.task_ctx().as_ref())?;
+ foreign_codec.try_encode(
+ Arc::clone(&exec),
+ &mut bytes,
+ &DefaultPhysicalProtoConverter {},
+ )?;
+
+ let returned_exec = foreign_codec.try_decode(
+ &bytes,
+ &input_execs,
+ ctx.task_ctx().as_ref(),
+ &DefaultPhysicalProtoConverter {},
+ )?;
assert!(returned_exec.is::<EmptyExec>());
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index 9a54891773..27284664b0 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -669,7 +669,7 @@ impl protobuf::PhysicalPlanNode {
}
let mut buf: Vec<u8> = vec![];
- match codec.try_encode(Arc::clone(&plan_clone), &mut buf) {
+ match codec.try_encode(Arc::clone(&plan_clone), &mut buf, proto_converter) {
Ok(_) => {
let inputs: Vec<protobuf::PhysicalPlanNode> = plan_clone
.children()
@@ -1843,9 +1843,12 @@ impl protobuf::PhysicalPlanNode {
.map(|i| proto_converter.proto_to_execution_plan(i, ctx))
.collect::<Result<_>>()?;
- let extension_node =
- ctx.codec()
- .try_decode(extension.node.as_slice(), &inputs, ctx.task_ctx())?;
+ let extension_node = ctx.codec().try_decode(
+ extension.node.as_slice(),
+ &inputs,
+ ctx.task_ctx(),
+ proto_converter,
+ )?;
Ok(extension_node)
}
@@ -3852,9 +3855,15 @@ pub trait PhysicalExtensionCodec: Debug + Send + Sync + Any {
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>>;
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()>;
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()>;
fn try_decode_udf(&self, name: &str, _buf: &[u8]) -> Result<Arc<ScalarUDF>> {
not_impl_err!("PhysicalExtensionCodec is not provided for scalar function {name}")
@@ -3908,6 +3917,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
_buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
not_impl_err!("PhysicalExtensionCodec is not provided")
}
@@ -3916,6 +3926,7 @@ impl PhysicalExtensionCodec for DefaultPhysicalExtensionCodec {
&self,
_node: Arc<dyn ExecutionPlan>,
_buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
not_impl_err!("PhysicalExtensionCodec is not provided")
}
@@ -4239,12 +4250,22 @@ impl PhysicalExtensionCodec for ComposedPhysicalExtensionCodec {
buf: &[u8],
inputs: &[Arc<dyn ExecutionPlan>],
ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
- self.decode_protobuf(buf, |codec, data| codec.try_decode(data, inputs, ctx))
+ self.decode_protobuf(buf, |codec, data| {
+ codec.try_decode(data, inputs, ctx, proto_converter)
+ })
}
- fn try_encode(&self, node: Arc<dyn ExecutionPlan>, buf: &mut Vec<u8>) -> Result<()> {
- self.encode_protobuf(buf, |codec, data| codec.try_encode(Arc::clone(&node), data))
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ self.encode_protobuf(buf, |codec, data| {
+ codec.try_encode(Arc::clone(&node), data, proto_converter)
+ })
}
fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result<Arc<ScalarUDF>> {
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index 0cb6068af3..3ea9108800 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -25,7 +25,7 @@ use arrow::csv::WriterBuilder;
use arrow::datatypes::{Fields, TimeUnit};
use datafusion::arrow::array::ArrayRef;
use datafusion::arrow::compute::kernels::sort::SortOptions;
-use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema};
+use datafusion::arrow::datatypes::{DataType, Field, IntervalUnit, Schema, SchemaRef};
use datafusion::datasource::empty::EmptyTable;
use datafusion::datasource::file_format::csv::CsvSink;
use datafusion::datasource::file_format::json::{JsonFormat, JsonSink};
@@ -65,7 +65,8 @@ use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::expressions::{
- BinaryExpr, Column, NotExpr, PhysicalSortExpr, binary, cast, col, in_list, like, lit,
+ BinaryExpr, Column, DynamicFilterPhysicalExpr, NotExpr, PhysicalSortExpr, binary,
+ cast, col, in_list, like, lit,
};
use datafusion::physical_plan::filter::{FilterExec, FilterExecBuilder};
use datafusion::physical_plan::joins::{
@@ -88,7 +89,8 @@ use datafusion::physical_plan::windows::{
create_udwf_window_expr,
};
use datafusion::physical_plan::{
- ExecutionPlan, InputOrderMode, Partitioning, PhysicalExpr, Statistics, displayable,
+ DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,
+ PhysicalExpr, SendableRecordBatchStream, Statistics, displayable,
};
use datafusion::prelude::{ParquetReadOptions, SessionContext};
use datafusion::scalar::ScalarValue;
@@ -97,6 +99,7 @@ use datafusion_common::file_options::csv_writer::CsvWriterOptions;
use datafusion_common::file_options::json_writer::JsonWriterOptions;
use datafusion_common::parsers::CompressionTypeVariant;
use datafusion_common::stats::Precision;
+use datafusion_common::tree_node::TreeNodeRecursion;
use datafusion_common::{
DataFusionError, NullEquality, Result, UnnestOptions, exec_datafusion_err,
internal_datafusion_err, internal_err, not_impl_err,
@@ -136,7 +139,6 @@ use crate::cases::{
CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,
MyRegexUdfNode,
};
-use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
use datafusion_physical_expr::utils::reassign_expr_columns;
/// Perform a serde roundtrip and assert that the string representation of the before and after plans
@@ -1143,6 +1145,7 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
_buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
unreachable!()
}
@@ -1151,6 +1154,7 @@ fn roundtrip_parquet_exec_with_custom_predicate_expr() -> Result<()> {
&self,
_node: Arc<dyn ExecutionPlan>,
_buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
unreachable!()
}
@@ -1252,6 +1256,7 @@ impl PhysicalExtensionCodec for UDFExtensionCodec {
_buf: &[u8],
_inputs: &[Arc<dyn ExecutionPlan>],
_ctx: &TaskContext,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<Arc<dyn ExecutionPlan>> {
not_impl_err!("No extension codec provided")
}
@@ -1260,6 +1265,7 @@ impl PhysicalExtensionCodec for UDFExtensionCodec {
&self,
_node: Arc<dyn ExecutionPlan>,
_buf: &mut Vec<u8>,
+ _proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<()> {
not_impl_err!("No extension codec provided")
}
@@ -3872,3 +3878,206 @@ fn test_sort_topk_with_dynamic_filter_roundtrip() -> Result<()> {
Ok(())
}
+
+/// A custom [`ExecutionPlan`] which stores [`PhysicalExpr`]s.
+struct CustomExecWithExprs {
+ exprs: Vec<Arc<dyn PhysicalExpr>>,
+ child: Arc<dyn ExecutionPlan>,
+}
+
+#[derive(Clone, PartialEq, Message)]
+struct CustomExecWithExprsProto {
+ #[prost(message, repeated, tag = "1")]
+ exprs: Vec<PhysicalExprNode>,
+}
+
+impl std::fmt::Debug for CustomExecWithExprs {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("CustomExecWithExprs")
+ .field("exprs", &self.exprs)
+ .field("child", &self.child)
+ .finish()
+ }
+}
+
+impl CustomExecWithExprs {
+ fn new(exprs: Vec<Arc<dyn PhysicalExpr>>, child: Arc<dyn ExecutionPlan>) -> Self {
+ Self { exprs, child }
+ }
+}
+
+impl DisplayAs for CustomExecWithExprs {
+ fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
+ write!(f, "CustomExecWithExprs")
+ }
+}
+
+impl ExecutionPlan for CustomExecWithExprs {
+ fn name(&self) -> &str {
+ "CustomExecWithExprs"
+ }
+
+ fn schema(&self) -> SchemaRef {
+ self.child.schema()
+ }
+
+ fn properties(&self) -> &Arc<datafusion::physical_plan::PlanProperties> {
+ self.child.properties()
+ }
+
+ fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+ vec![&self.child]
+ }
+
+ fn apply_expressions(
+ &self,
+ f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result<TreeNodeRecursion>,
+ ) -> Result<TreeNodeRecursion> {
+ let mut tnr = TreeNodeRecursion::Continue;
+ for expr in &self.exprs {
+ tnr = tnr.visit_sibling(|| f(expr.as_ref()))?;
+ }
+ Ok(tnr)
+ }
+
+ fn with_new_children(
+ self: Arc<Self>,
+ _children: Vec<Arc<dyn ExecutionPlan>>,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ unreachable!()
+ }
+
+ fn execute(
+ &self,
+ _partition: usize,
+ _context: Arc<TaskContext>,
+ ) -> Result<SendableRecordBatchStream> {
+ unreachable!()
+ }
+}
+
+/// A [`PhysicalExtensionCodec`] for [`CustomExecWithExprs`].
+#[derive(Debug)]
+struct CustomExecWithExprsCodec {}
+
+impl PhysicalExtensionCodec for CustomExecWithExprsCodec {
+ fn try_decode(
+ &self,
+ buf: &[u8],
+ inputs: &[Arc<dyn ExecutionPlan>],
+ ctx: &TaskContext,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<Arc<dyn ExecutionPlan>> {
+ let decode_ctx = PhysicalPlanDecodeContext::new(ctx, self);
+ let input_schema = inputs[0].schema();
+ let proto = CustomExecWithExprsProto::decode(buf)
+ .map_err(|e| internal_datafusion_err!("Failed to decode custom exec: {e}"))?;
+ let exprs = proto
+ .exprs
+ .iter()
+ .map(|expr_proto| {
+ proto_converter.proto_to_physical_expr(
+ expr_proto,
+ input_schema.as_ref(),
+ &decode_ctx,
+ )
+ })
+ .collect::<Result<Vec<_>>>()?;
+
+ Ok(Arc::new(CustomExecWithExprs::new(exprs, inputs[0].clone())))
+ }
+
+ fn try_encode(
+ &self,
+ node: Arc<dyn ExecutionPlan>,
+ buf: &mut Vec<u8>,
+ proto_converter: &dyn PhysicalProtoConverterExtension,
+ ) -> Result<()> {
+ let custom = node
+ .downcast_ref::<CustomExecWithExprs>()
+ .ok_or_else(|| internal_datafusion_err!("Expected CustomExecWithExprs"))?;
+ let proto = CustomExecWithExprsProto {
+ exprs: custom
+ .exprs
+ .iter()
+ .map(|expr| proto_converter.physical_expr_to_proto(expr, self))
+ .collect::<Result<Vec<_>>>()?,
+ };
+ proto
+ .encode(buf)
+ .map_err(|e| internal_datafusion_err!("Failed to encode custom exec: {e}"))?;
+
+ Ok(())
+ }
+}
+
+/// Tests that a custom [`ExecutionPlan`] with [`PhysicalExpr`] can
+/// dedupe dynamic filters by using the proto converter in its
+/// [`PhysicalExtensionCodec`] implementation.
+#[test]
+fn test_custom_node_with_dynamic_filter_dedup_roundtrip() -> Result<()> {
+ // Create the plan:
+ //
+ // FilterExec(dynamic_filter)
+ // -> CustomExecWithExprs(exprs: [dynamic_filter])
+ // -> EmptyExec
+ //
+ // The same dynamic filter expression is saved in both the FilterExec and CustomExecWithExprs.
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
+ lit(true),
+ ));
+ let dynamic_filter_expr: Arc<dyn PhysicalExpr> = dynamic_filter;
+
+ let empty = Arc::new(EmptyExec::new(Arc::clone(&schema)));
+ let custom_exec = Arc::new(CustomExecWithExprs::new(
+ vec![Arc::clone(&dynamic_filter_expr)],
+ empty,
+ ));
+ let filter_exec = Arc::new(FilterExec::try_new(
+ Arc::clone(&dynamic_filter_expr),
+ custom_exec,
+ )?) as Arc<dyn ExecutionPlan>;
+
+ // Roundtrip with DeduplicatingProtoConverter
+ let codec = CustomExecWithExprsCodec {};
+ let converter = DeduplicatingProtoConverter {};
+
+ let bytes = physical_plan_to_bytes_with_proto_converter(
+ Arc::clone(&filter_exec),
+ &codec,
+ &converter,
+ )?;
+
+ let ctx = SessionContext::new();
+ let deser_converter = DeduplicatingProtoConverter {};
+ let deserialized = physical_plan_from_bytes_with_proto_converter(
+ bytes.as_ref(),
+ ctx.task_ctx().as_ref(),
+ &codec,
+ &deser_converter,
+ )?;
+
+ // Extract the deserialized FilterExec's dynamic filter
+ let deser_filter = deserialized
+ .downcast_ref::<FilterExec>()
+ .expect("Top-level should be FilterExec");
+ let deser_filter_df = deser_filter.predicate();
+
+ // Extract the deserialized custom node's dynamic filter
+ let deser_custom = deser_filter
+ .input()
+ .downcast_ref::<CustomExecWithExprs>()
+ .expect("FilterExec child should be CustomExecWithExprs");
+ assert_eq!(deser_custom.exprs.len(), 1, "Should have one expression");
+ let [deser_custom_df] = deser_custom.exprs.as_slice() else {
+ return internal_err!("Custom node should have one expression");
+ };
+
+ // Pass the un-remapped filter first so the helper's `with_new_children`
+ // rewrite can reconstruct the remapped form on the other side.
+ assert_dynamic_filters_equal(deser_custom_df, deser_filter_df);
+ assert_dynamic_filter_update_is_visible(deser_custom_df, deser_filter_df)?;
+ Ok(())
+}