Apache DataFusion SQL Query Engine
proto: serialize and dedupe dynamic filters v2 (#21807)
Jayant Shrivastava948cd09c1060
diff --git a/Cargo.lock b/Cargo.lockindex c18fd20128..af52588e53 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2479,7 +2479,6 @@ dependencies = ["pbjson 0.9.0","pretty_assertions","prost",- "rand 0.9.4","serde","serde_json","tokio",diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rsindex 2fc3bf2afa..6595635024 100644--- a/datafusion/physical-expr-common/src/physical_expr.rs+++ b/datafusion/physical-expr-common/src/physical_expr.rs@@ -161,6 +161,9 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>>;/// Returns a new PhysicalExpr where all children were replaced by new exprs.+ ///+ /// If the implementation returns a [`PhysicalExpr::expression_id`], then+ /// the identifier should be preserved by the new expression.fn with_new_children(self: Arc<Self>,children: Vec<Arc<dyn PhysicalExpr>>,@@ -444,6 +447,23 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {fn placement(&self) -> ExpressionPlacement {ExpressionPlacement::KeepInPlace}++ /// Return a stable, globally-unique identifier for this [`PhysicalExpr`], if it+ /// has one.+ ///+ /// This identifier tracks which expressions which are connected (e.g. `DynamicFilterPhysicalExpr`+ /// where two expressions may be different but store the same mutable inner state). Tracking+ /// connected expressions helps preserve referential integrity within plan nodes+ /// during serialization and deserialization.+ ///+ /// This id must be preserved across [`PhysicalExpr::with_new_children`] or any other+ /// methods which may want to preserve identity.+ ///+ /// Default is `None`: the expression has no identity worth preserving across a+ /// serialization boundary.+ fn expression_id(&self) -> Option<u64> {+ None+ }}#[deprecated(diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rsindex 47398d87e2..2db328377a 100644--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs@@ -16,6 +16,7 @@// under the License.use parking_lot::RwLock;+use std::sync::atomic::{AtomicU64, Ordering};use std::{fmt::Display, hash::Hash, sync::Arc};use tokio::sync::watch;@@ -76,21 +77,51 @@ pub struct DynamicFilterPhysicalExpr {nullable: Arc<RwLock<Option<bool>>>,}-#[derive(Debug)]-struct Inner {+/// Atomic internal state of a [`DynamicFilterPhysicalExpr`].+///+/// `expression_id` lives here because it identifies the actual filter expression `expr`.+/// Derived `DynamicFilterPhysicalExpr`s (e.g. via [`PhysicalExpr::with_new_children`]) are+/// the same logical filter and must report the same `expression_id`.+///+/// **Warning:** exposed publicly solely so that proto (de)serialization in+/// `datafusion-proto` can read and rebuild this state. Do not treat this type+/// or its layout as a stable API.+#[derive(Clone)]+pub struct Inner {+ /// A unique identifier for the expression.+ pub expression_id: u64,/// A counter that gets incremented every time the expression is updated so that we can track changes cheaply./// This is used for [`PhysicalExpr::snapshot_generation`] to have a cheap check for changes.- generation: u64,- expr: Arc<dyn PhysicalExpr>,+ pub generation: u64,+ pub expr: Arc<dyn PhysicalExpr>,/// Flag for quick synchronous check if filter is complete./// This is redundant with the watch channel state, but allows us to return immediately/// from `wait_complete()` without subscribing if already complete.- is_complete: bool,+ pub is_complete: bool,+}++// TODO: Include expression_id in Debug output.+//+// See https://github.com/apache/datafusion/issues/20418. Currently, plan nodes+// like `HashJoinExec`, `AggregateExec`, `SortExec` do not serialize their+// dynamic filter. They auto-create one on decode with a fresh `expression_id`,+// so a round-trip Debug comparison would diverge purely on the id even when+// the rest of the state is preserved. Excluding it from Debug keeps those+// roundtrip equality assertions meaningful until that work lands.+impl std::fmt::Debug for Inner {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("Inner")+ .field("generation", &self.generation)+ .field("expr", &self.expr)+ .field("is_complete", &self.is_complete)+ .finish()+ }}impl Inner {fn new(expr: Arc<dyn PhysicalExpr>) -> Self {Self {+ expression_id: EXPR_ID_SOURCE.next(),// Start with generation 1 which gives us a different result for [`PhysicalExpr::generation`] than the default 0.// This is not currently used anywhere but it seems useful to have this simple distinction.generation: 1,@@ -243,6 +274,8 @@ impl DynamicFilterPhysicalExpr {let mut current = self.inner.write();let new_generation = current.generation + 1;*current = Inner {+ // Preserve the expression id across updates.+ expression_id: current.expression_id,generation: new_generation,expr: new_expr,is_complete: current.is_complete,@@ -346,6 +379,62 @@ impl DynamicFilterPhysicalExpr {write!(f, " ]")}++ /// Return the filter's original children (before any remapping).+ ///+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.+ /// Not a stable API.+ pub fn original_children(&self) -> &[Arc<dyn PhysicalExpr>] {+ &self.children+ }++ /// Return the filter's remapped children, if any have been set via+ /// [`PhysicalExpr::with_new_children`].+ ///+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.+ /// Not a stable API.+ pub fn remapped_children(&self) -> Option<&[Arc<dyn PhysicalExpr>]> {+ self.remapped_children.as_deref()+ }++ /// Rebuild a `DynamicFilterPhysicalExpr` from its stored parts. Used by+ /// proto deserialization.+ ///+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.+ /// Not a stable API.+ pub fn from_parts(+ children: Vec<Arc<dyn PhysicalExpr>>,+ remapped_children: Option<Vec<Arc<dyn PhysicalExpr>>>,+ inner: Inner,+ ) -> Self {+ let state = if inner.is_complete {+ FilterState::Complete {+ generation: inner.generation,+ }+ } else {+ FilterState::InProgress {+ generation: inner.generation,+ }+ };+ let (state_watch, _) = watch::channel(state);++ Self {+ children,+ remapped_children,+ inner: Arc::new(RwLock::new(inner)),+ state_watch,+ data_type: Arc::new(RwLock::new(None)),+ nullable: Arc::new(RwLock::new(None)),+ }+ }++ /// Return a clone of the atomically-captured `Inner` state.+ ///+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.+ /// Not a stable API.+ pub fn inner(&self) -> Inner {+ self.inner.read().clone()+ }}impl PhysicalExpr for DynamicFilterPhysicalExpr {@@ -364,6 +453,7 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {Ok(Arc::new(Self {children: self.children.clone(),remapped_children: Some(children),+ // Note: expression_id is preservedinner: Arc::clone(&self.inner),state_watch: self.state_watch.clone(),data_type: Arc::clone(&self.data_type),@@ -444,8 +534,36 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {// Return the current generation of the expression.self.inner.read().generation}++ fn expression_id(&self) -> Option<u64> {+ Some(self.inner.read().expression_id)+ }}+/// An atomic counter used to generate monotonic u64 ids.+struct ExpressionIdAtomicCounter {+ inner: AtomicU64,+}++impl ExpressionIdAtomicCounter {+ const fn new() -> Self {+ Self {+ inner: AtomicU64::new(0),+ }+ }++ /// Returns a fresh `expression_id` by incrementing the internal counter.+ fn next(&self) -> u64 {+ self.inner.fetch_add(1, Ordering::Relaxed)+ }+}++/// Process-wide source of deterministic `expression_id`s for [`DynamicFilterPhysicalExpr`].+///+/// Currently, no other [`PhysicalExpr`]s use this source. If needed, it can be moved out of this+/// file and be made public for other expressions to use.+static EXPR_ID_SOURCE: ExpressionIdAtomicCounter = ExpressionIdAtomicCounter::new();+#[cfg(test)]mod test {use crate::{@@ -861,4 +979,130 @@ mod test {"Hash should be stable after update (identity-based)");}++ /// Verifies that `from_parts` rebuilds a `DynamicFilterPhysicalExpr`+ /// whose observable state (original children, remapped children,+ /// expression id, inner generation/expr/is_complete) matches the source+ /// filter.+ #[test]+ fn test_from_parts_preserves_state() {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let col_a = col("a", &schema).unwrap();++ // Create a dynamic filter with children+ let expr = Arc::new(BinaryExpr::new(+ Arc::clone(&col_a),+ datafusion_expr::Operator::Gt,+ lit(10) as Arc<dyn PhysicalExpr>,+ ));+ let filter = DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&col_a)],+ expr as Arc<dyn PhysicalExpr>,+ );++ // Add remapped children.+ let reassigned_schema = Arc::new(Schema::new(vec![+ Field::new("b", DataType::Int32, false),+ Field::new("a", DataType::Int32, false),+ ]));+ let reassigned = reassign_expr_columns(+ Arc::new(filter) as Arc<dyn PhysicalExpr>,+ &reassigned_schema,+ )+ .expect("reassign_expr_columns should succeed");+ let reassigned = reassigned+ .downcast_ref::<DynamicFilterPhysicalExpr>()+ .expect("Expected dynamic filter after reassignment");++ reassigned+ .update(lit(42) as Arc<dyn PhysicalExpr>)+ .expect("Update should succeed");+ reassigned.mark_complete();++ // Capture the parts and reconstruct. `expression_id` rides in `inner`.+ let reconstructed = DynamicFilterPhysicalExpr::from_parts(+ reassigned.original_children().to_vec(),+ reassigned.remapped_children().map(|r| r.to_vec()),+ reassigned.inner(),+ );++ assert_eq!(+ reassigned.original_children(),+ reconstructed.original_children(),+ );+ assert_eq!(+ reassigned.remapped_children(),+ reconstructed.remapped_children(),+ );+ assert_eq!(reassigned.expression_id(), reconstructed.expression_id());+ let r = reassigned.inner();+ let c = reconstructed.inner();+ assert_eq!(r.generation, c.generation);+ assert_eq!(r.is_complete, c.is_complete);+ assert_eq!(format!("{:?}", r.expr), format!("{:?}", c.expr));+ }++ #[tokio::test]+ async fn test_expression_id() {+ let source_schema =+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));+ let col_a = col("a", &source_schema).unwrap();++ // Create a source filter+ let source = Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::clone(&col_a)],+ lit(true) as Arc<dyn PhysicalExpr>,+ ));+ let source_clone = Arc::clone(&source);++ // Create a derived filter by reassigning the source filter to a different schema.+ let derived_schema = Arc::new(Schema::new(vec![+ Field::new("x", DataType::Int32, false),+ Field::new("a", DataType::Int32, false),+ ]));+ let derived = reassign_expr_columns(+ Arc::clone(&source) as Arc<dyn PhysicalExpr>,+ &derived_schema,+ )+ .expect("reassign_expr_columns should succeed");++ let derived_expression_id = derived+ .expression_id()+ .expect("derived filter should have an expression id");+ let source_expression_id = source+ .expression_id()+ .expect("source filter should have an expression id");+ let source_clone_expression_id = source_clone+ .expression_id()+ .expect("source clone should have an expression id");++ assert_eq!(+ source_clone_expression_id, source_expression_id,+ "cloned filter should preserve its expression id",+ );++ assert_eq!(+ derived_expression_id, source_expression_id,+ "derived filters should carry forward the source expression id",+ );++ // `update()` rewrites the entire `Inner` struct in place; pin down+ // that the rewrite preserves `expression_id`.+ source+ .update(lit(99) as Arc<dyn PhysicalExpr>)+ .expect("update should succeed");+ assert_eq!(+ source.expression_id().unwrap(),+ source_expression_id,+ "update() must not change expression_id",+ );++ // `mark_complete()` also touches `Inner`; same invariant.+ source.mark_complete();+ assert_eq!(+ source.expression_id().unwrap(),+ source_expression_id,+ "mark_complete() must not change expression_id",+ );+ }}diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rsindex bb6d9ca9c9..7cf874c448 100644--- a/datafusion/physical-expr/src/expressions/mod.rs+++ b/datafusion/physical-expr/src/expressions/mod.rs@@ -45,7 +45,7 @@ pub use case::{CaseExpr, case};pub use cast::{CastExpr, cast};pub use column::{Column, col, with_new_schema};pub use datafusion_expr::utils::format_state_name;-pub use dynamic_filters::DynamicFilterPhysicalExpr;+pub use dynamic_filters::{DynamicFilterPhysicalExpr, Inner as DynamicFilterInner};pub use in_list::{InListExpr, in_list};pub use is_not_null::{IsNotNullExpr, is_not_null};pub use is_null::{IsNullExpr, is_null};diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex baa7dd8770..4484846813 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -66,7 +66,6 @@ datafusion-proto-common = { workspace = true }object_store = { workspace = true }pbjson = { workspace = true, optional = true }prost = { workspace = true }-rand = { workspace = true }serde = { version = "1.0", optional = true }serde_json = { workspace = true, optional = true }diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 5cd57b9f98..511e8eb1b0 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -889,11 +889,9 @@ message PhysicalExprNode {reserved 17;// Unique identifier for this expression to do deduplication during deserialization.- // When serializing, this is set to a unique identifier for each combination of- // expression, process and serialization run.- // When deserializing, if this ID has been seen before, the cached Arc is returned- // instead of creating a new one, enabling reconstruction of referential integrity- // across serde roundtrips.+ // When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing,+ // this id is used by the `DeduplicatingProtoConverter` to preserve referential+ // integrity across serde roundtrips for different expressions with the same id.optional uint64 expr_id = 30;oneof ExprType {@@ -935,9 +933,19 @@ message PhysicalExprNode {PhysicalHashExprNode hash_expr = 21;PhysicalScalarSubqueryExprNode scalar_subquery = 22;++ PhysicalDynamicFilterNode dynamic_filter = 23;}}+message PhysicalDynamicFilterNode {+ repeated PhysicalExprNode children = 1;+ repeated PhysicalExprNode remapped_children = 2;+ uint64 generation = 3;+ PhysicalExprNode inner_expr = 4;+ bool is_complete = 5;+}+message PhysicalScalarUdfNode {string name = 1;repeated PhysicalExprNode args = 2;@@ -1510,4 +1518,4 @@ message PhysicalScalarSubqueryExprNode {datafusion_common.ArrowType data_type = 1;bool nullable = 2;uint32 index = 3;-}\ No newline at end of file+}diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex 74f7b3b1c3..c05d3283ea 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -16702,6 +16702,172 @@ impl<'de> serde::Deserialize<'de> for PhysicalDateTimeIntervalExprNode {deserializer.deserialize_struct("datafusion.PhysicalDateTimeIntervalExprNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalDynamicFilterNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.children.is_empty() {+ len += 1;+ }+ if !self.remapped_children.is_empty() {+ len += 1;+ }+ if self.generation != 0 {+ len += 1;+ }+ if self.inner_expr.is_some() {+ len += 1;+ }+ if self.is_complete {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalDynamicFilterNode", len)?;+ if !self.children.is_empty() {+ struct_ser.serialize_field("children", &self.children)?;+ }+ if !self.remapped_children.is_empty() {+ struct_ser.serialize_field("remappedChildren", &self.remapped_children)?;+ }+ if self.generation != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("generation", ToString::to_string(&self.generation).as_str())?;+ }+ if let Some(v) = self.inner_expr.as_ref() {+ struct_ser.serialize_field("innerExpr", v)?;+ }+ if self.is_complete {+ struct_ser.serialize_field("isComplete", &self.is_complete)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalDynamicFilterNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "children",+ "remapped_children",+ "remappedChildren",+ "generation",+ "inner_expr",+ "innerExpr",+ "is_complete",+ "isComplete",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ Children,+ RemappedChildren,+ Generation,+ InnerExpr,+ IsComplete,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl serde::de::Visitor<'_> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "children" => Ok(GeneratedField::Children),+ "remappedChildren" | "remapped_children" => Ok(GeneratedField::RemappedChildren),+ "generation" => Ok(GeneratedField::Generation),+ "innerExpr" | "inner_expr" => Ok(GeneratedField::InnerExpr),+ "isComplete" | "is_complete" => Ok(GeneratedField::IsComplete),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalDynamicFilterNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalDynamicFilterNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalDynamicFilterNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut children__ = None;+ let mut remapped_children__ = None;+ let mut generation__ = None;+ let mut inner_expr__ = None;+ let mut is_complete__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::Children => {+ if children__.is_some() {+ return Err(serde::de::Error::duplicate_field("children"));+ }+ children__ = Some(map_.next_value()?);+ }+ GeneratedField::RemappedChildren => {+ if remapped_children__.is_some() {+ return Err(serde::de::Error::duplicate_field("remappedChildren"));+ }+ remapped_children__ = Some(map_.next_value()?);+ }+ GeneratedField::Generation => {+ if generation__.is_some() {+ return Err(serde::de::Error::duplicate_field("generation"));+ }+ generation__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::InnerExpr => {+ if inner_expr__.is_some() {+ return Err(serde::de::Error::duplicate_field("innerExpr"));+ }+ inner_expr__ = map_.next_value()?;+ }+ GeneratedField::IsComplete => {+ if is_complete__.is_some() {+ return Err(serde::de::Error::duplicate_field("isComplete"));+ }+ is_complete__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(PhysicalDynamicFilterNode {+ children: children__.unwrap_or_default(),+ remapped_children: remapped_children__.unwrap_or_default(),+ generation: generation__.unwrap_or_default(),+ inner_expr: inner_expr__,+ is_complete: is_complete__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalDynamicFilterNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalExprNode {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>@@ -16784,6 +16950,9 @@ impl serde::Serialize for PhysicalExprNode {physical_expr_node::ExprType::ScalarSubquery(v) => {struct_ser.serialize_field("scalarSubquery", v)?;}+ physical_expr_node::ExprType::DynamicFilter(v) => {+ struct_ser.serialize_field("dynamicFilter", v)?;+ }}}struct_ser.end()@@ -16832,6 +17001,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"hashExpr","scalar_subquery","scalarSubquery",+ "dynamic_filter",+ "dynamicFilter",];#[allow(clippy::enum_variant_names)]@@ -16857,6 +17028,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {UnknownColumn,HashExpr,ScalarSubquery,+ DynamicFilter,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -16899,6 +17071,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"unknownColumn" | "unknown_column" => Ok(GeneratedField::UnknownColumn),"hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),"scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery),+ "dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -17068,6 +17241,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {return Err(serde::de::Error::duplicate_field("scalarSubquery"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::ScalarSubquery)+;+ }+ GeneratedField::DynamicFilter => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("dynamicFilter"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::DynamicFilter);}}diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 3db2f2de8e..af9b1404bb 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1327,16 +1327,14 @@ pub struct PhysicalExtensionNode {#[derive(Clone, PartialEq, ::prost::Message)]pub struct PhysicalExprNode {/// Unique identifier for this expression to do deduplication during deserialization.- /// When serializing, this is set to a unique identifier for each combination of- /// expression, process and serialization run.- /// When deserializing, if this ID has been seen before, the cached Arc is returned- /// instead of creating a new one, enabling reconstruction of referential integrity- /// across serde roundtrips.+ /// When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing,+ /// this id is used by the `DeduplicatingProtoConverter` to preserve referential+ /// integrity across serde roundtrips for different expressions with the same id.#[prost(uint64, optional, tag = "30")]pub expr_id: ::core::option::Option<u64>,#[prost(oneof = "physical_expr_node::ExprType",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23")]pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,}@@ -1391,9 +1389,24 @@ pub mod physical_expr_node {HashExpr(super::PhysicalHashExprNode),#[prost(message, tag = "22")]ScalarSubquery(super::PhysicalScalarSubqueryExprNode),+ #[prost(message, tag = "23")]+ DynamicFilter(::prost::alloc::boxed::Box<super::PhysicalDynamicFilterNode>),}}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalDynamicFilterNode {+ #[prost(message, repeated, tag = "1")]+ pub children: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ #[prost(message, repeated, tag = "2")]+ pub remapped_children: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ #[prost(uint64, tag = "3")]+ pub generation: u64,+ #[prost(message, optional, boxed, tag = "4")]+ pub inner_expr: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalExprNode>>,+ #[prost(bool, tag = "5")]+ pub is_complete: bool,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct PhysicalScalarUdfNode {#[prost(string, tag = "1")]pub name: ::prost::alloc::string::String,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex 732676941b..43ebf04743 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -60,6 +60,9 @@ use super::{use crate::logical_plan::{self};use crate::protobuf::physical_expr_node::ExprType;use crate::{convert_required, protobuf};+use datafusion_physical_expr::expressions::{+ DynamicFilterInner, DynamicFilterPhysicalExpr,+};impl From<&protobuf::PhysicalColumn> for Column {fn from(c: &protobuf::PhysicalColumn) -> Column {@@ -511,6 +514,53 @@ pub fn parse_physical_expr_with_converter(results.clone(),))}+ ExprType::DynamicFilter(dynamic_filter) => {+ let children = parse_physical_exprs(+ &dynamic_filter.children,+ ctx,+ input_schema,+ proto_converter,+ )?;++ let remapped_children = if !dynamic_filter.remapped_children.is_empty() {+ Some(parse_physical_exprs(+ &dynamic_filter.remapped_children,+ ctx,+ input_schema,+ proto_converter,+ )?)+ } else {+ None+ };++ let inner_expr = parse_required_physical_expr(+ dynamic_filter.inner_expr.as_deref(),+ ctx,+ "inner_expr",+ input_schema,+ proto_converter,+ )?;++ let expression_id = proto.expr_id.ok_or_else(|| {+ proto_error(+ "DynamicFilterPhysicalExpr requires PhysicalExprNode.expr_id \+ to be set by the serializer",+ )+ })?;++ let base_filter: Arc<dyn PhysicalExpr> =+ Arc::new(DynamicFilterPhysicalExpr::from_parts(+ children,+ remapped_children,+ DynamicFilterInner {+ expression_id,+ generation: dynamic_filter.generation,+ expr: inner_expr,+ is_complete: dynamic_filter.is_complete,+ },+ ));+ base_filter+ }ExprType::Extension(extension) => {let inputs: Vec<Arc<dyn PhysicalExpr>> = extension.inputsdiff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rsindex d4a83a76f3..5172a552fa 100644--- a/datafusion/proto/src/physical_plan/mod.rs+++ b/datafusion/proto/src/physical_plan/mod.rs@@ -19,7 +19,6 @@ use std::any::Any;use std::cell::RefCell;use std::collections::HashMap;use std::fmt::Debug;-use std::hash::{DefaultHasher, Hash, Hasher};use std::sync::Arc;use arrow::compute::SortOptions;@@ -3888,83 +3887,11 @@ impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {}}-/// Internal serializer that adds expr_id to expressions.-/// Created fresh for each serialization operation.-struct DeduplicatingSerializer {- /// Random salt combined with pointer addresses and process ID to create globally unique expr_ids.- session_id: u64,-}--impl DeduplicatingSerializer {- fn new() -> Self {- Self {- session_id: rand::random(),- }- }-}--impl PhysicalProtoConverterExtension for DeduplicatingSerializer {- fn proto_to_execution_plan(- &self,- _proto: &protobuf::PhysicalPlanNode,- _ctx: &PhysicalPlanDecodeContext<'_>,- ) -> Result<Arc<dyn ExecutionPlan>> {- internal_err!("DeduplicatingSerializer cannot deserialize execution plans")- }-- fn execution_plan_to_proto(- &self,- plan: &Arc<dyn ExecutionPlan>,- codec: &dyn PhysicalExtensionCodec,- ) -> Result<protobuf::PhysicalPlanNode>- where- Self: Sized,- {- protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(- Arc::clone(plan),- codec,- self,- )- }-- fn proto_to_physical_expr(- &self,- _proto: &protobuf::PhysicalExprNode,- _input_schema: &Schema,- _ctx: &PhysicalPlanDecodeContext<'_>,- ) -> Result<Arc<dyn PhysicalExpr>>- where- Self: Sized,- {- internal_err!("DeduplicatingSerializer cannot deserialize physical expressions")- }-- fn physical_expr_to_proto(- &self,- expr: &Arc<dyn PhysicalExpr>,- codec: &dyn PhysicalExtensionCodec,- ) -> Result<protobuf::PhysicalExprNode> {- let mut proto = serialize_physical_expr_with_converter(expr, codec, self)?;-- // Hash session_id, pointer address, and process ID together to create expr_id.- // - session_id: random per serializer, prevents collisions when merging serializations- // - ptr: unique address per Arc within a process- // - pid: prevents collisions if serializer is shared across processes- let mut hasher = DefaultHasher::new();- self.session_id.hash(&mut hasher);- (Arc::as_ptr(expr) as *const () as u64).hash(&mut hasher);- std::process::id().hash(&mut hasher);- proto.expr_id = Some(hasher.finish());-- Ok(proto)- }-}--/// Internal deserializer that caches expressions by expr_id.-/// Created fresh for each deserialization operation.+/// Internal deserializer that caches expressions by their `expression_id()` so+/// multiple occurrences of the same expression are deduped.#[derive(Default)]struct DeduplicatingDeserializer {- /// Cache mapping expr_id to deserialized expressions.+ /// Cache mapping expression_id to deserialized expressions.cache: RefCell<HashMap<u64, Arc<dyn PhysicalExpr>>>,}@@ -3997,19 +3924,30 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {whereSelf: Sized,{- if let Some(expr_id) = proto.expr_id {- // Check cache first- if let Some(cached) = self.cache.borrow().get(&expr_id) {- return Ok(Arc::clone(cached));- }- // Deserialize and cache- let expr =- parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;- self.cache.borrow_mut().insert(expr_id, Arc::clone(&expr));- Ok(expr)- } else {- parse_physical_expr_with_converter(proto, input_schema, ctx, self)+ // `expr_id` is the generic identity slot on `PhysicalExprNode`.+ // The default serializer populates it from `PhysicalExpr::expression_id`.+ // A missing id means this expression type doesn't participate in deduping.+ let Some(id) = proto.expr_id else {+ return parse_physical_expr_with_converter(proto, input_schema, ctx, self);+ };++ let parsed = parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;++ let mut cache = self.cache.borrow_mut();+ if let Some(cached) = cache.get(&id) {+ // Since expressions may manage their own internal state when deriving+ // expressions via `with_new_children`, we use `with_new_children`+ // to opt into the same behavior.+ //+ // For example, one `DynamicFilterPhysicalExpr` may be derived from+ // another resulting in shared references. Using `with_new_children`+ // is meant to preserve those references.+ let children: Vec<_> = parsed.children().into_iter().cloned().collect();+ return Arc::clone(cached).with_new_children(children);}++ cache.insert(id, Arc::clone(&parsed));+ Ok(parsed)}fn physical_expr_to_proto(@@ -4021,19 +3959,12 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {}}-/// A proto converter that adds expression deduplication during serialization-/// and deserialization.-///-/// During serialization, each expression's Arc pointer address is XORed with a-/// random session_id to create a salted `expr_id`. This prevents cross-process-/// collisions when serialized plans are merged.+/// A proto converter that deduplicates [`PhysicalExpr`] by [`PhysicalExpr::expression_id`].+/// This helps preserve referential integrity when deserializing [`ExecutionPlan`]s+/// which may contain multiple occurrences of the same [`PhysicalExpr`] (ex. when+/// [`DynamicFilterPhysicalExpr`] are pushed down, it is important to preserve+/// referential integrity).///-/// During deserialization, expressions with the same `expr_id` share the same-/// Arc, reducing memory usage for plans with duplicate expressions (e.g., large-/// IN lists) and supporting correctly linking [`DynamicFilterPhysicalExpr`] instances.-///-/// This converter is stateless - it creates internal serializers/deserializers-/// on demand for each operation.////// [`DynamicFilterPhysicalExpr`]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/expressions/struct.DynamicFilterPhysicalExpr.html#[derive(Debug, Default, Clone, Copy)]@@ -4057,11 +3988,10 @@ impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {whereSelf: Sized,{- let serializer = DeduplicatingSerializer::new();protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(Arc::clone(plan),codec,- &serializer,+ self,)}@@ -4083,8 +4013,7 @@ impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {expr: &Arc<dyn PhysicalExpr>,codec: &dyn PhysicalExtensionCodec,) -> Result<protobuf::PhysicalExprNode> {- let serializer = DeduplicatingSerializer::new();- serializer.physical_expr_to_proto(expr, codec)+ serialize_physical_expr_with_converter(expr, codec, self)}}diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex cb69b913b5..83c11cfc6b 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -34,11 +34,11 @@ use datafusion_expr::WindowFrame;use datafusion_physical_expr::ScalarFunctionExpr;use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};-use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_plan::expressions::{- BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,- LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,+ BinaryExpr, CaseExpr, CastExpr, Column, DynamicFilterPhysicalExpr, InListExpr,+ IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr,+ UnKnownColumn,};use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};use datafusion_physical_plan::udaf::AggregateFunctionExpr;@@ -258,10 +258,8 @@ pub fn serialize_physical_expr_with_converter(codec: &dyn PhysicalExtensionCodec,proto_converter: &dyn PhysicalProtoConverterExtension,) -> Result<protobuf::PhysicalExprNode> {- // Snapshot the expr in case it has dynamic predicate state so- // it can be serialized- let expr = snapshot_physical_expr(Arc::clone(value))?;-+ let expr = value.as_ref();+ let expr_id = value.expression_id();// HashTableLookupExpr is used for dynamic filter pushdown in hash joins.// It contains an Arc<dyn JoinHashMapType> (the build-side hash table) which// cannot be serialized - the hash table is a runtime structure built during@@ -282,14 +280,14 @@ pub fn serialize_physical_expr_with_converter()),};return Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Literal(value)),});}if let Some(expr) = expr.downcast_ref::<Column>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Column(protobuf::PhysicalColumn {name: expr.name().to_string(),@@ -299,7 +297,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<UnKnownColumn>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::UnknownColumn(protobuf::UnknownColumn {name: expr.name().to_string(),@@ -341,14 +339,14 @@ pub fn serialize_physical_expr_with_converter(});Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::BinaryExpr(binary_expr,)),})} else if let Some(expr) = expr.downcast_ref::<CaseExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Case(Box::new(@@ -391,7 +389,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<NotExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::NotExpr(Box::new(protobuf::PhysicalNot {expr: Some(Box::new(@@ -402,7 +400,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<IsNullExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::IsNullExpr(Box::new(protobuf::PhysicalIsNull {expr: Some(Box::new(@@ -413,7 +411,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<IsNotNullExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::IsNotNullExpr(Box::new(protobuf::PhysicalIsNotNull {expr: Some(Box::new(@@ -424,7 +422,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<InListExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::InList(Box::new(protobuf::PhysicalInListNode {expr: Some(Box::new(@@ -437,7 +435,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<NegativeExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Negative(Box::new(protobuf::PhysicalNegativeNode {expr: Some(Box::new(@@ -448,14 +446,14 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(lit) = expr.downcast_ref::<Literal>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Literal(lit.value().try_into()?,)),})} else if let Some(cast) = expr.downcast_ref::<CastExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Cast(Box::new(protobuf::PhysicalCastNode {expr: Some(Box::new(@@ -467,7 +465,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(cast) = expr.downcast_ref::<TryCastExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(protobuf::PhysicalTryCastNode {expr: Some(Box::new(@@ -481,7 +479,7 @@ pub fn serialize_physical_expr_with_converter(let mut buf = Vec::new();codec.try_encode_udf(expr.fun(), &mut buf)?;Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarUdf(protobuf::PhysicalScalarUdfNode {name: expr.name().to_string(),@@ -498,7 +496,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<LikeExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::LikeExpr(Box::new(protobuf::PhysicalLikeExprNode {negated: expr.negated(),@@ -514,7 +512,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<HashExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(protobuf::PhysicalHashExprNode {on_columns: serialize_physical_exprs(@@ -529,7 +527,7 @@ pub fn serialize_physical_expr_with_converter(})} else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(protobuf::PhysicalScalarSubqueryExprNode {data_type: Some(expr.data_type().try_into()?),@@ -538,6 +536,39 @@ pub fn serialize_physical_expr_with_converter(},)),})+ } else if let Some(df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {+ let children = df+ .original_children()+ .iter()+ .map(|child| proto_converter.physical_expr_to_proto(child, codec))+ .collect::<Result<Vec<_>>>()?;++ let remapped_children = if let Some(remapped) = df.remapped_children() {+ remapped+ .iter()+ .map(|child| proto_converter.physical_expr_to_proto(child, codec))+ .collect::<Result<Vec<_>>>()?+ } else {+ vec![]+ };++ // Atomic snapshot of inner state.+ let inner = df.inner();+ let inner_expr =+ Box::new(proto_converter.physical_expr_to_proto(&inner.expr, codec)?);++ Ok(protobuf::PhysicalExprNode {+ expr_id,+ expr_type: Some(protobuf::physical_expr_node::ExprType::DynamicFilter(+ Box::new(protobuf::PhysicalDynamicFilterNode {+ children,+ remapped_children,+ generation: inner.generation,+ inner_expr: Some(inner_expr),+ is_complete: inner.is_complete,+ }),+ )),+ })} else {let mut buf: Vec<u8> = vec![];match codec.try_encode_expr(value, &mut buf) {@@ -548,7 +579,7 @@ pub fn serialize_physical_expr_with_converter(.map(|e| proto_converter.physical_expr_to_proto(e, codec)).collect::<Result<_>>()?;Ok(protobuf::PhysicalExprNode {- expr_id: None,+ expr_id,expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(protobuf::PhysicalExtensionExprNode { expr: buf, inputs },)),diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex 8465c2a196..fa342ae907 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -132,6 +132,8 @@ use crate::cases::{CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,MyRegexUdfNode,};+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;+use datafusion_physical_expr::utils::reassign_expr_columns;/// Perform a serde roundtrip and assert that the string representation of the before and after plans/// are identical. Note that this often isn't sufficient to guarantee that no information is@@ -2685,431 +2687,294 @@ fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {roundtrip_test(filter)}-/// Test that expression deduplication works during deserialization.-/// When the same expression Arc is serialized multiple times, it should be-/// deduplicated on deserialization (sharing the same Arc).-#[test]-fn test_expression_deduplication() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));-- // Create a shared expression that will be used multiple times- let shared_col: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));-- // Create an InList expression that uses the same column Arc multiple times- // This simulates a real-world scenario where expressions are shared- let in_list_expr = in_list(- Arc::clone(&shared_col),- vec![lit(1i64), lit(2i64), lit(3i64)],- &false,- &schema,- )?;-- // Create a binary expression that uses the shared column and the in_list result- let binary_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::clone(&shared_col),- Operator::Eq,- lit(42i64),- ));-- // Create a plan that has both expressions (they share the `shared_col` Arc)- let input = Arc::new(EmptyExec::new(schema.clone()));- let filter = FilterExecBuilder::new(in_list_expr, input).build()?;- let projection_exprs = vec![ProjectionExpr {- expr: binary_expr,- alias: "result".to_string(),- }];- let exec_plan =- Arc::new(ProjectionExec::try_new(projection_exprs, Arc::new(filter))?);-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};-- // Perform roundtrip- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,- &codec,- &proto_converter,- )?;-- // Create a new converter for deserialization (fresh cache)- let deser_converter = DeduplicatingProtoConverter {};- let result_plan = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &deser_converter,- )?;-- // Verify the plan structure is correct- pretty_assertions::assert_eq!(format!("{exec_plan:?}"), format!("{result_plan:?}"));-- Ok(())+/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.+fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {+ Arc::new(DynamicFilterPhysicalExpr::new(+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],+ lit(true),+ )) as Arc<dyn PhysicalExpr>}-/// Test that expression deduplication correctly shares Arcs for identical expressions.-/// This test verifies the core deduplication behavior.-#[test]-fn test_expression_deduplication_arc_sharing() -> Result<()> {- use datafusion_proto::bytes::{- physical_plan_from_bytes_with_proto_converter,- physical_plan_to_bytes_with_proto_converter,- };-- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));-- // Create a column expression- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));-- // Create a projection that uses the SAME Arc twice- // After roundtrip, both should point to the same Arc- let projection_exprs = vec![- ProjectionExpr {- expr: Arc::clone(&col_expr),- alias: "a1".to_string(),- },- ProjectionExpr {- expr: Arc::clone(&col_expr), // Same Arc!- alias: "a2".to_string(),- },- ];-- let input = Arc::new(EmptyExec::new(schema));- let exec_plan = Arc::new(ProjectionExec::try_new(projection_exprs, input)?);-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};-- // Serialize- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,- &codec,- &proto_converter,- )?;-- // Deserialize with a fresh converter- let deser_converter = DeduplicatingProtoConverter {};- let result_plan = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &deser_converter,- )?;-- // Get the projection from the result- let projection = result_plan- .downcast_ref::<ProjectionExec>()- .expect("Expected ProjectionExec");-- let exprs: Vec<_> = projection.expr().iter().collect();- assert_eq!(exprs.len(), 2);-- // The key test: both expressions should point to the same Arc after deduplication- // This is because they were the same Arc before serialization- assert!(- Arc::ptr_eq(&exprs[0].expr, &exprs[1].expr),- "Expected both expressions to share the same Arc after deduplication"- );-- Ok(())+/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.+fn make_reassigned_dynamic_filter(+ filter: Arc<dyn PhysicalExpr>,+) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("b", DataType::Int64, false),+ Field::new("a", DataType::Int64, false),+ ]));+ let reassigned = reassign_expr_columns(filter, &schema)?;+ Ok((schema, reassigned))}-/// Test backward compatibility: protos without expr_id should still deserialize correctly.-#[test]-fn test_backward_compatibility_no_expr_id() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));+/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the+/// default serializer from `PhysicalExpr::expression_id`.+fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {+ expr.expr_id+ .expect("expected PhysicalExprNode.expr_id to be populated")+}- // Manually create a proto without expr_id set- let proto = PhysicalExprNode {- expr_id: None, // Simulating old proto without this field- expr_type: Some(- datafusion_proto::protobuf::physical_expr_node::ExprType::Column(- datafusion_proto::protobuf::PhysicalColumn {- name: "a".to_string(),- index: 0,- },- ),- ),- };+/// Roundtrip a single physical expression shaped like so:+///+/// ```text+/// BinaryExpr(AND)+/// / \+/// filter_expr_1 filter_expr_2+/// ```+///+/// Returns filter_expr_1 and filter_expr_2 after deserialization.+fn roundtrip_dynamic_filter_expr_pair(+ filter_expr_1: Arc<dyn PhysicalExpr>,+ filter_expr_2: Arc<dyn PhysicalExpr>,+ schema: Arc<Schema>,+) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {+ let pair_expr = Arc::new(BinaryExpr::new(+ Arc::clone(&filter_expr_1),+ Operator::And,+ Arc::clone(&filter_expr_2),+ )) as Arc<dyn PhysicalExpr>;- let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DefaultPhysicalProtoConverter {};+ let converter = DeduplicatingProtoConverter {};+ let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;+ let ctx = SessionContext::new();let task_ctx = ctx.task_ctx();let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let deserialized_expr =+ converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;- // Should deserialize without error- let result = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;-- // Verify the result is correct- let col = result.downcast_ref::<Column>().expect("Expected Column");- assert_eq!(col.name(), "a");- assert_eq!(col.index(), 0);+ let binary = deserialized_expr+ .downcast_ref::<BinaryExpr>()+ .expect("Expected BinaryExpr");- Ok(())+ Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))}-/// Test that deduplication works within a single plan deserialization and that-/// separate deserializations produce independent expressions (no cross-operation sharing).-#[test]-fn test_deduplication_within_plan_deserialization() -> Result<()> {- use datafusion_proto::bytes::{- physical_plan_from_bytes_with_proto_converter,- physical_plan_to_bytes_with_proto_converter,- };-- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));+/// Roundtrip an execution plan shaped like so:+///+/// ```text+/// FilterExec(dynamic_filter_1 on a@0)+/// ProjectionExec(a := Column("a", source_index))+/// DataSourceExec+/// ParquetSource(predicate = dynamic_filter_2)+/// ```+///+/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with+/// different children.+///+/// Returns+/// - `dynamic_filter_1` before serialization+/// - `dynamic_filter_2` before serialization+/// - `dynamic_filter_1` after serialization+/// - `dynamic_filter_2` after serialization+#[allow(clippy::type_complexity)]+fn roundtrip_dynamic_filter_plan_pair() -> Result<(+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+ Arc<dyn PhysicalExpr>,+)> {+ let filter_expr_1 = make_dynamic_filter();+ let (data_source_schema, filter_expr_2) =+ make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;+ let left_before = Arc::clone(&filter_expr_1);+ let right_before = Arc::clone(&filter_expr_2);+ let file_source = Arc::new(+ ParquetSource::new(Arc::clone(&data_source_schema))+ .with_predicate(Arc::clone(&filter_expr_2)),+ );+ let scan_config =+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(+ "/path/to/file.parquet".to_string(),+ 1024,+ )])])+ .build();+ let data_source_exec =+ DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;- // Create a plan with expressions that will be deduplicated- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- let projection_exprs = vec![- ProjectionExpr {- expr: Arc::clone(&col_expr),- alias: "a1".to_string(),- },- ProjectionExpr {- expr: Arc::clone(&col_expr), // Same Arc - will be deduplicated- alias: "a2".to_string(),- },- ];- let exec_plan = Arc::new(ProjectionExec::try_new(- projection_exprs,- Arc::new(EmptyExec::new(schema)),- )?);+ let projection_exec = Arc::new(ProjectionExec::try_new(+ vec![ProjectionExpr {+ expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,+ alias: "a".to_string(),+ }],+ data_source_exec,+ )?) as Arc<dyn ExecutionPlan>;+ let filter_exec = Arc::new(FilterExec::try_new(+ Arc::clone(&filter_expr_1),+ projection_exec,+ )?) as Arc<dyn ExecutionPlan>;- let ctx = SessionContext::new();let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};-- // Serialize- let bytes = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,- &codec,- &proto_converter,- )?;-- // First deserialization- let plan1 = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &proto_converter,- )?;-- // Check that the plan was deserialized correctly with deduplication- let projection1 = plan1- .downcast_ref::<ProjectionExec>()- .expect("Expected ProjectionExec");- let exprs1: Vec<_> = projection1.expr().iter().collect();- assert_eq!(exprs1.len(), 2);- assert!(- Arc::ptr_eq(&exprs1[0].expr, &exprs1[1].expr),- "Expected both expressions to share the same Arc after deduplication"- );+ let converter = DeduplicatingProtoConverter {};+ let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;- // Second deserialization- let plan2 = physical_plan_from_bytes_with_proto_converter(- bytes.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &proto_converter,- )?;+ let ctx = SessionContext::new();+ let task_ctx = ctx.task_ctx();+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+ let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;- // Check that the second plan was also deserialized correctly- let projection2 = plan2+ let outer_filter = deserialized_plan+ .downcast_ref::<FilterExec>()+ .expect("Expected outer FilterExec");+ let left_filter = Arc::clone(outer_filter.predicate());+ let projection = outer_filter.children()[0].downcast_ref::<ProjectionExec>().expect("Expected ProjectionExec");- let exprs2: Vec<_> = projection2.expr().iter().collect();- assert_eq!(exprs2.len(), 2);- assert!(- Arc::ptr_eq(&exprs2[0].expr, &exprs2[1].expr),- "Expected both expressions to share the same Arc after deduplication"- );-- // Check that there was no deduplication across deserializations- assert!(- !Arc::ptr_eq(&exprs1[0].expr, &exprs2[0].expr),- "Expected expressions from different deserializations to be different Arcs"- );- assert!(- !Arc::ptr_eq(&exprs1[1].expr, &exprs2[1].expr),- "Expected expressions from different deserializations to be different Arcs"- );+ let data_source = projection+ .input()+ .downcast_ref::<DataSourceExec>()+ .expect("Expected DataSourceExec");+ let scan_config = data_source+ .data_source()+ .downcast_ref::<FileScanConfig>()+ .expect("Expected FileScanConfig");+ let right_filter = scan_config+ .file_source()+ .filter()+ .expect("Expected pushed-down predicate");- Ok(())+ Ok((left_before, right_before, left_filter, right_filter))}-/// Test that deduplication works within direct expression deserialization and that-/// separate deserializations produce independent expressions (no cross-operation sharing).-#[test]-fn test_deduplication_within_expr_deserialization() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));-- // Create a binary expression where both sides are the same Arc- // This allows us to test deduplication within a single deserialization- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- let binary_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(- Arc::clone(&col_expr),- Operator::Plus,- Arc::clone(&col_expr), // Same Arc - will be deduplicated- ));-- let ctx = SessionContext::new();- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};- let task_ctx = ctx.task_ctx();- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);+/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible+/// via the other. This helps assert that referential integrity is maintained after+/// deserializing.+fn assert_dynamic_filter_update_is_visible(+ left_filter: &Arc<dyn PhysicalExpr>,+ right_filter: &Arc<dyn PhysicalExpr>,+) -> Result<()> {+ let left_filter = left_filter+ .downcast_ref::<DynamicFilterPhysicalExpr>()+ .expect("Expected dynamic filter");+ let right_filter = right_filter+ .downcast_ref::<DynamicFilterPhysicalExpr>()+ .expect("Expected dynamic filter");- // Serialize the expression- let proto = proto_converter.physical_expr_to_proto(&binary_expr, &codec)?;+ // Sanity check that the filters have the same generation.+ let original_generation = left_filter.snapshot_generation();+ assert_eq!(original_generation, right_filter.snapshot_generation(),);- // First expression deserialization- let expr1 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;+ left_filter.update(lit(123_i64))?;- // Check that deduplication worked within the deserialization- let binary1 = expr1- .downcast_ref::<BinaryExpr>()- .expect("Expected BinaryExpr");- assert!(- Arc::ptr_eq(binary1.left(), binary1.right()),- "Expected both sides to share the same Arc after deduplication"- );-- // Second expression deserialization- let expr2 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;-- // Check that the second expression was also deserialized correctly- let binary2 = expr2- .downcast_ref::<BinaryExpr>()- .expect("Expected BinaryExpr");- assert!(- Arc::ptr_eq(binary2.left(), binary2.right()),- "Expected both sides to share the same Arc after deduplication"+ // Assert that both generations updated.+ assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);+ assert_eq!(+ left_filter.snapshot_generation(),+ right_filter.snapshot_generation(),);- // Check that there was no deduplication across deserializations- assert!(- !Arc::ptr_eq(binary1.left(), binary2.left()),- "Expected expressions from different deserializations to be different Arcs"- );- assert!(- !Arc::ptr_eq(binary1.right(), binary2.right()),- "Expected expressions from different deserializations to be different Arcs"- );+ // Ensure both filters have the updated expr.+ let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;+ assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);+ assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);Ok(())}-/// Test that session_id rotates between top-level serialization operations.-/// This verifies that each top-level serialization gets a fresh session_id,-/// which prevents cross-process collisions when serialized plans are merged.-#[test]-fn test_session_id_rotation_between_serializations() -> Result<()> {- let field_a = Field::new("a", DataType::Int64, false);- let _schema = Arc::new(Schema::new(vec![field_a]));-- // Create a simple expression- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));+/// Assert that two dynamic filters are equal both structurally (Debug output)+/// and by identity (`expression_id`).+///+fn assert_dynamic_filters_equal(+ expected: &Arc<dyn PhysicalExpr>,+ actual: &Arc<dyn PhysicalExpr>,+) {+ // TODO: Debug currently omits `expression_id` so the id has to be checked+ // separately here. Once plan nodes like `SortExec` / `AggregateExec` /+ // `HashJoinExec` serialize their own dynamic filter, Debug can include+ // `expression_id`.+ //+ // See https://github.com/apache/datafusion/issues/20418+ assert_eq!(expected.expression_id(), actual.expression_id());++ // Structural.+ let expected_dbg = format!("{expected:?}");+ let actual_dbg = format!("{actual:?}");+ if expected_dbg == actual_dbg {+ return;+ }- let codec = DefaultPhysicalExtensionCodec {};- let proto_converter = DeduplicatingProtoConverter {};+ // Note that the `DeduplicatingDeserializer` routes every cache hit through+ // `with_new_children`. This produces an equivalent expression, but with+ // remapped children that are equal to the original. Handle that case here.+ let rewritten = Arc::clone(expected)+ .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())+ .expect("with_new_children on a dynamic filter should not fail");+ assert_eq!(format!("{rewritten:?}"), actual_dbg);+}- // First serialization- let proto1 = proto_converter.physical_expr_to_proto(&col_expr, &codec)?;- let expr_id1 = proto1.expr_id.expect("Expected expr_id to be set");+// Two clones of a dynamic filter expression should be deduped to the exact same expression.+#[test]+fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));+ let filter_expr_1 = make_dynamic_filter();+ let filter_expr_2 = Arc::clone(&filter_expr_1);- // Second serialization with the same converter- // The session_id should have rotated, so the expr_id should be different- // even though we're serializing the same expression (same pointer address)- let proto2 = proto_converter.physical_expr_to_proto(&col_expr, &codec)?;- let expr_id2 = proto2.expr_id.expect("Expected expr_id to be set");+ let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =+ roundtrip_dynamic_filter_expr_pair(+ Arc::clone(&filter_expr_1),+ Arc::clone(&filter_expr_2),+ schema,+ )?;- // The expr_ids should be different because session_id rotated- assert_ne!(- expr_id1, expr_id2,- "Expected different expr_ids due to session_id rotation between serializations"+ // Assert the filters are not modified during roundtrip.+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);+ assert_dynamic_filters_equal(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,);- // Also test that serializing the same expression multiple times within- // the same top-level operation would give the same expr_id (not testable- // here directly since each physical_expr_to_proto is a top-level operation,- // but the deduplication tests verify this indirectly)+ // Assert referential integrity.+ assert_dynamic_filter_update_is_visible(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,+ )?;Ok(())}-/// Test that session_id rotation works correctly with execution plans.-/// This verifies the end-to-end behavior with plan serialization.+/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter+/// with different children.#[test]-fn test_session_id_rotation_with_execution_plans() -> Result<()> {- use datafusion_proto::bytes::physical_plan_to_bytes_with_proto_converter;-- let field_a = Field::new("a", DataType::Int64, false);- let schema = Arc::new(Schema::new(vec![field_a]));+fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {+ let (+ filter_expr_1,+ filter_expr_2,+ filter_expr_1_after_roundtrip,+ filter_expr_2_after_roundtrip,+ ) = roundtrip_dynamic_filter_plan_pair()?;++ // Assert the filters are not modified during roundtrip.+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);++ // Assert referential integrity.+ assert_dynamic_filter_update_is_visible(+ &filter_expr_1_after_roundtrip,+ &filter_expr_2_after_roundtrip,+ )?;- // Create a simple plan- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));- let projection_exprs = vec![ProjectionExpr {- expr: Arc::clone(&col_expr),- alias: "a1".to_string(),- }];- let exec_plan = Arc::new(ProjectionExec::try_new(- projection_exprs.clone(),- Arc::new(EmptyExec::new(Arc::clone(&schema))),- )?);+ Ok(())+}+#[test]+fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {+ let filter_expr = make_dynamic_filter();let codec = DefaultPhysicalExtensionCodec {};let proto_converter = DeduplicatingProtoConverter {};- // First serialization- let bytes1 = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,- &codec,- &proto_converter,- )?;+ let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;+ let expr_id1 = proto_expression_id(&proto1);- // Second serialization with the same converter- let bytes2 = physical_plan_to_bytes_with_proto_converter(- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,- &codec,- &proto_converter,- )?;+ let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;+ let expr_id2 = proto_expression_id(&proto2);- // The serialized bytes should be different due to different session_ids- // (specifically, the expr_id values embedded in the protobuf will differ)- assert_ne!(- bytes1.as_ref(),- bytes2.as_ref(),- "Expected different serialized bytes due to session_id rotation"+ assert_eq!(+ expr_id1, expr_id2,+ "Expected the same dynamic filter expression id across serializations");- // But both should deserialize correctly- let ctx = SessionContext::new();- let deser_converter = DeduplicatingProtoConverter {};-- let plan1 = datafusion_proto::bytes::physical_plan_from_bytes_with_proto_converter(- bytes1.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &deser_converter,- )?;-- let plan2 = datafusion_proto::bytes::physical_plan_from_bytes_with_proto_converter(- bytes2.as_ref(),- ctx.task_ctx().as_ref(),- &codec,- &deser_converter,- )?;-- // Verify both plans have the expected structure- assert_eq!(plan1.schema(), plan2.schema());-Ok(())}