Skip to content

Apache DataFusion SQL Query Engine

proto: serialize and dedupe dynamic filters v2 (#21807)

Jayant Shrivastava948cd09c1060
diff --git a/Cargo.lock b/Cargo.lock
index c18fd20128..af52588e53 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2479,7 +2479,6 @@ dependencies = [
"pbjson 0.9.0",
"pretty_assertions",
"prost",
- "rand 0.9.4",
"serde",
"serde_json",
"tokio",
diff --git a/datafusion/physical-expr-common/src/physical_expr.rs b/datafusion/physical-expr-common/src/physical_expr.rs
index 2fc3bf2afa..6595635024 100644
--- a/datafusion/physical-expr-common/src/physical_expr.rs
+++ b/datafusion/physical-expr-common/src/physical_expr.rs
@@ -161,6 +161,9 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>>;
/// Returns a new PhysicalExpr where all children were replaced by new exprs.
+ ///
+ /// If the implementation returns a [`PhysicalExpr::expression_id`], then
+ /// the identifier should be preserved by the new expression.
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn PhysicalExpr>>,
@@ -444,6 +447,23 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
fn placement(&self) -> ExpressionPlacement {
ExpressionPlacement::KeepInPlace
}
+
+ /// Return a stable, globally-unique identifier for this [`PhysicalExpr`], if it
+ /// has one.
+ ///
+ /// This identifier tracks which expressions which are connected (e.g. `DynamicFilterPhysicalExpr`
+ /// where two expressions may be different but store the same mutable inner state). Tracking
+ /// connected expressions helps preserve referential integrity within plan nodes
+ /// during serialization and deserialization.
+ ///
+ /// This id must be preserved across [`PhysicalExpr::with_new_children`] or any other
+ /// methods which may want to preserve identity.
+ ///
+ /// Default is `None`: the expression has no identity worth preserving across a
+ /// serialization boundary.
+ fn expression_id(&self) -> Option<u64> {
+ None
+ }
}
#[deprecated(
diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rs
index 47398d87e2..2db328377a 100644
--- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs
+++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs
@@ -16,6 +16,7 @@
// under the License.
use parking_lot::RwLock;
+use std::sync::atomic::{AtomicU64, Ordering};
use std::{fmt::Display, hash::Hash, sync::Arc};
use tokio::sync::watch;
@@ -76,21 +77,51 @@ pub struct DynamicFilterPhysicalExpr {
nullable: Arc<RwLock<Option<bool>>>,
}
-#[derive(Debug)]
-struct Inner {
+/// Atomic internal state of a [`DynamicFilterPhysicalExpr`].
+///
+/// `expression_id` lives here because it identifies the actual filter expression `expr`.
+/// Derived `DynamicFilterPhysicalExpr`s (e.g. via [`PhysicalExpr::with_new_children`]) are
+/// the same logical filter and must report the same `expression_id`.
+///
+/// **Warning:** exposed publicly solely so that proto (de)serialization in
+/// `datafusion-proto` can read and rebuild this state. Do not treat this type
+/// or its layout as a stable API.
+#[derive(Clone)]
+pub struct Inner {
+ /// A unique identifier for the expression.
+ pub expression_id: u64,
/// A counter that gets incremented every time the expression is updated so that we can track changes cheaply.
/// This is used for [`PhysicalExpr::snapshot_generation`] to have a cheap check for changes.
- generation: u64,
- expr: Arc<dyn PhysicalExpr>,
+ pub generation: u64,
+ pub expr: Arc<dyn PhysicalExpr>,
/// Flag for quick synchronous check if filter is complete.
/// This is redundant with the watch channel state, but allows us to return immediately
/// from `wait_complete()` without subscribing if already complete.
- is_complete: bool,
+ pub is_complete: bool,
+}
+
+// TODO: Include expression_id in Debug output.
+//
+// See https://github.com/apache/datafusion/issues/20418. Currently, plan nodes
+// like `HashJoinExec`, `AggregateExec`, `SortExec` do not serialize their
+// dynamic filter. They auto-create one on decode with a fresh `expression_id`,
+// so a round-trip Debug comparison would diverge purely on the id even when
+// the rest of the state is preserved. Excluding it from Debug keeps those
+// roundtrip equality assertions meaningful until that work lands.
+impl std::fmt::Debug for Inner {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Inner")
+ .field("generation", &self.generation)
+ .field("expr", &self.expr)
+ .field("is_complete", &self.is_complete)
+ .finish()
+ }
}
impl Inner {
fn new(expr: Arc<dyn PhysicalExpr>) -> Self {
Self {
+ expression_id: EXPR_ID_SOURCE.next(),
// Start with generation 1 which gives us a different result for [`PhysicalExpr::generation`] than the default 0.
// This is not currently used anywhere but it seems useful to have this simple distinction.
generation: 1,
@@ -243,6 +274,8 @@ impl DynamicFilterPhysicalExpr {
let mut current = self.inner.write();
let new_generation = current.generation + 1;
*current = Inner {
+ // Preserve the expression id across updates.
+ expression_id: current.expression_id,
generation: new_generation,
expr: new_expr,
is_complete: current.is_complete,
@@ -346,6 +379,62 @@ impl DynamicFilterPhysicalExpr {
write!(f, " ]")
}
+
+ /// Return the filter's original children (before any remapping).
+ ///
+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.
+ /// Not a stable API.
+ pub fn original_children(&self) -> &[Arc<dyn PhysicalExpr>] {
+ &self.children
+ }
+
+ /// Return the filter's remapped children, if any have been set via
+ /// [`PhysicalExpr::with_new_children`].
+ ///
+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.
+ /// Not a stable API.
+ pub fn remapped_children(&self) -> Option<&[Arc<dyn PhysicalExpr>]> {
+ self.remapped_children.as_deref()
+ }
+
+ /// Rebuild a `DynamicFilterPhysicalExpr` from its stored parts. Used by
+ /// proto deserialization.
+ ///
+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.
+ /// Not a stable API.
+ pub fn from_parts(
+ children: Vec<Arc<dyn PhysicalExpr>>,
+ remapped_children: Option<Vec<Arc<dyn PhysicalExpr>>>,
+ inner: Inner,
+ ) -> Self {
+ let state = if inner.is_complete {
+ FilterState::Complete {
+ generation: inner.generation,
+ }
+ } else {
+ FilterState::InProgress {
+ generation: inner.generation,
+ }
+ };
+ let (state_watch, _) = watch::channel(state);
+
+ Self {
+ children,
+ remapped_children,
+ inner: Arc::new(RwLock::new(inner)),
+ state_watch,
+ data_type: Arc::new(RwLock::new(None)),
+ nullable: Arc::new(RwLock::new(None)),
+ }
+ }
+
+ /// Return a clone of the atomically-captured `Inner` state.
+ ///
+ /// **Warning:** intended only for `datafusion-proto` (de)serialization.
+ /// Not a stable API.
+ pub fn inner(&self) -> Inner {
+ self.inner.read().clone()
+ }
}
impl PhysicalExpr for DynamicFilterPhysicalExpr {
@@ -364,6 +453,7 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {
Ok(Arc::new(Self {
children: self.children.clone(),
remapped_children: Some(children),
+ // Note: expression_id is preserved
inner: Arc::clone(&self.inner),
state_watch: self.state_watch.clone(),
data_type: Arc::clone(&self.data_type),
@@ -444,8 +534,36 @@ impl PhysicalExpr for DynamicFilterPhysicalExpr {
// Return the current generation of the expression.
self.inner.read().generation
}
+
+ fn expression_id(&self) -> Option<u64> {
+ Some(self.inner.read().expression_id)
+ }
}
+/// An atomic counter used to generate monotonic u64 ids.
+struct ExpressionIdAtomicCounter {
+ inner: AtomicU64,
+}
+
+impl ExpressionIdAtomicCounter {
+ const fn new() -> Self {
+ Self {
+ inner: AtomicU64::new(0),
+ }
+ }
+
+ /// Returns a fresh `expression_id` by incrementing the internal counter.
+ fn next(&self) -> u64 {
+ self.inner.fetch_add(1, Ordering::Relaxed)
+ }
+}
+
+/// Process-wide source of deterministic `expression_id`s for [`DynamicFilterPhysicalExpr`].
+///
+/// Currently, no other [`PhysicalExpr`]s use this source. If needed, it can be moved out of this
+/// file and be made public for other expressions to use.
+static EXPR_ID_SOURCE: ExpressionIdAtomicCounter = ExpressionIdAtomicCounter::new();
+
#[cfg(test)]
mod test {
use crate::{
@@ -861,4 +979,130 @@ mod test {
"Hash should be stable after update (identity-based)"
);
}
+
+ /// Verifies that `from_parts` rebuilds a `DynamicFilterPhysicalExpr`
+ /// whose observable state (original children, remapped children,
+ /// expression id, inner generation/expr/is_complete) matches the source
+ /// filter.
+ #[test]
+ fn test_from_parts_preserves_state() {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let col_a = col("a", &schema).unwrap();
+
+ // Create a dynamic filter with children
+ let expr = Arc::new(BinaryExpr::new(
+ Arc::clone(&col_a),
+ datafusion_expr::Operator::Gt,
+ lit(10) as Arc<dyn PhysicalExpr>,
+ ));
+ let filter = DynamicFilterPhysicalExpr::new(
+ vec![Arc::clone(&col_a)],
+ expr as Arc<dyn PhysicalExpr>,
+ );
+
+ // Add remapped children.
+ let reassigned_schema = Arc::new(Schema::new(vec![
+ Field::new("b", DataType::Int32, false),
+ Field::new("a", DataType::Int32, false),
+ ]));
+ let reassigned = reassign_expr_columns(
+ Arc::new(filter) as Arc<dyn PhysicalExpr>,
+ &reassigned_schema,
+ )
+ .expect("reassign_expr_columns should succeed");
+ let reassigned = reassigned
+ .downcast_ref::<DynamicFilterPhysicalExpr>()
+ .expect("Expected dynamic filter after reassignment");
+
+ reassigned
+ .update(lit(42) as Arc<dyn PhysicalExpr>)
+ .expect("Update should succeed");
+ reassigned.mark_complete();
+
+ // Capture the parts and reconstruct. `expression_id` rides in `inner`.
+ let reconstructed = DynamicFilterPhysicalExpr::from_parts(
+ reassigned.original_children().to_vec(),
+ reassigned.remapped_children().map(|r| r.to_vec()),
+ reassigned.inner(),
+ );
+
+ assert_eq!(
+ reassigned.original_children(),
+ reconstructed.original_children(),
+ );
+ assert_eq!(
+ reassigned.remapped_children(),
+ reconstructed.remapped_children(),
+ );
+ assert_eq!(reassigned.expression_id(), reconstructed.expression_id());
+ let r = reassigned.inner();
+ let c = reconstructed.inner();
+ assert_eq!(r.generation, c.generation);
+ assert_eq!(r.is_complete, c.is_complete);
+ assert_eq!(format!("{:?}", r.expr), format!("{:?}", c.expr));
+ }
+
+ #[tokio::test]
+ async fn test_expression_id() {
+ let source_schema =
+ Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
+ let col_a = col("a", &source_schema).unwrap();
+
+ // Create a source filter
+ let source = Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::clone(&col_a)],
+ lit(true) as Arc<dyn PhysicalExpr>,
+ ));
+ let source_clone = Arc::clone(&source);
+
+ // Create a derived filter by reassigning the source filter to a different schema.
+ let derived_schema = Arc::new(Schema::new(vec![
+ Field::new("x", DataType::Int32, false),
+ Field::new("a", DataType::Int32, false),
+ ]));
+ let derived = reassign_expr_columns(
+ Arc::clone(&source) as Arc<dyn PhysicalExpr>,
+ &derived_schema,
+ )
+ .expect("reassign_expr_columns should succeed");
+
+ let derived_expression_id = derived
+ .expression_id()
+ .expect("derived filter should have an expression id");
+ let source_expression_id = source
+ .expression_id()
+ .expect("source filter should have an expression id");
+ let source_clone_expression_id = source_clone
+ .expression_id()
+ .expect("source clone should have an expression id");
+
+ assert_eq!(
+ source_clone_expression_id, source_expression_id,
+ "cloned filter should preserve its expression id",
+ );
+
+ assert_eq!(
+ derived_expression_id, source_expression_id,
+ "derived filters should carry forward the source expression id",
+ );
+
+ // `update()` rewrites the entire `Inner` struct in place; pin down
+ // that the rewrite preserves `expression_id`.
+ source
+ .update(lit(99) as Arc<dyn PhysicalExpr>)
+ .expect("update should succeed");
+ assert_eq!(
+ source.expression_id().unwrap(),
+ source_expression_id,
+ "update() must not change expression_id",
+ );
+
+ // `mark_complete()` also touches `Inner`; same invariant.
+ source.mark_complete();
+ assert_eq!(
+ source.expression_id().unwrap(),
+ source_expression_id,
+ "mark_complete() must not change expression_id",
+ );
+ }
}
diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs
index bb6d9ca9c9..7cf874c448 100644
--- a/datafusion/physical-expr/src/expressions/mod.rs
+++ b/datafusion/physical-expr/src/expressions/mod.rs
@@ -45,7 +45,7 @@ pub use case::{CaseExpr, case};
pub use cast::{CastExpr, cast};
pub use column::{Column, col, with_new_schema};
pub use datafusion_expr::utils::format_state_name;
-pub use dynamic_filters::DynamicFilterPhysicalExpr;
+pub use dynamic_filters::{DynamicFilterPhysicalExpr, Inner as DynamicFilterInner};
pub use in_list::{InListExpr, in_list};
pub use is_not_null::{IsNotNullExpr, is_not_null};
pub use is_null::{IsNullExpr, is_null};
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index baa7dd8770..4484846813 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -66,7 +66,6 @@ datafusion-proto-common = { workspace = true }
object_store = { workspace = true }
pbjson = { workspace = true, optional = true }
prost = { workspace = true }
-rand = { workspace = true }
serde = { version = "1.0", optional = true }
serde_json = { workspace = true, optional = true }
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
index 5cd57b9f98..511e8eb1b0 100644
--- a/datafusion/proto/proto/datafusion.proto
+++ b/datafusion/proto/proto/datafusion.proto
@@ -889,11 +889,9 @@ message PhysicalExprNode {
reserved 17;
// Unique identifier for this expression to do deduplication during deserialization.
- // When serializing, this is set to a unique identifier for each combination of
- // expression, process and serialization run.
- // When deserializing, if this ID has been seen before, the cached Arc is returned
- // instead of creating a new one, enabling reconstruction of referential integrity
- // across serde roundtrips.
+ // When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing,
+ // this id is used by the `DeduplicatingProtoConverter` to preserve referential
+ // integrity across serde roundtrips for different expressions with the same id.
optional uint64 expr_id = 30;
oneof ExprType {
@@ -935,9 +933,19 @@ message PhysicalExprNode {
PhysicalHashExprNode hash_expr = 21;
PhysicalScalarSubqueryExprNode scalar_subquery = 22;
+
+ PhysicalDynamicFilterNode dynamic_filter = 23;
}
}
+message PhysicalDynamicFilterNode {
+ repeated PhysicalExprNode children = 1;
+ repeated PhysicalExprNode remapped_children = 2;
+ uint64 generation = 3;
+ PhysicalExprNode inner_expr = 4;
+ bool is_complete = 5;
+}
+
message PhysicalScalarUdfNode {
string name = 1;
repeated PhysicalExprNode args = 2;
@@ -1510,4 +1518,4 @@ message PhysicalScalarSubqueryExprNode {
datafusion_common.ArrowType data_type = 1;
bool nullable = 2;
uint32 index = 3;
-}
\ No newline at end of file
+}
diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs
index 74f7b3b1c3..c05d3283ea 100644
--- a/datafusion/proto/src/generated/pbjson.rs
+++ b/datafusion/proto/src/generated/pbjson.rs
@@ -16702,6 +16702,172 @@ impl<'de> serde::Deserialize<'de> for PhysicalDateTimeIntervalExprNode {
deserializer.deserialize_struct("datafusion.PhysicalDateTimeIntervalExprNode", FIELDS, GeneratedVisitor)
}
}
+impl serde::Serialize for PhysicalDynamicFilterNode {
+ #[allow(deprecated)]
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+ where
+ S: serde::Serializer,
+ {
+ use serde::ser::SerializeStruct;
+ let mut len = 0;
+ if !self.children.is_empty() {
+ len += 1;
+ }
+ if !self.remapped_children.is_empty() {
+ len += 1;
+ }
+ if self.generation != 0 {
+ len += 1;
+ }
+ if self.inner_expr.is_some() {
+ len += 1;
+ }
+ if self.is_complete {
+ len += 1;
+ }
+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalDynamicFilterNode", len)?;
+ if !self.children.is_empty() {
+ struct_ser.serialize_field("children", &self.children)?;
+ }
+ if !self.remapped_children.is_empty() {
+ struct_ser.serialize_field("remappedChildren", &self.remapped_children)?;
+ }
+ if self.generation != 0 {
+ #[allow(clippy::needless_borrow)]
+ #[allow(clippy::needless_borrows_for_generic_args)]
+ struct_ser.serialize_field("generation", ToString::to_string(&self.generation).as_str())?;
+ }
+ if let Some(v) = self.inner_expr.as_ref() {
+ struct_ser.serialize_field("innerExpr", v)?;
+ }
+ if self.is_complete {
+ struct_ser.serialize_field("isComplete", &self.is_complete)?;
+ }
+ struct_ser.end()
+ }
+}
+impl<'de> serde::Deserialize<'de> for PhysicalDynamicFilterNode {
+ #[allow(deprecated)]
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ const FIELDS: &[&str] = &[
+ "children",
+ "remapped_children",
+ "remappedChildren",
+ "generation",
+ "inner_expr",
+ "innerExpr",
+ "is_complete",
+ "isComplete",
+ ];
+
+ #[allow(clippy::enum_variant_names)]
+ enum GeneratedField {
+ Children,
+ RemappedChildren,
+ Generation,
+ InnerExpr,
+ IsComplete,
+ }
+ impl<'de> serde::Deserialize<'de> for GeneratedField {
+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ struct GeneratedVisitor;
+
+ impl serde::de::Visitor<'_> for GeneratedVisitor {
+ type Value = GeneratedField;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(formatter, "expected one of: {:?}", &FIELDS)
+ }
+
+ #[allow(unused_variables)]
+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>
+ where
+ E: serde::de::Error,
+ {
+ match value {
+ "children" => Ok(GeneratedField::Children),
+ "remappedChildren" | "remapped_children" => Ok(GeneratedField::RemappedChildren),
+ "generation" => Ok(GeneratedField::Generation),
+ "innerExpr" | "inner_expr" => Ok(GeneratedField::InnerExpr),
+ "isComplete" | "is_complete" => Ok(GeneratedField::IsComplete),
+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),
+ }
+ }
+ }
+ deserializer.deserialize_identifier(GeneratedVisitor)
+ }
+ }
+ struct GeneratedVisitor;
+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {
+ type Value = PhysicalDynamicFilterNode;
+
+ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ formatter.write_str("struct datafusion.PhysicalDynamicFilterNode")
+ }
+
+ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalDynamicFilterNode, V::Error>
+ where
+ V: serde::de::MapAccess<'de>,
+ {
+ let mut children__ = None;
+ let mut remapped_children__ = None;
+ let mut generation__ = None;
+ let mut inner_expr__ = None;
+ let mut is_complete__ = None;
+ while let Some(k) = map_.next_key()? {
+ match k {
+ GeneratedField::Children => {
+ if children__.is_some() {
+ return Err(serde::de::Error::duplicate_field("children"));
+ }
+ children__ = Some(map_.next_value()?);
+ }
+ GeneratedField::RemappedChildren => {
+ if remapped_children__.is_some() {
+ return Err(serde::de::Error::duplicate_field("remappedChildren"));
+ }
+ remapped_children__ = Some(map_.next_value()?);
+ }
+ GeneratedField::Generation => {
+ if generation__.is_some() {
+ return Err(serde::de::Error::duplicate_field("generation"));
+ }
+ generation__ =
+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)
+ ;
+ }
+ GeneratedField::InnerExpr => {
+ if inner_expr__.is_some() {
+ return Err(serde::de::Error::duplicate_field("innerExpr"));
+ }
+ inner_expr__ = map_.next_value()?;
+ }
+ GeneratedField::IsComplete => {
+ if is_complete__.is_some() {
+ return Err(serde::de::Error::duplicate_field("isComplete"));
+ }
+ is_complete__ = Some(map_.next_value()?);
+ }
+ }
+ }
+ Ok(PhysicalDynamicFilterNode {
+ children: children__.unwrap_or_default(),
+ remapped_children: remapped_children__.unwrap_or_default(),
+ generation: generation__.unwrap_or_default(),
+ inner_expr: inner_expr__,
+ is_complete: is_complete__.unwrap_or_default(),
+ })
+ }
+ }
+ deserializer.deserialize_struct("datafusion.PhysicalDynamicFilterNode", FIELDS, GeneratedVisitor)
+ }
+}
impl serde::Serialize for PhysicalExprNode {
#[allow(deprecated)]
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
@@ -16784,6 +16950,9 @@ impl serde::Serialize for PhysicalExprNode {
physical_expr_node::ExprType::ScalarSubquery(v) => {
struct_ser.serialize_field("scalarSubquery", v)?;
}
+ physical_expr_node::ExprType::DynamicFilter(v) => {
+ struct_ser.serialize_field("dynamicFilter", v)?;
+ }
}
}
struct_ser.end()
@@ -16832,6 +17001,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {
"hashExpr",
"scalar_subquery",
"scalarSubquery",
+ "dynamic_filter",
+ "dynamicFilter",
];
#[allow(clippy::enum_variant_names)]
@@ -16857,6 +17028,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {
UnknownColumn,
HashExpr,
ScalarSubquery,
+ DynamicFilter,
}
impl<'de> serde::Deserialize<'de> for GeneratedField {
fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>
@@ -16899,6 +17071,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {
"unknownColumn" | "unknown_column" => Ok(GeneratedField::UnknownColumn),
"hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),
"scalarSubquery" | "scalar_subquery" => Ok(GeneratedField::ScalarSubquery),
+ "dynamicFilter" | "dynamic_filter" => Ok(GeneratedField::DynamicFilter),
_ => Err(serde::de::Error::unknown_field(value, FIELDS)),
}
}
@@ -17068,6 +17241,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {
return Err(serde::de::Error::duplicate_field("scalarSubquery"));
}
expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::ScalarSubquery)
+;
+ }
+ GeneratedField::DynamicFilter => {
+ if expr_type__.is_some() {
+ return Err(serde::de::Error::duplicate_field("dynamicFilter"));
+ }
+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::DynamicFilter)
;
}
}
diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs
index 3db2f2de8e..af9b1404bb 100644
--- a/datafusion/proto/src/generated/prost.rs
+++ b/datafusion/proto/src/generated/prost.rs
@@ -1327,16 +1327,14 @@ pub struct PhysicalExtensionNode {
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PhysicalExprNode {
/// Unique identifier for this expression to do deduplication during deserialization.
- /// When serializing, this is set to a unique identifier for each combination of
- /// expression, process and serialization run.
- /// When deserializing, if this ID has been seen before, the cached Arc is returned
- /// instead of creating a new one, enabling reconstruction of referential integrity
- /// across serde roundtrips.
+ /// When serializing, this is set via `PhysicalExpr::expression_id`. When deserializing,
+ /// this id is used by the `DeduplicatingProtoConverter` to preserve referential
+ /// integrity across serde roundtrips for different expressions with the same id.
#[prost(uint64, optional, tag = "30")]
pub expr_id: ::core::option::Option<u64>,
#[prost(
oneof = "physical_expr_node::ExprType",
- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22"
+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21, 22, 23"
)]
pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,
}
@@ -1391,9 +1389,24 @@ pub mod physical_expr_node {
HashExpr(super::PhysicalHashExprNode),
#[prost(message, tag = "22")]
ScalarSubquery(super::PhysicalScalarSubqueryExprNode),
+ #[prost(message, tag = "23")]
+ DynamicFilter(::prost::alloc::boxed::Box<super::PhysicalDynamicFilterNode>),
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct PhysicalDynamicFilterNode {
+ #[prost(message, repeated, tag = "1")]
+ pub children: ::prost::alloc::vec::Vec<PhysicalExprNode>,
+ #[prost(message, repeated, tag = "2")]
+ pub remapped_children: ::prost::alloc::vec::Vec<PhysicalExprNode>,
+ #[prost(uint64, tag = "3")]
+ pub generation: u64,
+ #[prost(message, optional, boxed, tag = "4")]
+ pub inner_expr: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalExprNode>>,
+ #[prost(bool, tag = "5")]
+ pub is_complete: bool,
+}
+#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PhysicalScalarUdfNode {
#[prost(string, tag = "1")]
pub name: ::prost::alloc::string::String,
diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rs
index 732676941b..43ebf04743 100644
--- a/datafusion/proto/src/physical_plan/from_proto.rs
+++ b/datafusion/proto/src/physical_plan/from_proto.rs
@@ -60,6 +60,9 @@ use super::{
use crate::logical_plan::{self};
use crate::protobuf::physical_expr_node::ExprType;
use crate::{convert_required, protobuf};
+use datafusion_physical_expr::expressions::{
+ DynamicFilterInner, DynamicFilterPhysicalExpr,
+};
impl From<&protobuf::PhysicalColumn> for Column {
fn from(c: &protobuf::PhysicalColumn) -> Column {
@@ -511,6 +514,53 @@ pub fn parse_physical_expr_with_converter(
results.clone(),
))
}
+ ExprType::DynamicFilter(dynamic_filter) => {
+ let children = parse_physical_exprs(
+ &dynamic_filter.children,
+ ctx,
+ input_schema,
+ proto_converter,
+ )?;
+
+ let remapped_children = if !dynamic_filter.remapped_children.is_empty() {
+ Some(parse_physical_exprs(
+ &dynamic_filter.remapped_children,
+ ctx,
+ input_schema,
+ proto_converter,
+ )?)
+ } else {
+ None
+ };
+
+ let inner_expr = parse_required_physical_expr(
+ dynamic_filter.inner_expr.as_deref(),
+ ctx,
+ "inner_expr",
+ input_schema,
+ proto_converter,
+ )?;
+
+ let expression_id = proto.expr_id.ok_or_else(|| {
+ proto_error(
+ "DynamicFilterPhysicalExpr requires PhysicalExprNode.expr_id \
+ to be set by the serializer",
+ )
+ })?;
+
+ let base_filter: Arc<dyn PhysicalExpr> =
+ Arc::new(DynamicFilterPhysicalExpr::from_parts(
+ children,
+ remapped_children,
+ DynamicFilterInner {
+ expression_id,
+ generation: dynamic_filter.generation,
+ expr: inner_expr,
+ is_complete: dynamic_filter.is_complete,
+ },
+ ));
+ base_filter
+ }
ExprType::Extension(extension) => {
let inputs: Vec<Arc<dyn PhysicalExpr>> = extension
.inputs
diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs
index d4a83a76f3..5172a552fa 100644
--- a/datafusion/proto/src/physical_plan/mod.rs
+++ b/datafusion/proto/src/physical_plan/mod.rs
@@ -19,7 +19,6 @@ use std::any::Any;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fmt::Debug;
-use std::hash::{DefaultHasher, Hash, Hasher};
use std::sync::Arc;
use arrow::compute::SortOptions;
@@ -3888,83 +3887,11 @@ impl PhysicalProtoConverterExtension for DefaultPhysicalProtoConverter {
}
}
-/// Internal serializer that adds expr_id to expressions.
-/// Created fresh for each serialization operation.
-struct DeduplicatingSerializer {
- /// Random salt combined with pointer addresses and process ID to create globally unique expr_ids.
- session_id: u64,
-}
-
-impl DeduplicatingSerializer {
- fn new() -> Self {
- Self {
- session_id: rand::random(),
- }
- }
-}
-
-impl PhysicalProtoConverterExtension for DeduplicatingSerializer {
- fn proto_to_execution_plan(
- &self,
- _proto: &protobuf::PhysicalPlanNode,
- _ctx: &PhysicalPlanDecodeContext<'_>,
- ) -> Result<Arc<dyn ExecutionPlan>> {
- internal_err!("DeduplicatingSerializer cannot deserialize execution plans")
- }
-
- fn execution_plan_to_proto(
- &self,
- plan: &Arc<dyn ExecutionPlan>,
- codec: &dyn PhysicalExtensionCodec,
- ) -> Result<protobuf::PhysicalPlanNode>
- where
- Self: Sized,
- {
- protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
- Arc::clone(plan),
- codec,
- self,
- )
- }
-
- fn proto_to_physical_expr(
- &self,
- _proto: &protobuf::PhysicalExprNode,
- _input_schema: &Schema,
- _ctx: &PhysicalPlanDecodeContext<'_>,
- ) -> Result<Arc<dyn PhysicalExpr>>
- where
- Self: Sized,
- {
- internal_err!("DeduplicatingSerializer cannot deserialize physical expressions")
- }
-
- fn physical_expr_to_proto(
- &self,
- expr: &Arc<dyn PhysicalExpr>,
- codec: &dyn PhysicalExtensionCodec,
- ) -> Result<protobuf::PhysicalExprNode> {
- let mut proto = serialize_physical_expr_with_converter(expr, codec, self)?;
-
- // Hash session_id, pointer address, and process ID together to create expr_id.
- // - session_id: random per serializer, prevents collisions when merging serializations
- // - ptr: unique address per Arc within a process
- // - pid: prevents collisions if serializer is shared across processes
- let mut hasher = DefaultHasher::new();
- self.session_id.hash(&mut hasher);
- (Arc::as_ptr(expr) as *const () as u64).hash(&mut hasher);
- std::process::id().hash(&mut hasher);
- proto.expr_id = Some(hasher.finish());
-
- Ok(proto)
- }
-}
-
-/// Internal deserializer that caches expressions by expr_id.
-/// Created fresh for each deserialization operation.
+/// Internal deserializer that caches expressions by their `expression_id()` so
+/// multiple occurrences of the same expression are deduped.
#[derive(Default)]
struct DeduplicatingDeserializer {
- /// Cache mapping expr_id to deserialized expressions.
+ /// Cache mapping expression_id to deserialized expressions.
cache: RefCell<HashMap<u64, Arc<dyn PhysicalExpr>>>,
}
@@ -3997,19 +3924,30 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {
where
Self: Sized,
{
- if let Some(expr_id) = proto.expr_id {
- // Check cache first
- if let Some(cached) = self.cache.borrow().get(&expr_id) {
- return Ok(Arc::clone(cached));
- }
- // Deserialize and cache
- let expr =
- parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;
- self.cache.borrow_mut().insert(expr_id, Arc::clone(&expr));
- Ok(expr)
- } else {
- parse_physical_expr_with_converter(proto, input_schema, ctx, self)
+ // `expr_id` is the generic identity slot on `PhysicalExprNode`.
+ // The default serializer populates it from `PhysicalExpr::expression_id`.
+ // A missing id means this expression type doesn't participate in deduping.
+ let Some(id) = proto.expr_id else {
+ return parse_physical_expr_with_converter(proto, input_schema, ctx, self);
+ };
+
+ let parsed = parse_physical_expr_with_converter(proto, input_schema, ctx, self)?;
+
+ let mut cache = self.cache.borrow_mut();
+ if let Some(cached) = cache.get(&id) {
+ // Since expressions may manage their own internal state when deriving
+ // expressions via `with_new_children`, we use `with_new_children`
+ // to opt into the same behavior.
+ //
+ // For example, one `DynamicFilterPhysicalExpr` may be derived from
+ // another resulting in shared references. Using `with_new_children`
+ // is meant to preserve those references.
+ let children: Vec<_> = parsed.children().into_iter().cloned().collect();
+ return Arc::clone(cached).with_new_children(children);
}
+
+ cache.insert(id, Arc::clone(&parsed));
+ Ok(parsed)
}
fn physical_expr_to_proto(
@@ -4021,19 +3959,12 @@ impl PhysicalProtoConverterExtension for DeduplicatingDeserializer {
}
}
-/// A proto converter that adds expression deduplication during serialization
-/// and deserialization.
-///
-/// During serialization, each expression's Arc pointer address is XORed with a
-/// random session_id to create a salted `expr_id`. This prevents cross-process
-/// collisions when serialized plans are merged.
+/// A proto converter that deduplicates [`PhysicalExpr`] by [`PhysicalExpr::expression_id`].
+/// This helps preserve referential integrity when deserializing [`ExecutionPlan`]s
+/// which may contain multiple occurrences of the same [`PhysicalExpr`] (ex. when
+/// [`DynamicFilterPhysicalExpr`] are pushed down, it is important to preserve
+/// referential integrity).
///
-/// During deserialization, expressions with the same `expr_id` share the same
-/// Arc, reducing memory usage for plans with duplicate expressions (e.g., large
-/// IN lists) and supporting correctly linking [`DynamicFilterPhysicalExpr`] instances.
-///
-/// This converter is stateless - it creates internal serializers/deserializers
-/// on demand for each operation.
///
/// [`DynamicFilterPhysicalExpr`]: https://docs.rs/datafusion-physical-expr/latest/datafusion_physical_expr/expressions/struct.DynamicFilterPhysicalExpr.html
#[derive(Debug, Default, Clone, Copy)]
@@ -4057,11 +3988,10 @@ impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {
where
Self: Sized,
{
- let serializer = DeduplicatingSerializer::new();
protobuf::PhysicalPlanNode::try_from_physical_plan_with_converter(
Arc::clone(plan),
codec,
- &serializer,
+ self,
)
}
@@ -4083,8 +4013,7 @@ impl PhysicalProtoConverterExtension for DeduplicatingProtoConverter {
expr: &Arc<dyn PhysicalExpr>,
codec: &dyn PhysicalExtensionCodec,
) -> Result<protobuf::PhysicalExprNode> {
- let serializer = DeduplicatingSerializer::new();
- serializer.physical_expr_to_proto(expr, codec)
+ serialize_physical_expr_with_converter(expr, codec, self)
}
}
diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs
index cb69b913b5..83c11cfc6b 100644
--- a/datafusion/proto/src/physical_plan/to_proto.rs
+++ b/datafusion/proto/src/physical_plan/to_proto.rs
@@ -34,11 +34,11 @@ use datafusion_expr::WindowFrame;
use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::scalar_subquery::ScalarSubqueryExpr;
use datafusion_physical_expr::window::{SlidingAggregateWindowExpr, StandardWindowExpr};
-use datafusion_physical_expr_common::physical_expr::snapshot_physical_expr;
use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;
use datafusion_physical_plan::expressions::{
- BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,
- LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,
+ BinaryExpr, CaseExpr, CastExpr, Column, DynamicFilterPhysicalExpr, InListExpr,
+ IsNotNullExpr, IsNullExpr, LikeExpr, Literal, NegativeExpr, NotExpr, TryCastExpr,
+ UnKnownColumn,
};
use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};
use datafusion_physical_plan::udaf::AggregateFunctionExpr;
@@ -258,10 +258,8 @@ pub fn serialize_physical_expr_with_converter(
codec: &dyn PhysicalExtensionCodec,
proto_converter: &dyn PhysicalProtoConverterExtension,
) -> Result<protobuf::PhysicalExprNode> {
- // Snapshot the expr in case it has dynamic predicate state so
- // it can be serialized
- let expr = snapshot_physical_expr(Arc::clone(value))?;
-
+ let expr = value.as_ref();
+ let expr_id = value.expression_id();
// HashTableLookupExpr is used for dynamic filter pushdown in hash joins.
// It contains an Arc<dyn JoinHashMapType> (the build-side hash table) which
// cannot be serialized - the hash table is a runtime structure built during
@@ -282,14 +280,14 @@ pub fn serialize_physical_expr_with_converter(
)),
};
return Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Literal(value)),
});
}
if let Some(expr) = expr.downcast_ref::<Column>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Column(
protobuf::PhysicalColumn {
name: expr.name().to_string(),
@@ -299,7 +297,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<UnKnownColumn>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::UnknownColumn(
protobuf::UnknownColumn {
name: expr.name().to_string(),
@@ -341,14 +339,14 @@ pub fn serialize_physical_expr_with_converter(
});
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::BinaryExpr(
binary_expr,
)),
})
} else if let Some(expr) = expr.downcast_ref::<CaseExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(
protobuf::physical_expr_node::ExprType::Case(
Box::new(
@@ -391,7 +389,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<NotExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::NotExpr(Box::new(
protobuf::PhysicalNot {
expr: Some(Box::new(
@@ -402,7 +400,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<IsNullExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::IsNullExpr(
Box::new(protobuf::PhysicalIsNull {
expr: Some(Box::new(
@@ -413,7 +411,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<IsNotNullExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::IsNotNullExpr(
Box::new(protobuf::PhysicalIsNotNull {
expr: Some(Box::new(
@@ -424,7 +422,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<InListExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::InList(Box::new(
protobuf::PhysicalInListNode {
expr: Some(Box::new(
@@ -437,7 +435,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<NegativeExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Negative(Box::new(
protobuf::PhysicalNegativeNode {
expr: Some(Box::new(
@@ -448,14 +446,14 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(lit) = expr.downcast_ref::<Literal>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Literal(
lit.value().try_into()?,
)),
})
} else if let Some(cast) = expr.downcast_ref::<CastExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Cast(Box::new(
protobuf::PhysicalCastNode {
expr: Some(Box::new(
@@ -467,7 +465,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(cast) = expr.downcast_ref::<TryCastExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::TryCast(Box::new(
protobuf::PhysicalTryCastNode {
expr: Some(Box::new(
@@ -481,7 +479,7 @@ pub fn serialize_physical_expr_with_converter(
let mut buf = Vec::new();
codec.try_encode_udf(expr.fun(), &mut buf)?;
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarUdf(
protobuf::PhysicalScalarUdfNode {
name: expr.name().to_string(),
@@ -498,7 +496,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<LikeExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::LikeExpr(Box::new(
protobuf::PhysicalLikeExprNode {
negated: expr.negated(),
@@ -514,7 +512,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<HashExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(
protobuf::PhysicalHashExprNode {
on_columns: serialize_physical_exprs(
@@ -529,7 +527,7 @@ pub fn serialize_physical_expr_with_converter(
})
} else if let Some(expr) = expr.downcast_ref::<ScalarSubqueryExpr>() {
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::ScalarSubquery(
protobuf::PhysicalScalarSubqueryExprNode {
data_type: Some(expr.data_type().try_into()?),
@@ -538,6 +536,39 @@ pub fn serialize_physical_expr_with_converter(
},
)),
})
+ } else if let Some(df) = expr.downcast_ref::<DynamicFilterPhysicalExpr>() {
+ let children = df
+ .original_children()
+ .iter()
+ .map(|child| proto_converter.physical_expr_to_proto(child, codec))
+ .collect::<Result<Vec<_>>>()?;
+
+ let remapped_children = if let Some(remapped) = df.remapped_children() {
+ remapped
+ .iter()
+ .map(|child| proto_converter.physical_expr_to_proto(child, codec))
+ .collect::<Result<Vec<_>>>()?
+ } else {
+ vec![]
+ };
+
+ // Atomic snapshot of inner state.
+ let inner = df.inner();
+ let inner_expr =
+ Box::new(proto_converter.physical_expr_to_proto(&inner.expr, codec)?);
+
+ Ok(protobuf::PhysicalExprNode {
+ expr_id,
+ expr_type: Some(protobuf::physical_expr_node::ExprType::DynamicFilter(
+ Box::new(protobuf::PhysicalDynamicFilterNode {
+ children,
+ remapped_children,
+ generation: inner.generation,
+ inner_expr: Some(inner_expr),
+ is_complete: inner.is_complete,
+ }),
+ )),
+ })
} else {
let mut buf: Vec<u8> = vec![];
match codec.try_encode_expr(value, &mut buf) {
@@ -548,7 +579,7 @@ pub fn serialize_physical_expr_with_converter(
.map(|e| proto_converter.physical_expr_to_proto(e, codec))
.collect::<Result<_>>()?;
Ok(protobuf::PhysicalExprNode {
- expr_id: None,
+ expr_id,
expr_type: Some(protobuf::physical_expr_node::ExprType::Extension(
protobuf::PhysicalExtensionExprNode { expr: buf, inputs },
)),
diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
index 8465c2a196..fa342ae907 100644
--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs
@@ -132,6 +132,8 @@ use crate::cases::{
CustomUDWF, CustomUDWFNode, MyAggregateUDF, MyAggregateUdfNode, MyRegexUdf,
MyRegexUdfNode,
};
+use datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr;
+use datafusion_physical_expr::utils::reassign_expr_columns;
/// Perform a serde roundtrip and assert that the string representation of the before and after plans
/// are identical. Note that this often isn't sufficient to guarantee that no information is
@@ -2685,431 +2687,294 @@ fn roundtrip_call_null_scalar_struct_dict() -> Result<()> {
roundtrip_test(filter)
}
-/// Test that expression deduplication works during deserialization.
-/// When the same expression Arc is serialized multiple times, it should be
-/// deduplicated on deserialization (sharing the same Arc).
-#[test]
-fn test_expression_deduplication() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
-
- // Create a shared expression that will be used multiple times
- let shared_col: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
-
- // Create an InList expression that uses the same column Arc multiple times
- // This simulates a real-world scenario where expressions are shared
- let in_list_expr = in_list(
- Arc::clone(&shared_col),
- vec![lit(1i64), lit(2i64), lit(3i64)],
- &false,
- &schema,
- )?;
-
- // Create a binary expression that uses the shared column and the in_list result
- let binary_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- Arc::clone(&shared_col),
- Operator::Eq,
- lit(42i64),
- ));
-
- // Create a plan that has both expressions (they share the `shared_col` Arc)
- let input = Arc::new(EmptyExec::new(schema.clone()));
- let filter = FilterExecBuilder::new(in_list_expr, input).build()?;
- let projection_exprs = vec![ProjectionExpr {
- expr: binary_expr,
- alias: "result".to_string(),
- }];
- let exec_plan =
- Arc::new(ProjectionExec::try_new(projection_exprs, Arc::new(filter))?);
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
-
- // Perform roundtrip
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,
- &codec,
- &proto_converter,
- )?;
-
- // Create a new converter for deserialization (fresh cache)
- let deser_converter = DeduplicatingProtoConverter {};
- let result_plan = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &deser_converter,
- )?;
-
- // Verify the plan structure is correct
- pretty_assertions::assert_eq!(format!("{exec_plan:?}"), format!("{result_plan:?}"));
-
- Ok(())
+/// Create a [`DynamicFilterPhysicalExpr`] with child column expression "a" @ index 0.
+fn make_dynamic_filter() -> Arc<dyn PhysicalExpr> {
+ Arc::new(DynamicFilterPhysicalExpr::new(
+ vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>],
+ lit(true),
+ )) as Arc<dyn PhysicalExpr>
}
-/// Test that expression deduplication correctly shares Arcs for identical expressions.
-/// This test verifies the core deduplication behavior.
-#[test]
-fn test_expression_deduplication_arc_sharing() -> Result<()> {
- use datafusion_proto::bytes::{
- physical_plan_from_bytes_with_proto_converter,
- physical_plan_to_bytes_with_proto_converter,
- };
-
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
-
- // Create a column expression
- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
-
- // Create a projection that uses the SAME Arc twice
- // After roundtrip, both should point to the same Arc
- let projection_exprs = vec![
- ProjectionExpr {
- expr: Arc::clone(&col_expr),
- alias: "a1".to_string(),
- },
- ProjectionExpr {
- expr: Arc::clone(&col_expr), // Same Arc!
- alias: "a2".to_string(),
- },
- ];
-
- let input = Arc::new(EmptyExec::new(schema));
- let exec_plan = Arc::new(ProjectionExec::try_new(projection_exprs, input)?);
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
-
- // Serialize
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,
- &codec,
- &proto_converter,
- )?;
-
- // Deserialize with a fresh converter
- let deser_converter = DeduplicatingProtoConverter {};
- let result_plan = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &deser_converter,
- )?;
-
- // Get the projection from the result
- let projection = result_plan
- .downcast_ref::<ProjectionExec>()
- .expect("Expected ProjectionExec");
-
- let exprs: Vec<_> = projection.expr().iter().collect();
- assert_eq!(exprs.len(), 2);
-
- // The key test: both expressions should point to the same Arc after deduplication
- // This is because they were the same Arc before serialization
- assert!(
- Arc::ptr_eq(&exprs[0].expr, &exprs[1].expr),
- "Expected both expressions to share the same Arc after deduplication"
- );
-
- Ok(())
+/// Update a [`DynamicFilterPhysicalExpr`]'s children to support child schema "b" @ 0, "a" @ 1.
+fn make_reassigned_dynamic_filter(
+ filter: Arc<dyn PhysicalExpr>,
+) -> Result<(Arc<Schema>, Arc<dyn PhysicalExpr>)> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("b", DataType::Int64, false),
+ Field::new("a", DataType::Int64, false),
+ ]));
+ let reassigned = reassign_expr_columns(filter, &schema)?;
+ Ok((schema, reassigned))
}
-/// Test backward compatibility: protos without expr_id should still deserialize correctly.
-#[test]
-fn test_backward_compatibility_no_expr_id() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
+/// Extract the expression id from a [`PhysicalExpr`] proto. Populated by the
+/// default serializer from `PhysicalExpr::expression_id`.
+fn proto_expression_id(expr: &PhysicalExprNode) -> u64 {
+ expr.expr_id
+ .expect("expected PhysicalExprNode.expr_id to be populated")
+}
- // Manually create a proto without expr_id set
- let proto = PhysicalExprNode {
- expr_id: None, // Simulating old proto without this field
- expr_type: Some(
- datafusion_proto::protobuf::physical_expr_node::ExprType::Column(
- datafusion_proto::protobuf::PhysicalColumn {
- name: "a".to_string(),
- index: 0,
- },
- ),
- ),
- };
+/// Roundtrip a single physical expression shaped like so:
+///
+/// ```text
+/// BinaryExpr(AND)
+/// / \
+/// filter_expr_1 filter_expr_2
+/// ```
+///
+/// Returns filter_expr_1 and filter_expr_2 after deserialization.
+fn roundtrip_dynamic_filter_expr_pair(
+ filter_expr_1: Arc<dyn PhysicalExpr>,
+ filter_expr_2: Arc<dyn PhysicalExpr>,
+ schema: Arc<Schema>,
+) -> Result<(Arc<dyn PhysicalExpr>, Arc<dyn PhysicalExpr>)> {
+ let pair_expr = Arc::new(BinaryExpr::new(
+ Arc::clone(&filter_expr_1),
+ Operator::And,
+ Arc::clone(&filter_expr_2),
+ )) as Arc<dyn PhysicalExpr>;
- let ctx = SessionContext::new();
let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DefaultPhysicalProtoConverter {};
+ let converter = DeduplicatingProtoConverter {};
+ let proto = converter.physical_expr_to_proto(&pair_expr, &codec)?;
+ let ctx = SessionContext::new();
let task_ctx = ctx.task_ctx();
let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let deserialized_expr =
+ converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
- // Should deserialize without error
- let result = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
-
- // Verify the result is correct
- let col = result.downcast_ref::<Column>().expect("Expected Column");
- assert_eq!(col.name(), "a");
- assert_eq!(col.index(), 0);
+ let binary = deserialized_expr
+ .downcast_ref::<BinaryExpr>()
+ .expect("Expected BinaryExpr");
- Ok(())
+ Ok((Arc::clone(binary.left()), Arc::clone(binary.right())))
}
-/// Test that deduplication works within a single plan deserialization and that
-/// separate deserializations produce independent expressions (no cross-operation sharing).
-#[test]
-fn test_deduplication_within_plan_deserialization() -> Result<()> {
- use datafusion_proto::bytes::{
- physical_plan_from_bytes_with_proto_converter,
- physical_plan_to_bytes_with_proto_converter,
- };
-
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
+/// Roundtrip an execution plan shaped like so:
+///
+/// ```text
+/// FilterExec(dynamic_filter_1 on a@0)
+/// ProjectionExec(a := Column("a", source_index))
+/// DataSourceExec
+/// ParquetSource(predicate = dynamic_filter_2)
+/// ```
+///
+/// `dynamic_filter_1` and `dynamic_filter_2` are the same dynamic filter, except with
+/// different children.
+///
+/// Returns
+/// - `dynamic_filter_1` before serialization
+/// - `dynamic_filter_2` before serialization
+/// - `dynamic_filter_1` after serialization
+/// - `dynamic_filter_2` after serialization
+#[allow(clippy::type_complexity)]
+fn roundtrip_dynamic_filter_plan_pair() -> Result<(
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+ Arc<dyn PhysicalExpr>,
+)> {
+ let filter_expr_1 = make_dynamic_filter();
+ let (data_source_schema, filter_expr_2) =
+ make_reassigned_dynamic_filter(Arc::clone(&filter_expr_1))?;
+ let left_before = Arc::clone(&filter_expr_1);
+ let right_before = Arc::clone(&filter_expr_2);
+ let file_source = Arc::new(
+ ParquetSource::new(Arc::clone(&data_source_schema))
+ .with_predicate(Arc::clone(&filter_expr_2)),
+ );
+ let scan_config =
+ FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source)
+ .with_file_groups(vec![FileGroup::new(vec![PartitionedFile::new(
+ "/path/to/file.parquet".to_string(),
+ 1024,
+ )])])
+ .build();
+ let data_source_exec =
+ DataSourceExec::from_data_source(scan_config) as Arc<dyn ExecutionPlan>;
- // Create a plan with expressions that will be deduplicated
- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
- let projection_exprs = vec![
- ProjectionExpr {
- expr: Arc::clone(&col_expr),
- alias: "a1".to_string(),
- },
- ProjectionExpr {
- expr: Arc::clone(&col_expr), // Same Arc - will be deduplicated
- alias: "a2".to_string(),
- },
- ];
- let exec_plan = Arc::new(ProjectionExec::try_new(
- projection_exprs,
- Arc::new(EmptyExec::new(schema)),
- )?);
+ let projection_exec = Arc::new(ProjectionExec::try_new(
+ vec![ProjectionExpr {
+ expr: Arc::new(Column::new("a", 1)) as Arc<dyn PhysicalExpr>,
+ alias: "a".to_string(),
+ }],
+ data_source_exec,
+ )?) as Arc<dyn ExecutionPlan>;
+ let filter_exec = Arc::new(FilterExec::try_new(
+ Arc::clone(&filter_expr_1),
+ projection_exec,
+ )?) as Arc<dyn ExecutionPlan>;
- let ctx = SessionContext::new();
let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
-
- // Serialize
- let bytes = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,
- &codec,
- &proto_converter,
- )?;
-
- // First deserialization
- let plan1 = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &proto_converter,
- )?;
-
- // Check that the plan was deserialized correctly with deduplication
- let projection1 = plan1
- .downcast_ref::<ProjectionExec>()
- .expect("Expected ProjectionExec");
- let exprs1: Vec<_> = projection1.expr().iter().collect();
- assert_eq!(exprs1.len(), 2);
- assert!(
- Arc::ptr_eq(&exprs1[0].expr, &exprs1[1].expr),
- "Expected both expressions to share the same Arc after deduplication"
- );
+ let converter = DeduplicatingProtoConverter {};
+ let proto = converter.execution_plan_to_proto(&filter_exec, &codec)?;
- // Second deserialization
- let plan2 = physical_plan_from_bytes_with_proto_converter(
- bytes.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &proto_converter,
- )?;
+ let ctx = SessionContext::new();
+ let task_ctx = ctx.task_ctx();
+ let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+ let deserialized_plan = converter.proto_to_execution_plan(&proto, &decode_ctx)?;
- // Check that the second plan was also deserialized correctly
- let projection2 = plan2
+ let outer_filter = deserialized_plan
+ .downcast_ref::<FilterExec>()
+ .expect("Expected outer FilterExec");
+ let left_filter = Arc::clone(outer_filter.predicate());
+ let projection = outer_filter.children()[0]
.downcast_ref::<ProjectionExec>()
.expect("Expected ProjectionExec");
- let exprs2: Vec<_> = projection2.expr().iter().collect();
- assert_eq!(exprs2.len(), 2);
- assert!(
- Arc::ptr_eq(&exprs2[0].expr, &exprs2[1].expr),
- "Expected both expressions to share the same Arc after deduplication"
- );
-
- // Check that there was no deduplication across deserializations
- assert!(
- !Arc::ptr_eq(&exprs1[0].expr, &exprs2[0].expr),
- "Expected expressions from different deserializations to be different Arcs"
- );
- assert!(
- !Arc::ptr_eq(&exprs1[1].expr, &exprs2[1].expr),
- "Expected expressions from different deserializations to be different Arcs"
- );
+ let data_source = projection
+ .input()
+ .downcast_ref::<DataSourceExec>()
+ .expect("Expected DataSourceExec");
+ let scan_config = data_source
+ .data_source()
+ .downcast_ref::<FileScanConfig>()
+ .expect("Expected FileScanConfig");
+ let right_filter = scan_config
+ .file_source()
+ .filter()
+ .expect("Expected pushed-down predicate");
- Ok(())
+ Ok((left_before, right_before, left_filter, right_filter))
}
-/// Test that deduplication works within direct expression deserialization and that
-/// separate deserializations produce independent expressions (no cross-operation sharing).
-#[test]
-fn test_deduplication_within_expr_deserialization() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
-
- // Create a binary expression where both sides are the same Arc
- // This allows us to test deduplication within a single deserialization
- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
- let binary_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
- Arc::clone(&col_expr),
- Operator::Plus,
- Arc::clone(&col_expr), // Same Arc - will be deduplicated
- ));
-
- let ctx = SessionContext::new();
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
- let task_ctx = ctx.task_ctx();
- let decode_ctx = PhysicalPlanDecodeContext::new(task_ctx.as_ref(), &codec);
+/// Takes two [`DynamicFilterPhysicalExpr`] and asserts that updates to one are visible
+/// via the other. This helps assert that referential integrity is maintained after
+/// deserializing.
+fn assert_dynamic_filter_update_is_visible(
+ left_filter: &Arc<dyn PhysicalExpr>,
+ right_filter: &Arc<dyn PhysicalExpr>,
+) -> Result<()> {
+ let left_filter = left_filter
+ .downcast_ref::<DynamicFilterPhysicalExpr>()
+ .expect("Expected dynamic filter");
+ let right_filter = right_filter
+ .downcast_ref::<DynamicFilterPhysicalExpr>()
+ .expect("Expected dynamic filter");
- // Serialize the expression
- let proto = proto_converter.physical_expr_to_proto(&binary_expr, &codec)?;
+ // Sanity check that the filters have the same generation.
+ let original_generation = left_filter.snapshot_generation();
+ assert_eq!(original_generation, right_filter.snapshot_generation(),);
- // First expression deserialization
- let expr1 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
+ left_filter.update(lit(123_i64))?;
- // Check that deduplication worked within the deserialization
- let binary1 = expr1
- .downcast_ref::<BinaryExpr>()
- .expect("Expected BinaryExpr");
- assert!(
- Arc::ptr_eq(binary1.left(), binary1.right()),
- "Expected both sides to share the same Arc after deduplication"
- );
-
- // Second expression deserialization
- let expr2 = proto_converter.proto_to_physical_expr(&proto, &schema, &decode_ctx)?;
-
- // Check that the second expression was also deserialized correctly
- let binary2 = expr2
- .downcast_ref::<BinaryExpr>()
- .expect("Expected BinaryExpr");
- assert!(
- Arc::ptr_eq(binary2.left(), binary2.right()),
- "Expected both sides to share the same Arc after deduplication"
+ // Assert that both generations updated.
+ assert_eq!(original_generation + 1, right_filter.snapshot_generation(),);
+ assert_eq!(
+ left_filter.snapshot_generation(),
+ right_filter.snapshot_generation(),
);
- // Check that there was no deduplication across deserializations
- assert!(
- !Arc::ptr_eq(binary1.left(), binary2.left()),
- "Expected expressions from different deserializations to be different Arcs"
- );
- assert!(
- !Arc::ptr_eq(binary1.right(), binary2.right()),
- "Expected expressions from different deserializations to be different Arcs"
- );
+ // Ensure both filters have the updated expr.
+ let expected_current = r#"Literal { value: Int64(123), field: Field { name: "lit", data_type: Int64 } }"#;
+ assert_eq!(expected_current, format!("{:?}", left_filter.current()?),);
+ assert_eq!(expected_current, format!("{:?}", right_filter.current()?),);
Ok(())
}
-/// Test that session_id rotates between top-level serialization operations.
-/// This verifies that each top-level serialization gets a fresh session_id,
-/// which prevents cross-process collisions when serialized plans are merged.
-#[test]
-fn test_session_id_rotation_between_serializations() -> Result<()> {
- let field_a = Field::new("a", DataType::Int64, false);
- let _schema = Arc::new(Schema::new(vec![field_a]));
-
- // Create a simple expression
- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
+/// Assert that two dynamic filters are equal both structurally (Debug output)
+/// and by identity (`expression_id`).
+///
+fn assert_dynamic_filters_equal(
+ expected: &Arc<dyn PhysicalExpr>,
+ actual: &Arc<dyn PhysicalExpr>,
+) {
+ // TODO: Debug currently omits `expression_id` so the id has to be checked
+ // separately here. Once plan nodes like `SortExec` / `AggregateExec` /
+ // `HashJoinExec` serialize their own dynamic filter, Debug can include
+ // `expression_id`.
+ //
+ // See https://github.com/apache/datafusion/issues/20418
+ assert_eq!(expected.expression_id(), actual.expression_id());
+
+ // Structural.
+ let expected_dbg = format!("{expected:?}");
+ let actual_dbg = format!("{actual:?}");
+ if expected_dbg == actual_dbg {
+ return;
+ }
- let codec = DefaultPhysicalExtensionCodec {};
- let proto_converter = DeduplicatingProtoConverter {};
+ // Note that the `DeduplicatingDeserializer` routes every cache hit through
+ // `with_new_children`. This produces an equivalent expression, but with
+ // remapped children that are equal to the original. Handle that case here.
+ let rewritten = Arc::clone(expected)
+ .with_new_children(expected.children().iter().map(|c| Arc::clone(c)).collect())
+ .expect("with_new_children on a dynamic filter should not fail");
+ assert_eq!(format!("{rewritten:?}"), actual_dbg);
+}
- // First serialization
- let proto1 = proto_converter.physical_expr_to_proto(&col_expr, &codec)?;
- let expr_id1 = proto1.expr_id.expect("Expected expr_id to be set");
+// Two clones of a dynamic filter expression should be deduped to the exact same expression.
+#[test]
+fn test_dynamic_filter_roundtrip_dedupe() -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int64, false)]));
+ let filter_expr_1 = make_dynamic_filter();
+ let filter_expr_2 = Arc::clone(&filter_expr_1);
- // Second serialization with the same converter
- // The session_id should have rotated, so the expr_id should be different
- // even though we're serializing the same expression (same pointer address)
- let proto2 = proto_converter.physical_expr_to_proto(&col_expr, &codec)?;
- let expr_id2 = proto2.expr_id.expect("Expected expr_id to be set");
+ let (filter_expr_1_after_roundtrip, filter_expr_2_after_roundtrip) =
+ roundtrip_dynamic_filter_expr_pair(
+ Arc::clone(&filter_expr_1),
+ Arc::clone(&filter_expr_2),
+ schema,
+ )?;
- // The expr_ids should be different because session_id rotated
- assert_ne!(
- expr_id1, expr_id2,
- "Expected different expr_ids due to session_id rotation between serializations"
+ // Assert the filters are not modified during roundtrip.
+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
+ assert_dynamic_filters_equal(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
);
- // Also test that serializing the same expression multiple times within
- // the same top-level operation would give the same expr_id (not testable
- // here directly since each physical_expr_to_proto is a top-level operation,
- // but the deduplication tests verify this indirectly)
+ // Assert referential integrity.
+ assert_dynamic_filter_update_is_visible(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
+ )?;
Ok(())
}
-/// Test that session_id rotation works correctly with execution plans.
-/// This verifies the end-to-end behavior with plan serialization.
+/// Roundtrip test for an execution plan where there are multiple instances of a dynamic filter
+/// with different children.
#[test]
-fn test_session_id_rotation_with_execution_plans() -> Result<()> {
- use datafusion_proto::bytes::physical_plan_to_bytes_with_proto_converter;
-
- let field_a = Field::new("a", DataType::Int64, false);
- let schema = Arc::new(Schema::new(vec![field_a]));
+fn test_dynamic_filter_plan_roundtrip_dedupe() -> Result<()> {
+ let (
+ filter_expr_1,
+ filter_expr_2,
+ filter_expr_1_after_roundtrip,
+ filter_expr_2_after_roundtrip,
+ ) = roundtrip_dynamic_filter_plan_pair()?;
+
+ // Assert the filters are not modified during roundtrip.
+ assert_dynamic_filters_equal(&filter_expr_1, &filter_expr_1_after_roundtrip);
+ assert_dynamic_filters_equal(&filter_expr_2, &filter_expr_2_after_roundtrip);
+
+ // Assert referential integrity.
+ assert_dynamic_filter_update_is_visible(
+ &filter_expr_1_after_roundtrip,
+ &filter_expr_2_after_roundtrip,
+ )?;
- // Create a simple plan
- let col_expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("a", 0));
- let projection_exprs = vec![ProjectionExpr {
- expr: Arc::clone(&col_expr),
- alias: "a1".to_string(),
- }];
- let exec_plan = Arc::new(ProjectionExec::try_new(
- projection_exprs.clone(),
- Arc::new(EmptyExec::new(Arc::clone(&schema))),
- )?);
+ Ok(())
+}
+#[test]
+fn test_dynamic_filter_expression_id_is_stable_between_serializations() -> Result<()> {
+ let filter_expr = make_dynamic_filter();
let codec = DefaultPhysicalExtensionCodec {};
let proto_converter = DeduplicatingProtoConverter {};
- // First serialization
- let bytes1 = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,
- &codec,
- &proto_converter,
- )?;
+ let proto1 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
+ let expr_id1 = proto_expression_id(&proto1);
- // Second serialization with the same converter
- let bytes2 = physical_plan_to_bytes_with_proto_converter(
- Arc::clone(&exec_plan) as Arc<dyn ExecutionPlan>,
- &codec,
- &proto_converter,
- )?;
+ let proto2 = proto_converter.physical_expr_to_proto(&filter_expr, &codec)?;
+ let expr_id2 = proto_expression_id(&proto2);
- // The serialized bytes should be different due to different session_ids
- // (specifically, the expr_id values embedded in the protobuf will differ)
- assert_ne!(
- bytes1.as_ref(),
- bytes2.as_ref(),
- "Expected different serialized bytes due to session_id rotation"
+ assert_eq!(
+ expr_id1, expr_id2,
+ "Expected the same dynamic filter expression id across serializations"
);
- // But both should deserialize correctly
- let ctx = SessionContext::new();
- let deser_converter = DeduplicatingProtoConverter {};
-
- let plan1 = datafusion_proto::bytes::physical_plan_from_bytes_with_proto_converter(
- bytes1.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &deser_converter,
- )?;
-
- let plan2 = datafusion_proto::bytes::physical_plan_from_bytes_with_proto_converter(
- bytes2.as_ref(),
- ctx.task_ctx().as_ref(),
- &codec,
- &deser_converter,
- )?;
-
- // Verify both plans have the expected structure
- assert_eq!(plan1.schema(), plan2.schema());
-
Ok(())
}