Apache DataFusion SQL Query Engine
refactor: add `try_to_proto` to `HashTableLookupExpr` (#22451)
Anurag Tryambak Rauteb3c56479334
diff --git a/Cargo.lock b/Cargo.lockindex a421d6d992..7bd9afc040 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2448,6 +2448,8 @@ dependencies = ["datafusion-functions-window-common","datafusion-physical-expr","datafusion-physical-expr-common",+ "datafusion-proto-common",+ "datafusion-proto-models","futures","half","hashbrown 0.17.1",diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 5a05173eb3..c64d3cad69 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -42,6 +42,11 @@ force_hash_collisions = []test_utils = ["arrow/test_utils"]tokio_coop = []tokio_coop_fallback = []+proto = [+ "dep:datafusion-proto-models",+ "dep:datafusion-proto-common",+ "datafusion-physical-expr-common/proto",+][lib]name = "datafusion_physical_plan"@@ -65,6 +70,8 @@ datafusion-functions-aggregate-common = { workspace = true }datafusion-functions-window-common = { workspace = true }datafusion-physical-expr = { workspace = true, default-features = true }datafusion-physical-expr-common = { workspace = true }+datafusion-proto-common = { workspace = true, optional = true }+datafusion-proto-models = { workspace = true, optional = true }futures = { workspace = true }half = { workspace = true }hashbrown = { workspace = true }diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rsindex 0daac0bb86..46b087ad70 100644--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs@@ -215,7 +215,6 @@ pub struct HashTableLookupExpr {/// Description for displaydescription: String,}-impl HashTableLookupExpr {/// Create a new HashTableLookupExpr///@@ -241,7 +240,6 @@ impl HashTableLookupExpr {}}}-impl std::fmt::Debug for HashTableLookupExpr {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {let cols = self@@ -337,7 +335,38 @@ impl PhysicalExpr for HashTableLookupExpr {}}}-+ #[cfg(feature = "proto")]+ fn try_to_proto(+ &self,+ _ctx: &datafusion_physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx<'_>,+ ) -> Result<Option<datafusion_proto_models::protobuf::PhysicalExprNode>> {+ use datafusion_proto_models::protobuf;+ use datafusion_proto_models::protobuf::physical_expr_node::ExprType;++ // HashTableLookupExpr holds a runtime Arc<Map> (the build-side hash+ // table) that cannot be serialized, so it is replaced with lit(true).+ //+ // Dynamic filtering is a performance optimisation only — replacing the+ // lookup with lit(true) preserves correctness by allowing all rows+ // through.+ //+ // If a plan is serialized before execution, HashTableLookupExpr is not+ // yet present in the dynamic filter expression.+ //+ // If a plan is serialized after execution, any runtime-created+ // HashTableLookupExpr is replaced during serialization. Re-executing+ // the plan requires reset_state(), after which HashJoinExec rebuilds+ // fresh dynamic filters at runtime.+ let value = datafusion_proto_common::ScalarValue {+ value: Some(datafusion_proto_common::scalar_value::Value::BoolValue(+ true,+ )),+ };+ Ok(Some(protobuf::PhysicalExprNode {+ expr_id: None,+ expr_type: Some(ExprType::Literal(value)),+ }))+ }fn fmt_sql(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {write!(f, "{}", self.description)}diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.tomlindex 53e7cd78dc..cfff8a9494 100644--- a/datafusion/proto/Cargo.toml+++ b/datafusion/proto/Cargo.toml@@ -65,7 +65,7 @@ datafusion-expr = { workspace = true }datafusion-functions-table = { workspace = true }datafusion-physical-expr = { workspace = true, features = ["proto"] }datafusion-physical-expr-common = { workspace = true, features = ["proto"] }-datafusion-physical-plan = { workspace = true }+datafusion-physical-plan = { workspace = true, features = ["proto"] }datafusion-proto-common = { workspace = true }datafusion-proto-models = { workspace = true }object_store = { workspace = true }diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex 5dd643c84b..c359f651c0 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -39,7 +39,7 @@ use datafusion_physical_plan::expressions::{CaseExpr, CastExpr, DynamicFilterPhysicalExpr, IsNotNullExpr, IsNullExpr, Literal,NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,};-use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};+use datafusion_physical_plan::joins::HashExpr;use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};@@ -302,31 +302,6 @@ pub fn serialize_physical_expr_with_converter(return Ok(node);}- // HashTableLookupExpr is used for dynamic filter pushdown in hash joins.- // It contains an Arc<dyn JoinHashMapType> (the build-side hash table) which- // cannot be serialized - the hash table is a runtime structure built during- // execution on the build side.- //- // We replace it with lit(true) which is safe because:- // 1. The filter is a performance optimization, not a correctness requirement- // 2. lit(true) passes all rows, so no valid rows are incorrectly filtered out- // 3. The join itself will still produce correct results, just without the- // benefit of early filtering on the probe side- //- // In distributed execution, the remote worker won't have access to the hash- // table anyway, so the best we can do is skip this optimization.- if expr.downcast_ref::<HashTableLookupExpr>().is_some() {- let value = datafusion_proto_common::ScalarValue {- value: Some(datafusion_proto_common::scalar_value::Value::BoolValue(- true,- )),- };- return Ok(protobuf::PhysicalExprNode {- expr_id,- expr_type: Some(protobuf::physical_expr_node::ExprType::Literal(value)),- });- }-if let Some(expr) = expr.downcast_ref::<UnKnownColumn>() {Ok(protobuf::PhysicalExprNode {expr_id,