Apache DataFusion SQL Query Engine
feat(proto): Add protobuf serialization for HashExpr (#19379)
Adrian Garcia Badaracco91cfb6990319
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 13d1e0a982..bd92cf4964 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -86,9 +86,11 @@ use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;use futures::TryStreamExt;use parking_lot::Mutex;+use super::partitioned_hash_eval::SeededRandomState;+/// Hard-coded seed to ensure hash values from the hash join differ from `RepartitionExec`, avoiding collisions.-pub(crate) const HASH_JOIN_SEED: RandomState =- RandomState::with_seeds('J' as u64, 'O' as u64, 'I' as u64, 'N' as u64);+pub(crate) const HASH_JOIN_SEED: SeededRandomState =+ SeededRandomState::with_seeds('J' as u64, 'O' as u64, 'I' as u64, 'N' as u64);/// HashTable and input data for the left (build side) of a joinpub(super) struct JoinLeftData {@@ -334,8 +336,8 @@ pub struct HashJoinExec {/// Each output stream waits on the `OnceAsync` to signal the completion of/// the hash table creation.left_fut: Arc<OnceAsync<JoinLeftData>>,- /// Shared the `RandomState` for the hashing algorithm- random_state: RandomState,+ /// Shared the `SeededRandomState` for the hashing algorithm (seeds preserved for serialization)+ random_state: SeededRandomState,/// Partitioning mode to usepub mode: PartitionMode,/// Execution metrics@@ -930,7 +932,7 @@ impl ExecutionPlan for HashJoinExec {MemoryConsumer::new("HashJoinInput").register(context.memory_pool());Ok(collect_left_input(- self.random_state.clone(),+ self.random_state.random_state().clone(),left_stream,on_left.clone(),join_metrics.clone(),@@ -958,7 +960,7 @@ impl ExecutionPlan for HashJoinExec {.register(context.memory_pool());OnceFut::new(collect_left_input(- self.random_state.clone(),+ self.random_state.random_state().clone(),left_stream,on_left.clone(),join_metrics.clone(),@@ -1041,7 +1043,7 @@ impl ExecutionPlan for HashJoinExec {self.filter.clone(),self.join_type,right_stream,- self.random_state.clone(),+ self.random_state.random_state().clone(),join_metrics,column_indices_after_projection,self.null_equality,diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rsindex 352209e9c3..8592e1d968 100644--- a/datafusion/physical-plan/src/joins/hash_join/mod.rs+++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs@@ -18,7 +18,7 @@//! [`HashJoinExec`] Partitioned Hash Join Operatorpub use exec::HashJoinExec;-pub use partitioned_hash_eval::HashTableLookupExpr;+pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState};mod exec;mod inlist_builder;diff --git a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rsindex b91f5f46d7..4c437e8131 100644--- a/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs+++ b/datafusion/physical-plan/src/joins/hash_join/partitioned_hash_eval.rs@@ -34,6 +34,36 @@ use datafusion_physical_expr_common::physical_expr::{use crate::{hash_utils::create_hashes, joins::utils::JoinHashMapType};+/// RandomState wrapper that preserves the seeds used to create it.+///+/// This is needed because ahash's `RandomState` doesn't expose its seeds after creation,+/// but we need them for serialization (e.g., protobuf serde).+#[derive(Clone, Debug)]+pub struct SeededRandomState {+ random_state: RandomState,+ seeds: (u64, u64, u64, u64),+}++impl SeededRandomState {+ /// Create a new SeededRandomState with the given seeds.+ pub const fn with_seeds(k0: u64, k1: u64, k2: u64, k3: u64) -> Self {+ Self {+ random_state: RandomState::with_seeds(k0, k1, k2, k3),+ seeds: (k0, k1, k2, k3),+ }+ }++ /// Get the inner RandomState.+ pub fn random_state(&self) -> &RandomState {+ &self.random_state+ }++ /// Get the seeds used to create this RandomState.+ pub fn seeds(&self) -> (u64, u64, u64, u64) {+ self.seeds+ }+}+/// Physical expression that computes hash values for a set of columns////// This expression computes the hash of join key columns using a specific RandomState.@@ -45,8 +75,8 @@ use crate::{hash_utils::create_hashes, joins::utils::JoinHashMapType};pub struct HashExpr {/// Columns to hashon_columns: Vec<PhysicalExprRef>,- /// Random state for hashing- random_state: RandomState,+ /// Random state for hashing (with seeds preserved for serialization)+ random_state: SeededRandomState,/// Description for displaydescription: String,}@@ -56,11 +86,11 @@ impl HashExpr {////// # Arguments/// * `on_columns` - Columns to hash- /// * `random_state` - RandomState for hashing+ /// * `random_state` - SeededRandomState for hashing/// * `description` - Description for debugging (e.g., "hash_repartition", "hash_join")- pub(super) fn new(+ pub fn new(on_columns: Vec<PhysicalExprRef>,- random_state: RandomState,+ random_state: SeededRandomState,description: String,) -> Self {Self {@@ -69,6 +99,21 @@ impl HashExpr {description,}}++ /// Get the columns being hashed.+ pub fn on_columns(&self) -> &[PhysicalExprRef] {+ &self.on_columns+ }++ /// Get the seeds used for hashing.+ pub fn seeds(&self) -> (u64, u64, u64, u64) {+ self.random_state.seeds()+ }++ /// Get the description.+ pub fn description(&self) -> &str {+ &self.description+ }}impl std::fmt::Debug for HashExpr {@@ -79,7 +124,8 @@ impl std::fmt::Debug for HashExpr {.map(|e| e.to_string()).collect::<Vec<_>>().join(", ");- write!(f, "{}({})", self.description, cols)+ let (s1, s2, s3, s4) = self.seeds();+ write!(f, "{}({cols}, [{s1},{s2},{s3},{s4}])", self.description)}}@@ -87,12 +133,15 @@ impl Hash for HashExpr {fn hash<H: std::hash::Hasher>(&self, state: &mut H) {self.on_columns.dyn_hash(state);self.description.hash(state);+ self.seeds().hash(state);}}impl PartialEq for HashExpr {fn eq(&self, other: &Self) -> bool {- self.on_columns == other.on_columns && self.description == other.description+ self.on_columns == other.on_columns+ && self.description == other.description+ && self.seeds() == other.seeds()}}@@ -147,7 +196,11 @@ impl PhysicalExpr for HashExpr {// Compute hasheslet mut hashes_buffer = vec![0; num_rows];- create_hashes(&keys_values, &self.random_state, &mut hashes_buffer)?;+ create_hashes(+ &keys_values,+ self.random_state.random_state(),+ &mut hashes_buffer,+ )?;Ok(ColumnarValue::Array(Arc::new(UInt64Array::from(hashes_buffer,@@ -206,13 +259,29 @@ impl Hash for HashTableLookupExpr {fn hash<H: std::hash::Hasher>(&self, state: &mut H) {self.hash_expr.dyn_hash(state);self.description.hash(state);+ // Note that we compare hash_map by pointer equality.+ // Actually comparing the contents of the hash maps would be expensive.+ // The way these hash maps are used in actuality is that HashJoinExec creates+ // one per partition per query execution, thus it is never possible for two different+ // hash maps to have the same content in practice.+ // Theoretically this is a public API and users could create identical hash maps,+ // but that seems unlikely and not worth paying the cost of deep comparison all the time.+ Arc::as_ptr(&self.hash_map).hash(state);}}impl PartialEq for HashTableLookupExpr {fn eq(&self, other: &Self) -> bool {- Arc::ptr_eq(&self.hash_expr, &other.hash_expr)+ // Note that we compare hash_map by pointer equality.+ // Actually comparing the contents of the hash maps would be expensive.+ // The way these hash maps are used in actuality is that HashJoinExec creates+ // one per partition per query execution, thus it is never possible for two different+ // hash maps to have the same content in practice.+ // Theoretically this is a public API and users could create identical hash maps,+ // but that seems unlikely and not worth paying the cost of deep comparison all the time.+ self.hash_expr.as_ref() == other.hash_expr.as_ref()&& self.description == other.description+ && Arc::ptr_eq(&self.hash_map, &other.hash_map)}}@@ -294,3 +363,265 @@ impl PhysicalExpr for HashTableLookupExpr {write!(f, "{}", self.description)}}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::joins::join_hash_map::JoinHashMapU32;+ use datafusion_physical_expr::expressions::Column;+ use std::collections::hash_map::DefaultHasher;+ use std::hash::Hasher;++ fn compute_hash<T: Hash>(value: &T) -> u64 {+ let mut hasher = DefaultHasher::new();+ value.hash(&mut hasher);+ hasher.finish()+ }++ #[test]+ fn test_hash_expr_eq_same() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));++ let expr1 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ let expr2 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ assert_eq!(expr1, expr2);+ }++ #[test]+ fn test_hash_expr_eq_different_columns() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));+ let col_c: PhysicalExprRef = Arc::new(Column::new("c", 2));++ let expr1 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ let expr2 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_c)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_expr_eq_different_description() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));++ let expr1 = HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "hash_one".to_string(),+ );++ let expr2 = HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "hash_two".to_string(),+ );++ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_expr_eq_different_seeds() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));++ let expr1 = HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ let expr2 = HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(5, 6, 7, 8),+ "test_hash".to_string(),+ );++ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_expr_hash_consistency() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));++ let expr1 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ let expr2 = HashExpr::new(+ vec![Arc::clone(&col_a), Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "test_hash".to_string(),+ );++ // Equal expressions should have equal hashes+ assert_eq!(expr1, expr2);+ assert_eq!(compute_hash(&expr1), compute_hash(&expr2));+ }++ #[test]+ fn test_hash_table_lookup_expr_eq_same() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let hash_expr: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));+ let hash_map: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));++ let expr1 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ let expr2 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ assert_eq!(expr1, expr2);+ }++ #[test]+ fn test_hash_table_lookup_expr_eq_different_hash_expr() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let col_b: PhysicalExprRef = Arc::new(Column::new("b", 1));++ let hash_expr1: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));++ let hash_expr2: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_b)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));++ let hash_map: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));++ let expr1 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr1),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ let expr2 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr2),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_table_lookup_expr_eq_different_description() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let hash_expr: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));+ let hash_map: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));++ let expr1 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup_one".to_string(),+ );++ let expr2 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup_two".to_string(),+ );++ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_table_lookup_expr_eq_different_hash_map() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let hash_expr: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));++ // Two different Arc pointers (even with same content) should not be equal+ let hash_map1: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));+ let hash_map2: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));++ let expr1 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ hash_map1,+ "lookup".to_string(),+ );++ let expr2 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ hash_map2,+ "lookup".to_string(),+ );++ // Different Arc pointers means not equal (uses Arc::ptr_eq)+ assert_ne!(expr1, expr2);+ }++ #[test]+ fn test_hash_table_lookup_expr_hash_consistency() {+ let col_a: PhysicalExprRef = Arc::new(Column::new("a", 0));+ let hash_expr: PhysicalExprRef = Arc::new(HashExpr::new(+ vec![Arc::clone(&col_a)],+ SeededRandomState::with_seeds(1, 2, 3, 4),+ "inner_hash".to_string(),+ ));+ let hash_map: Arc<dyn JoinHashMapType> =+ Arc::new(JoinHashMapU32::with_capacity(10));++ let expr1 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ let expr2 = HashTableLookupExpr::new(+ Arc::clone(&hash_expr),+ Arc::clone(&hash_map),+ "lookup".to_string(),+ );++ // Equal expressions should have equal hashes+ assert_eq!(expr1, expr2);+ assert_eq!(compute_hash(&expr1), compute_hash(&expr2));+ }+}diff --git a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rsindex 5aa2bbb57d..7d34ce9acb 100644--- a/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs+++ b/datafusion/physical-plan/src/joins/hash_join/shared_bounds.rs@@ -26,10 +26,10 @@ use crate::ExecutionPlanProperties;use crate::joins::PartitionMode;use crate::joins::hash_join::exec::HASH_JOIN_SEED;use crate::joins::hash_join::inlist_builder::build_struct_fields;-use crate::joins::hash_join::partitioned_hash_eval::{HashExpr, HashTableLookupExpr};+use crate::joins::hash_join::partitioned_hash_eval::{+ HashExpr, HashTableLookupExpr, SeededRandomState,+};use crate::joins::utils::JoinHashMapType;--use ahash::RandomState;use arrow::array::ArrayRef;use arrow::datatypes::{DataType, Field, Schema};use datafusion_common::config::ConfigOptions;@@ -88,7 +88,7 @@ impl PartitionBounds {fn create_membership_predicate(on_right: &[PhysicalExprRef],pushdown: PushdownStrategy,- random_state: &RandomState,+ random_state: &SeededRandomState,schema: &Schema,) -> Result<Option<Arc<dyn PhysicalExpr>>> {match pushdown {@@ -230,7 +230,7 @@ pub(crate) struct SharedBuildAccumulator {on_right: Vec<PhysicalExprRef>,/// Random state for partitioning (RepartitionExec's hash function with 0,0,0,0 seeds)/// Used for PartitionedHashLookupPhysicalExpr- repartition_random_state: RandomState,+ repartition_random_state: SeededRandomState,/// Schema of the probe (right) side for evaluating filter expressionsprobe_schema: Arc<Schema>,}@@ -308,7 +308,7 @@ impl SharedBuildAccumulator {right_child: &dyn ExecutionPlan,dynamic_filter: Arc<DynamicFilterPhysicalExpr>,on_right: Vec<PhysicalExprRef>,- repartition_random_state: RandomState,+ repartition_random_state: SeededRandomState,) -> Self {// Troubleshooting: If partition counts are incorrect, verify this logic matches// the actual execution pattern in collect_build_side()diff --git a/datafusion/physical-plan/src/joins/join_hash_map.rs b/datafusion/physical-plan/src/joins/join_hash_map.rsindex ed370fdb16..b0ed6dcc7c 100644--- a/datafusion/physical-plan/src/joins/join_hash_map.rs+++ b/datafusion/physical-plan/src/joins/join_hash_map.rs@@ -94,6 +94,12 @@ use hashbrown::hash_table::Entry::{Occupied, Vacant};////// At runtime we choose between using `JoinHashMapU32` and `JoinHashMapU64` which oth implement/// `JoinHashMapType`.+///+/// ## Note on use of this trait as a public API+/// This is currently a public trait but is mainly intended for internal use within DataFusion.+/// For example, we may compare references to `JoinHashMapType` implementations by pointer equality+/// rather than deep equality of contents, as deep equality would be expensive and in our usage+/// patterns it is impossible for two different hash maps to have identical contents in a practical sense.pub trait JoinHashMapType: Send + Sync {fn extend_zero(&mut self, len: usize);diff --git a/datafusion/physical-plan/src/joins/mod.rs b/datafusion/physical-plan/src/joins/mod.rsindex 0ca77b3cae..3ff61ecf1d 100644--- a/datafusion/physical-plan/src/joins/mod.rs+++ b/datafusion/physical-plan/src/joins/mod.rs@@ -20,7 +20,7 @@use arrow::array::BooleanBufferBuilder;pub use cross_join::CrossJoinExec;use datafusion_physical_expr::PhysicalExprRef;-pub use hash_join::{HashJoinExec, HashTableLookupExpr};+pub use hash_join::{HashExpr, HashJoinExec, HashTableLookupExpr, SeededRandomState};pub use nested_loop_join::NestedLoopJoinExec;use parking_lot::Mutex;// Note: SortMergeJoin is not used in plans yetdiff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 5c9472182b..1efdaaabc7 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -61,6 +61,7 @@ use crate::filter_pushdown::{ChildPushdownResult, FilterDescription, FilterPushdownPhase,FilterPushdownPropagation,};+use crate::joins::SeededRandomState;use crate::sort_pushdown::SortOrderPushdownResult;use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr;use datafusion_physical_expr_common::utils::evaluate_expressions_to_arrays;@@ -429,8 +430,8 @@ enum BatchPartitionerState {/// Fixed RandomState used for hash repartitioning to ensure consistent behavior across/// executions and runs.-pub const REPARTITION_RANDOM_STATE: ahash::RandomState =- ahash::RandomState::with_seeds(0, 0, 0, 0);+pub const REPARTITION_RANDOM_STATE: SeededRandomState =+ SeededRandomState::with_seeds(0, 0, 0, 0);impl BatchPartitioner {/// Create a new [`BatchPartitioner`] with the provided [`Partitioning`]@@ -514,7 +515,11 @@ impl BatchPartitioner {hash_buffer.clear();hash_buffer.resize(batch.num_rows(), 0);- create_hashes(&arrays, &REPARTITION_RANDOM_STATE, hash_buffer)?;+ create_hashes(+ &arrays,+ REPARTITION_RANDOM_STATE.random_state(),+ hash_buffer,+ )?;let mut indices: Vec<_> = (0..*partitions).map(|_| Vec::with_capacity(batch.num_rows()))diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.protoindex 6616f77a5b..bd7dd3a6af 100644--- a/datafusion/proto/proto/datafusion.proto+++ b/datafusion/proto/proto/datafusion.proto@@ -872,6 +872,8 @@ message PhysicalExprNode {PhysicalExtensionExprNode extension = 19;UnknownColumn unknown_column = 20;++ PhysicalHashExprNode hash_expr = 21;}}@@ -990,6 +992,15 @@ message PhysicalExtensionExprNode {repeated PhysicalExprNode inputs = 2;}+message PhysicalHashExprNode {+ repeated PhysicalExprNode on_columns = 1;+ uint64 seed0 = 2;+ uint64 seed1 = 3;+ uint64 seed2 = 4;+ uint64 seed3 = 5;+ string description = 6;+}+message FilterExecNode {PhysicalPlanNode input = 1;PhysicalExprNode expr = 2;diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rsindex cf3dcfe01b..e269606d16 100644--- a/datafusion/proto/src/generated/pbjson.rs+++ b/datafusion/proto/src/generated/pbjson.rs@@ -15934,6 +15934,9 @@ impl serde::Serialize for PhysicalExprNode {physical_expr_node::ExprType::UnknownColumn(v) => {struct_ser.serialize_field("unknownColumn", v)?;}+ physical_expr_node::ExprType::HashExpr(v) => {+ struct_ser.serialize_field("hashExpr", v)?;+ }}}struct_ser.end()@@ -15976,6 +15979,8 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"extension","unknown_column","unknownColumn",+ "hash_expr",+ "hashExpr",];#[allow(clippy::enum_variant_names)]@@ -15998,6 +16003,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {LikeExpr,Extension,UnknownColumn,+ HashExpr,}impl<'de> serde::Deserialize<'de> for GeneratedField {fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>@@ -16037,6 +16043,7 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {"likeExpr" | "like_expr" => Ok(GeneratedField::LikeExpr),"extension" => Ok(GeneratedField::Extension),"unknownColumn" | "unknown_column" => Ok(GeneratedField::UnknownColumn),+ "hashExpr" | "hash_expr" => Ok(GeneratedField::HashExpr),_ => Err(serde::de::Error::unknown_field(value, FIELDS)),}}@@ -16183,6 +16190,13 @@ impl<'de> serde::Deserialize<'de> for PhysicalExprNode {return Err(serde::de::Error::duplicate_field("unknownColumn"));}expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::UnknownColumn)+;+ }+ GeneratedField::HashExpr => {+ if expr_type__.is_some() {+ return Err(serde::de::Error::duplicate_field("hashExpr"));+ }+ expr_type__ = map_.next_value::<::std::option::Option<_>>()?.map(physical_expr_node::ExprType::HashExpr);}}@@ -16419,6 +16433,199 @@ impl<'de> serde::Deserialize<'de> for PhysicalExtensionNode {deserializer.deserialize_struct("datafusion.PhysicalExtensionNode", FIELDS, GeneratedVisitor)}}+impl serde::Serialize for PhysicalHashExprNode {+ #[allow(deprecated)]+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>+ where+ S: serde::Serializer,+ {+ use serde::ser::SerializeStruct;+ let mut len = 0;+ if !self.on_columns.is_empty() {+ len += 1;+ }+ if self.seed0 != 0 {+ len += 1;+ }+ if self.seed1 != 0 {+ len += 1;+ }+ if self.seed2 != 0 {+ len += 1;+ }+ if self.seed3 != 0 {+ len += 1;+ }+ if !self.description.is_empty() {+ len += 1;+ }+ let mut struct_ser = serializer.serialize_struct("datafusion.PhysicalHashExprNode", len)?;+ if !self.on_columns.is_empty() {+ struct_ser.serialize_field("onColumns", &self.on_columns)?;+ }+ if self.seed0 != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("seed0", ToString::to_string(&self.seed0).as_str())?;+ }+ if self.seed1 != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("seed1", ToString::to_string(&self.seed1).as_str())?;+ }+ if self.seed2 != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("seed2", ToString::to_string(&self.seed2).as_str())?;+ }+ if self.seed3 != 0 {+ #[allow(clippy::needless_borrow)]+ #[allow(clippy::needless_borrows_for_generic_args)]+ struct_ser.serialize_field("seed3", ToString::to_string(&self.seed3).as_str())?;+ }+ if !self.description.is_empty() {+ struct_ser.serialize_field("description", &self.description)?;+ }+ struct_ser.end()+ }+}+impl<'de> serde::Deserialize<'de> for PhysicalHashExprNode {+ #[allow(deprecated)]+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ const FIELDS: &[&str] = &[+ "on_columns",+ "onColumns",+ "seed0",+ "seed1",+ "seed2",+ "seed3",+ "description",+ ];++ #[allow(clippy::enum_variant_names)]+ enum GeneratedField {+ OnColumns,+ Seed0,+ Seed1,+ Seed2,+ Seed3,+ Description,+ }+ impl<'de> serde::Deserialize<'de> for GeneratedField {+ fn deserialize<D>(deserializer: D) -> std::result::Result<GeneratedField, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ struct GeneratedVisitor;++ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = GeneratedField;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(formatter, "expected one of: {:?}", &FIELDS)+ }++ #[allow(unused_variables)]+ fn visit_str<E>(self, value: &str) -> std::result::Result<GeneratedField, E>+ where+ E: serde::de::Error,+ {+ match value {+ "onColumns" | "on_columns" => Ok(GeneratedField::OnColumns),+ "seed0" => Ok(GeneratedField::Seed0),+ "seed1" => Ok(GeneratedField::Seed1),+ "seed2" => Ok(GeneratedField::Seed2),+ "seed3" => Ok(GeneratedField::Seed3),+ "description" => Ok(GeneratedField::Description),+ _ => Err(serde::de::Error::unknown_field(value, FIELDS)),+ }+ }+ }+ deserializer.deserialize_identifier(GeneratedVisitor)+ }+ }+ struct GeneratedVisitor;+ impl<'de> serde::de::Visitor<'de> for GeneratedVisitor {+ type Value = PhysicalHashExprNode;++ fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ formatter.write_str("struct datafusion.PhysicalHashExprNode")+ }++ fn visit_map<V>(self, mut map_: V) -> std::result::Result<PhysicalHashExprNode, V::Error>+ where+ V: serde::de::MapAccess<'de>,+ {+ let mut on_columns__ = None;+ let mut seed0__ = None;+ let mut seed1__ = None;+ let mut seed2__ = None;+ let mut seed3__ = None;+ let mut description__ = None;+ while let Some(k) = map_.next_key()? {+ match k {+ GeneratedField::OnColumns => {+ if on_columns__.is_some() {+ return Err(serde::de::Error::duplicate_field("onColumns"));+ }+ on_columns__ = Some(map_.next_value()?);+ }+ GeneratedField::Seed0 => {+ if seed0__.is_some() {+ return Err(serde::de::Error::duplicate_field("seed0"));+ }+ seed0__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Seed1 => {+ if seed1__.is_some() {+ return Err(serde::de::Error::duplicate_field("seed1"));+ }+ seed1__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Seed2 => {+ if seed2__.is_some() {+ return Err(serde::de::Error::duplicate_field("seed2"));+ }+ seed2__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Seed3 => {+ if seed3__.is_some() {+ return Err(serde::de::Error::duplicate_field("seed3"));+ }+ seed3__ =+ Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0)+ ;+ }+ GeneratedField::Description => {+ if description__.is_some() {+ return Err(serde::de::Error::duplicate_field("description"));+ }+ description__ = Some(map_.next_value()?);+ }+ }+ }+ Ok(PhysicalHashExprNode {+ on_columns: on_columns__.unwrap_or_default(),+ seed0: seed0__.unwrap_or_default(),+ seed1: seed1__.unwrap_or_default(),+ seed2: seed2__.unwrap_or_default(),+ seed3: seed3__.unwrap_or_default(),+ description: description__.unwrap_or_default(),+ })+ }+ }+ deserializer.deserialize_struct("datafusion.PhysicalHashExprNode", FIELDS, GeneratedVisitor)+ }+}impl serde::Serialize for PhysicalHashRepartition {#[allow(deprecated)]fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rsindex 885b61001c..cf343e0258 100644--- a/datafusion/proto/src/generated/prost.rs+++ b/datafusion/proto/src/generated/prost.rs@@ -1276,7 +1276,7 @@ pub struct PhysicalExtensionNode {pub struct PhysicalExprNode {#[prost(oneof = "physical_expr_node::ExprType",- tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20"+ tags = "1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 18, 19, 20, 21")]pub expr_type: ::core::option::Option<physical_expr_node::ExprType>,}@@ -1327,6 +1327,8 @@ pub mod physical_expr_node {Extension(super::PhysicalExtensionExprNode),#[prost(message, tag = "20")]UnknownColumn(super::UnknownColumn),+ #[prost(message, tag = "21")]+ HashExpr(super::PhysicalHashExprNode),}}#[derive(Clone, PartialEq, ::prost::Message)]@@ -1517,6 +1519,21 @@ pub struct PhysicalExtensionExprNode {pub inputs: ::prost::alloc::vec::Vec<PhysicalExprNode>,}#[derive(Clone, PartialEq, ::prost::Message)]+pub struct PhysicalHashExprNode {+ #[prost(message, repeated, tag = "1")]+ pub on_columns: ::prost::alloc::vec::Vec<PhysicalExprNode>,+ #[prost(uint64, tag = "2")]+ pub seed0: u64,+ #[prost(uint64, tag = "3")]+ pub seed1: u64,+ #[prost(uint64, tag = "4")]+ pub seed2: u64,+ #[prost(uint64, tag = "5")]+ pub seed3: u64,+ #[prost(string, tag = "6")]+ pub description: ::prost::alloc::string::String,+}+#[derive(Clone, PartialEq, ::prost::Message)]pub struct FilterExecNode {#[prost(message, optional, boxed, tag = "1")]pub input: ::core::option::Option<::prost::alloc::boxed::Box<PhysicalPlanNode>>,diff --git a/datafusion/proto/src/physical_plan/from_proto.rs b/datafusion/proto/src/physical_plan/from_proto.rsindex aa02e63a5d..073fdd858c 100644--- a/datafusion/proto/src/physical_plan/from_proto.rs+++ b/datafusion/proto/src/physical_plan/from_proto.rs@@ -48,6 +48,7 @@ use datafusion_physical_plan::expressions::{BinaryExpr, CaseExpr, CastExpr, Column, IsNotNullExpr, IsNullExpr, LikeExpr, Literal,NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn, in_list,};+use datafusion_physical_plan::joins::{HashExpr, SeededRandomState};use datafusion_physical_plan::windows::{create_window_expr, schema_add_window_field};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};use datafusion_proto_common::common::proto_error;@@ -399,6 +400,20 @@ pub fn parse_physical_expr(codec,)?,)),+ ExprType::HashExpr(hash_expr) => {+ let on_columns =+ parse_physical_exprs(&hash_expr.on_columns, ctx, input_schema, codec)?;+ Arc::new(HashExpr::new(+ on_columns,+ SeededRandomState::with_seeds(+ hash_expr.seed0,+ hash_expr.seed1,+ hash_expr.seed2,+ hash_expr.seed3,+ ),+ hash_expr.description.clone(),+ ))+ }ExprType::Extension(extension) => {let inputs: Vec<Arc<dyn PhysicalExpr>> = extension.inputsdiff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rsindex b06dec592d..9558effb8a 100644--- a/datafusion/proto/src/physical_plan/to_proto.rs+++ b/datafusion/proto/src/physical_plan/to_proto.rs@@ -41,7 +41,7 @@ use datafusion_physical_plan::expressions::{BinaryExpr, CaseExpr, CastExpr, Column, InListExpr, IsNotNullExpr, IsNullExpr,Literal, NegativeExpr, NotExpr, TryCastExpr, UnKnownColumn,};-use datafusion_physical_plan::joins::HashTableLookupExpr;+use datafusion_physical_plan::joins::{HashExpr, HashTableLookupExpr};use datafusion_physical_plan::udaf::AggregateFunctionExpr;use datafusion_physical_plan::windows::{PlainAggregateWindowExpr, WindowUDFExpr};use datafusion_physical_plan::{Partitioning, PhysicalExpr, WindowExpr};@@ -410,6 +410,20 @@ pub fn serialize_physical_expr(},))),})+ } else if let Some(expr) = expr.downcast_ref::<HashExpr>() {+ let (s0, s1, s2, s3) = expr.seeds();+ Ok(protobuf::PhysicalExprNode {+ expr_type: Some(protobuf::physical_expr_node::ExprType::HashExpr(+ protobuf::PhysicalHashExprNode {+ on_columns: serialize_physical_exprs(expr.on_columns(), codec)?,+ seed0: s0,+ seed1: s1,+ seed2: s2,+ seed3: s3,+ description: expr.description().to_string(),+ },+ )),+ })} else {let mut buf: Vec<u8> = vec![];match codec.try_encode_expr(&value, &mut buf) {diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rsindex f9babeba56..aa54588493 100644--- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs+++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs@@ -2371,3 +2371,36 @@ fn roundtrip_hash_table_lookup_expr_to_lit() -> Result<()> {Ok(())}++#[test]+fn roundtrip_hash_expr() -> Result<()> {+ use datafusion::physical_plan::joins::{HashExpr, SeededRandomState};++ let schema = Arc::new(Schema::new(vec![+ Field::new("a", DataType::Int64, false),+ Field::new("b", DataType::Utf8, false),+ ]));++ // Create a HashExpr with test columns and seeds+ let on_columns = vec![col("a", &schema)?, col("b", &schema)?];+ let hash_expr: Arc<dyn PhysicalExpr> = Arc::new(HashExpr::new(+ on_columns,+ SeededRandomState::with_seeds(0, 1, 2, 3), // arbitrary random seeds for testing+ "test_hash".to_string(),+ ));++ // Wrap in a filter by comparing hash value to a literal+ // hash_expr > 0 is always boolean+ let filter_expr = binary(hash_expr, Operator::Gt, lit(0u64), &schema)?;+ let filter = Arc::new(FilterExec::try_new(+ filter_expr,+ Arc::new(EmptyExec::new(schema)),+ )?);++ // Confirm that the debug string contains the random state seeds+ assert!(+ format!("{filter:?}").contains("test_hash(a@0, b@1, [0,1,2,3])"),+ "Debug string missing seeds: {filter:?}"+ );+ roundtrip_test(filter)+}