Apache DataFusion SQL Query Engine
Refactor create_hashes to accept array references (#18448)
Adrian Garcia Badaraccoa899ca0c8626
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex b4488c770d..d60189fb6f 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -17,9 +17,6 @@//! Functionality used both on logical and physical plans-#[cfg(not(feature = "force_hash_collisions"))]-use std::sync::Arc;-use ahash::RandomState;use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano};use arrow::array::*;@@ -215,12 +212,11 @@ fn hash_dictionary<K: ArrowDictionaryKeyType>(// Hash each dictionary value once, and then use that computed// hash for each key value to avoid a potentially expensive// redundant hashing for large dictionary elements (e.g. strings)- let dict_values = Arc::clone(array.values());+ let dict_values = array.values();let mut dict_hashes = vec![0; dict_values.len()];- create_hashes(&[dict_values], random_state, &mut dict_hashes)?;+ create_hashes([dict_values], random_state, &mut dict_hashes)?;// combine hash for each index in values- let dict_values = array.values();for (hash, key) in hashes_buffer.iter_mut().zip(array.keys().iter()) {if let Some(key) = key {let idx = key.as_usize();@@ -308,11 +304,11 @@ fn hash_list_array<OffsetSize>(whereOffsetSize: OffsetSizeTrait,{- let values = Arc::clone(array.values());+ let values = array.values();let offsets = array.value_offsets();let nulls = array.nulls();let mut values_hashes = vec![0u64; values.len()];- create_hashes(&[values], random_state, &mut values_hashes)?;+ create_hashes([values], random_state, &mut values_hashes)?;if let Some(nulls) = nulls {for (i, (start, stop)) in offsets.iter().zip(offsets.iter().skip(1)).enumerate() {if nulls.is_valid(i) {@@ -339,11 +335,11 @@ fn hash_fixed_list_array(random_state: &RandomState,hashes_buffer: &mut [u64],) -> Result<()> {- let values = Arc::clone(array.values());+ let values = array.values();let value_length = array.value_length() as usize;let nulls = array.nulls();let mut values_hashes = vec![0u64; values.len()];- create_hashes(&[values], random_state, &mut values_hashes)?;+ create_hashes([values], random_state, &mut values_hashes)?;if let Some(nulls) = nulls {for i in 0..array.len() {if nulls.is_valid(i) {@@ -366,83 +362,132 @@ fn hash_fixed_list_array(Ok(())}-/// Test version of `create_hashes` that produces the same value for-/// all hashes (to test collisions)-///-/// See comments on `hashes_buffer` for more details+/// Internal helper function that hashes a single array and either initializes or combines+/// the hash values in the buffer.+#[cfg(not(feature = "force_hash_collisions"))]+fn hash_single_array(+ array: &dyn Array,+ random_state: &RandomState,+ hashes_buffer: &mut [u64],+ rehash: bool,+) -> Result<()> {+ downcast_primitive_array! {+ array => hash_array_primitive(array, random_state, hashes_buffer, rehash),+ DataType::Null => hash_null(random_state, hashes_buffer, rehash),+ DataType::Boolean => hash_array(&as_boolean_array(array)?, random_state, hashes_buffer, rehash),+ DataType::Utf8 => hash_array(&as_string_array(array)?, random_state, hashes_buffer, rehash),+ DataType::Utf8View => hash_array(&as_string_view_array(array)?, random_state, hashes_buffer, rehash),+ DataType::LargeUtf8 => hash_array(&as_largestring_array(array), random_state, hashes_buffer, rehash),+ DataType::Binary => hash_array(&as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, rehash),+ DataType::BinaryView => hash_array(&as_binary_view_array(array)?, random_state, hashes_buffer, rehash),+ DataType::LargeBinary => hash_array(&as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, rehash),+ DataType::FixedSizeBinary(_) => {+ let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();+ hash_array(&array, random_state, hashes_buffer, rehash)+ }+ DataType::Dictionary(_, _) => downcast_dictionary_array! {+ array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,+ _ => unreachable!()+ }+ DataType::Struct(_) => {+ let array = as_struct_array(array)?;+ hash_struct_array(array, random_state, hashes_buffer)?;+ }+ DataType::List(_) => {+ let array = as_list_array(array)?;+ hash_list_array(array, random_state, hashes_buffer)?;+ }+ DataType::LargeList(_) => {+ let array = as_large_list_array(array)?;+ hash_list_array(array, random_state, hashes_buffer)?;+ }+ DataType::Map(_, _) => {+ let array = as_map_array(array)?;+ hash_map_array(array, random_state, hashes_buffer)?;+ }+ DataType::FixedSizeList(_,_) => {+ let array = as_fixed_size_list_array(array)?;+ hash_fixed_list_array(array, random_state, hashes_buffer)?;+ }+ _ => {+ // This is internal because we should have caught this before.+ return _internal_err!(+ "Unsupported data type in hasher: {}",+ array.data_type()+ );+ }+ }+ Ok(())+}++/// Test version of `hash_single_array` that forces all hashes to collide to zero.#[cfg(feature = "force_hash_collisions")]-pub fn create_hashes<'a>(- _arrays: &[ArrayRef],+fn hash_single_array(+ _array: &dyn Array,_random_state: &RandomState,- hashes_buffer: &'a mut Vec<u64>,-) -> Result<&'a mut Vec<u64>> {+ hashes_buffer: &mut [u64],+ _rehash: bool,+) -> Result<()> {for hash in hashes_buffer.iter_mut() {*hash = 0}- Ok(hashes_buffer)+ Ok(())+}++/// Something that can be returned as a `&dyn Array`.+///+/// We want `create_hashes` to accept either `&dyn Array` or `ArrayRef`,+/// and this seems the best way to do so.+///+/// We tried having it accept `AsRef<dyn Array>`+/// but that is not implemented for and cannot be implemented for+/// `&dyn Array` so callers that have the latter would not be able+/// to call `create_hashes` directly. This shim trait makes it possible.+pub trait AsDynArray {+ fn as_dyn_array(&self) -> &dyn Array;+}++impl AsDynArray for dyn Array {+ fn as_dyn_array(&self) -> &dyn Array {+ self+ }+}++impl AsDynArray for &dyn Array {+ fn as_dyn_array(&self) -> &dyn Array {+ *self+ }+}++impl AsDynArray for ArrayRef {+ fn as_dyn_array(&self) -> &dyn Array {+ self.as_ref()+ }}-/// Creates hash values for every row, based on the values in the-/// columns.+impl AsDynArray for &ArrayRef {+ fn as_dyn_array(&self) -> &dyn Array {+ self.as_ref()+ }+}++/// Creates hash values for every row, based on the values in the columns.////// The number of rows to hash is determined by `hashes_buffer.len()`.-/// `hashes_buffer` should be pre-sized appropriately-#[cfg(not(feature = "force_hash_collisions"))]-pub fn create_hashes<'a>(- arrays: &[ArrayRef],+/// `hashes_buffer` should be pre-sized appropriately.+pub fn create_hashes<'a, I, T>(+ arrays: I,random_state: &RandomState,hashes_buffer: &'a mut Vec<u64>,-) -> Result<&'a mut Vec<u64>> {- for (i, col) in arrays.iter().enumerate() {- let array = col.as_ref();+) -> Result<&'a mut Vec<u64>>+where+ I: IntoIterator<Item = T>,+ T: AsDynArray,+{+ for (i, array) in arrays.into_iter().enumerate() {// combine hashes with `combine_hashes` for all columns besides the firstlet rehash = i >= 1;- downcast_primitive_array! {- array => hash_array_primitive(array, random_state, hashes_buffer, rehash),- DataType::Null => hash_null(random_state, hashes_buffer, rehash),- DataType::Boolean => hash_array(&as_boolean_array(array)?, random_state, hashes_buffer, rehash),- DataType::Utf8 => hash_array(&as_string_array(array)?, random_state, hashes_buffer, rehash),- DataType::Utf8View => hash_array(&as_string_view_array(array)?, random_state, hashes_buffer, rehash),- DataType::LargeUtf8 => hash_array(&as_largestring_array(array), random_state, hashes_buffer, rehash),- DataType::Binary => hash_array(&as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, rehash),- DataType::BinaryView => hash_array(&as_binary_view_array(array)?, random_state, hashes_buffer, rehash),- DataType::LargeBinary => hash_array(&as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, rehash),- DataType::FixedSizeBinary(_) => {- let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();- hash_array(&array, random_state, hashes_buffer, rehash)- }- DataType::Dictionary(_, _) => downcast_dictionary_array! {- array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,- _ => unreachable!()- }- DataType::Struct(_) => {- let array = as_struct_array(array)?;- hash_struct_array(array, random_state, hashes_buffer)?;- }- DataType::List(_) => {- let array = as_list_array(array)?;- hash_list_array(array, random_state, hashes_buffer)?;- }- DataType::LargeList(_) => {- let array = as_large_list_array(array)?;- hash_list_array(array, random_state, hashes_buffer)?;- }- DataType::Map(_, _) => {- let array = as_map_array(array)?;- hash_map_array(array, random_state, hashes_buffer)?;- }- DataType::FixedSizeList(_,_) => {- let array = as_fixed_size_list_array(array)?;- hash_fixed_list_array(array, random_state, hashes_buffer)?;- }- _ => {- // This is internal because we should have caught this before.- return _internal_err!(- "Unsupported data type in hasher: {}",- col.data_type()- );- }- }+ hash_single_array(array.as_dyn_array(), random_state, hashes_buffer, rehash)?;}Ok(hashes_buffer)}@@ -465,7 +510,7 @@ mod tests {.collect::<Decimal128Array>().with_precision_and_scale(20, 3).unwrap();- let array_ref = Arc::new(array);+ let array_ref: ArrayRef = Arc::new(array);let random_state = RandomState::with_seeds(0, 0, 0, 0);let hashes_buff = &mut vec![0; array_ref.len()];let hashes = create_hashes(&[array_ref], &random_state, hashes_buff)?;@@ -478,15 +523,21 @@ mod tests {let empty_array = FixedSizeListBuilder::new(StringBuilder::new(), 1).finish();let random_state = RandomState::with_seeds(0, 0, 0, 0);let hashes_buff = &mut vec![0; 0];- let hashes = create_hashes(&[Arc::new(empty_array)], &random_state, hashes_buff)?;+ let hashes = create_hashes(+ &[Arc::new(empty_array) as ArrayRef],+ &random_state,+ hashes_buff,+ )?;assert_eq!(hashes, &Vec::<u64>::new());Ok(())}#[test]fn create_hashes_for_float_arrays() -> Result<()> {- let f32_arr = Arc::new(Float32Array::from(vec![0.12, 0.5, 1f32, 444.7]));- let f64_arr = Arc::new(Float64Array::from(vec![0.12, 0.5, 1f64, 444.7]));+ let f32_arr: ArrayRef =+ Arc::new(Float32Array::from(vec![0.12, 0.5, 1f32, 444.7]));+ let f64_arr: ArrayRef =+ Arc::new(Float64Array::from(vec![0.12, 0.5, 1f64, 444.7]));let random_state = RandomState::with_seeds(0, 0, 0, 0);let hashes_buff = &mut vec![0; f32_arr.len()];@@ -514,8 +565,10 @@ mod tests {Some(b"Longer than 12 bytes string"),];- let binary_array = Arc::new(binary.iter().cloned().collect::<$ARRAY>());- let ref_array = Arc::new(binary.iter().cloned().collect::<BinaryArray>());+ let binary_array: ArrayRef =+ Arc::new(binary.iter().cloned().collect::<$ARRAY>());+ let ref_array: ArrayRef =+ Arc::new(binary.iter().cloned().collect::<BinaryArray>());let random_state = RandomState::with_seeds(0, 0, 0, 0);@@ -553,7 +606,7 @@ mod tests {#[test]fn create_hashes_fixed_size_binary() -> Result<()> {let input_arg = vec![vec![1, 2], vec![5, 6], vec![5, 6]];- let fixed_size_binary_array =+ let fixed_size_binary_array: ArrayRef =Arc::new(FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap());let random_state = RandomState::with_seeds(0, 0, 0, 0);@@ -580,8 +633,9 @@ mod tests {Some("Longer than 12 bytes string"),];- let string_array = Arc::new(strings.iter().cloned().collect::<$ARRAY>());- let dict_array = Arc::new(+ let string_array: ArrayRef =+ Arc::new(strings.iter().cloned().collect::<$ARRAY>());+ let dict_array: ArrayRef = Arc::new(strings.iter().cloned()@@ -629,8 +683,9 @@ mod tests {fn create_hashes_for_dict_arrays() {let strings = [Some("foo"), None, Some("bar"), Some("foo"), None];- let string_array = Arc::new(strings.iter().cloned().collect::<StringArray>());- let dict_array = Arc::new(+ let string_array: ArrayRef =+ Arc::new(strings.iter().cloned().collect::<StringArray>());+ let dict_array: ArrayRef = Arc::new(strings.iter().cloned()@@ -865,8 +920,9 @@ mod tests {let strings1 = [Some("foo"), None, Some("bar")];let strings2 = [Some("blarg"), Some("blah"), None];- let string_array = Arc::new(strings1.iter().cloned().collect::<StringArray>());- let dict_array = Arc::new(+ let string_array: ArrayRef =+ Arc::new(strings1.iter().cloned().collect::<StringArray>());+ let dict_array: ArrayRef = Arc::new(strings2.iter().cloned()@@ -896,4 +952,52 @@ mod tests {assert_ne!(one_col_hashes, two_col_hashes);}++ #[test]+ fn test_create_hashes_from_arrays() {+ let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));+ let float_array: ArrayRef =+ Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));++ let random_state = RandomState::with_seeds(0, 0, 0, 0);+ let hashes_buff = &mut vec![0; int_array.len()];+ let hashes =+ create_hashes(&[int_array, float_array], &random_state, hashes_buff).unwrap();+ assert_eq!(hashes.len(), 4,);+ }++ #[test]+ fn test_create_hashes_from_dyn_arrays() {+ let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));+ let float_array: ArrayRef =+ Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));++ // Verify that we can call create_hashes with only &dyn Array+ fn test(arr1: &dyn Array, arr2: &dyn Array) {+ let random_state = RandomState::with_seeds(0, 0, 0, 0);+ let hashes_buff = &mut vec![0; arr1.len()];+ let hashes = create_hashes([arr1, arr2], &random_state, hashes_buff).unwrap();+ assert_eq!(hashes.len(), 4,);+ }+ test(&*int_array, &*float_array);+ }++ #[test]+ fn test_create_hashes_equivalence() {+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));+ let random_state = RandomState::with_seeds(0, 0, 0, 0);++ let mut hashes1 = vec![0; array.len()];+ create_hashes(+ &[Arc::clone(&array) as ArrayRef],+ &random_state,+ &mut hashes1,+ )+ .unwrap();++ let mut hashes2 = vec![0; array.len()];+ create_hashes([array], &random_state, &mut hashes2).unwrap();++ assert_eq!(hashes1, hashes2);+ }}diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rsindex f2e18f7de8..52e0159111 100644--- a/datafusion/common/src/scalar/mod.rs+++ b/datafusion/common/src/scalar/mod.rs@@ -878,10 +878,10 @@ impl Hash for ScalarValue {fn hash_nested_array<H: Hasher>(arr: ArrayRef, state: &mut H) {let len = arr.len();- let arrays = vec![arr];let hashes_buffer = &mut vec![0; len];let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);- let hashes = create_hashes(&arrays, &random_state, hashes_buffer).unwrap();+ let hashes = create_hashes(&[arr], &random_state, hashes_buffer)+ .expect("hash_nested_array: failed to create row hashes");// Hash back to std::hash::Hasherhashes.hash(state);}diff --git a/datafusion/physical-expr-common/src/binary_map.rs b/datafusion/physical-expr-common/src/binary_map.rsindex b37d9a7773..24bc430630 100644--- a/datafusion/physical-expr-common/src/binary_map.rs+++ b/datafusion/physical-expr-common/src/binary_map.rs@@ -349,7 +349,7 @@ wherelet batch_hashes = &mut self.hashes_buffer;batch_hashes.clear();batch_hashes.resize(values.len(), 0);- create_hashes(&[Arc::clone(values)], &self.random_state, batch_hashes)+ create_hashes([values], &self.random_state, batch_hashes)// hash is supported for all types and create_hashes only// returns errors for unsupported types.unwrap();diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs b/datafusion/physical-expr-common/src/binary_view_map.rsindex 7ce943030a..2de563472c 100644--- a/datafusion/physical-expr-common/src/binary_view_map.rs+++ b/datafusion/physical-expr-common/src/binary_view_map.rs@@ -19,6 +19,7 @@//! `StringViewArray`/`BinaryViewArray`.//! Much of the code is from `binary_map.rs`, but with simpler implementation because we directly use the//! [`GenericByteViewBuilder`].+use crate::binary_map::OutputType;use ahash::RandomState;use arrow::array::cast::AsArray;use arrow::array::{Array, ArrayBuilder, ArrayRef, GenericByteViewBuilder};@@ -28,8 +29,6 @@ use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};use std::fmt::Debug;use std::sync::Arc;-use crate::binary_map::OutputType;-/// HashSet optimized for storing string or binary values that can produce that/// the final set as a `GenericBinaryViewArray` with minimal copies.#[derive(Debug)]@@ -243,7 +242,7 @@ wherelet batch_hashes = &mut self.hashes_buffer;batch_hashes.clear();batch_hashes.resize(values.len(), 0);- create_hashes(&[Arc::clone(values)], &self.random_state, batch_hashes)+ create_hashes([values], &self.random_state, batch_hashes)// hash is supported for all types and create_hashes only// returns errors for unsupported types.unwrap();diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rsindex 0a582bd911..c552e6954c 100644--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs@@ -3452,11 +3452,7 @@ mod tests {let random_state = RandomState::with_seeds(0, 0, 0, 0);let hashes_buff = &mut vec![0; left.num_rows()];- let hashes = create_hashes(- &[Arc::clone(&left.columns()[0])],- &random_state,- hashes_buff,- )?;+ let hashes = create_hashes([&left.columns()[0]], &random_state, hashes_buff)?;// Maps both values to both indices (1 and 2, representing input 0 and 1)// 0 -> (0, 1)@@ -3485,11 +3481,7 @@ mod tests {let right_keys_values =key_column.evaluate(&right)?.into_array(right.num_rows())?;let mut hashes_buffer = vec![0; right.num_rows()];- create_hashes(- &[Arc::clone(&right_keys_values)],- &random_state,- &mut hashes_buffer,- )?;+ create_hashes([&right_keys_values], &random_state, &mut hashes_buffer)?;let (l, r, _) = lookup_join_hashmap(&join_hash_map,@@ -3523,11 +3515,7 @@ mod tests {let random_state = RandomState::with_seeds(0, 0, 0, 0);let hashes_buff = &mut vec![0; left.num_rows()];- let hashes = create_hashes(- &[Arc::clone(&left.columns()[0])],- &random_state,- hashes_buff,- )?;+ let hashes = create_hashes([&left.columns()[0]], &random_state, hashes_buff)?;hashmap_left.insert_unique(hashes[0], (hashes[0], 1u32), |(h, _)| *h);hashmap_left.insert_unique(hashes[0], (hashes[0], 2u32), |(h, _)| *h);@@ -3550,11 +3538,7 @@ mod tests {let right_keys_values =key_column.evaluate(&right)?.into_array(right.num_rows())?;let mut hashes_buffer = vec![0; right.num_rows()];- create_hashes(- &[Arc::clone(&right_keys_values)],- &random_state,- &mut hashes_buffer,- )?;+ create_hashes([&right_keys_values], &random_state, &mut hashes_buffer)?;let (l, r, _) = lookup_join_hashmap(&join_hash_map,