Skip to content

Apache DataFusion SQL Query Engine

Refactor create_hashes to accept array references (#18448)

Adrian Garcia Badaraccoa899ca0c8626
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs
index b4488c770d..d60189fb6f 100644
--- a/datafusion/common/src/hash_utils.rs
+++ b/datafusion/common/src/hash_utils.rs
@@ -17,9 +17,6 @@
//! Functionality used both on logical and physical plans
-#[cfg(not(feature = "force_hash_collisions"))]
-use std::sync::Arc;
-
use ahash::RandomState;
use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano};
use arrow::array::*;
@@ -215,12 +212,11 @@ fn hash_dictionary<K: ArrowDictionaryKeyType>(
// Hash each dictionary value once, and then use that computed
// hash for each key value to avoid a potentially expensive
// redundant hashing for large dictionary elements (e.g. strings)
- let dict_values = Arc::clone(array.values());
+ let dict_values = array.values();
let mut dict_hashes = vec![0; dict_values.len()];
- create_hashes(&[dict_values], random_state, &mut dict_hashes)?;
+ create_hashes([dict_values], random_state, &mut dict_hashes)?;
// combine hash for each index in values
- let dict_values = array.values();
for (hash, key) in hashes_buffer.iter_mut().zip(array.keys().iter()) {
if let Some(key) = key {
let idx = key.as_usize();
@@ -308,11 +304,11 @@ fn hash_list_array<OffsetSize>(
where
OffsetSize: OffsetSizeTrait,
{
- let values = Arc::clone(array.values());
+ let values = array.values();
let offsets = array.value_offsets();
let nulls = array.nulls();
let mut values_hashes = vec![0u64; values.len()];
- create_hashes(&[values], random_state, &mut values_hashes)?;
+ create_hashes([values], random_state, &mut values_hashes)?;
if let Some(nulls) = nulls {
for (i, (start, stop)) in offsets.iter().zip(offsets.iter().skip(1)).enumerate() {
if nulls.is_valid(i) {
@@ -339,11 +335,11 @@ fn hash_fixed_list_array(
random_state: &RandomState,
hashes_buffer: &mut [u64],
) -> Result<()> {
- let values = Arc::clone(array.values());
+ let values = array.values();
let value_length = array.value_length() as usize;
let nulls = array.nulls();
let mut values_hashes = vec![0u64; values.len()];
- create_hashes(&[values], random_state, &mut values_hashes)?;
+ create_hashes([values], random_state, &mut values_hashes)?;
if let Some(nulls) = nulls {
for i in 0..array.len() {
if nulls.is_valid(i) {
@@ -366,83 +362,132 @@ fn hash_fixed_list_array(
Ok(())
}
-/// Test version of `create_hashes` that produces the same value for
-/// all hashes (to test collisions)
-///
-/// See comments on `hashes_buffer` for more details
+/// Internal helper function that hashes a single array and either initializes or combines
+/// the hash values in the buffer.
+#[cfg(not(feature = "force_hash_collisions"))]
+fn hash_single_array(
+ array: &dyn Array,
+ random_state: &RandomState,
+ hashes_buffer: &mut [u64],
+ rehash: bool,
+) -> Result<()> {
+ downcast_primitive_array! {
+ array => hash_array_primitive(array, random_state, hashes_buffer, rehash),
+ DataType::Null => hash_null(random_state, hashes_buffer, rehash),
+ DataType::Boolean => hash_array(&as_boolean_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::Utf8 => hash_array(&as_string_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::Utf8View => hash_array(&as_string_view_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::LargeUtf8 => hash_array(&as_largestring_array(array), random_state, hashes_buffer, rehash),
+ DataType::Binary => hash_array(&as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, rehash),
+ DataType::BinaryView => hash_array(&as_binary_view_array(array)?, random_state, hashes_buffer, rehash),
+ DataType::LargeBinary => hash_array(&as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, rehash),
+ DataType::FixedSizeBinary(_) => {
+ let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();
+ hash_array(&array, random_state, hashes_buffer, rehash)
+ }
+ DataType::Dictionary(_, _) => downcast_dictionary_array! {
+ array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,
+ _ => unreachable!()
+ }
+ DataType::Struct(_) => {
+ let array = as_struct_array(array)?;
+ hash_struct_array(array, random_state, hashes_buffer)?;
+ }
+ DataType::List(_) => {
+ let array = as_list_array(array)?;
+ hash_list_array(array, random_state, hashes_buffer)?;
+ }
+ DataType::LargeList(_) => {
+ let array = as_large_list_array(array)?;
+ hash_list_array(array, random_state, hashes_buffer)?;
+ }
+ DataType::Map(_, _) => {
+ let array = as_map_array(array)?;
+ hash_map_array(array, random_state, hashes_buffer)?;
+ }
+ DataType::FixedSizeList(_,_) => {
+ let array = as_fixed_size_list_array(array)?;
+ hash_fixed_list_array(array, random_state, hashes_buffer)?;
+ }
+ _ => {
+ // This is internal because we should have caught this before.
+ return _internal_err!(
+ "Unsupported data type in hasher: {}",
+ array.data_type()
+ );
+ }
+ }
+ Ok(())
+}
+
+/// Test version of `hash_single_array` that forces all hashes to collide to zero.
#[cfg(feature = "force_hash_collisions")]
-pub fn create_hashes<'a>(
- _arrays: &[ArrayRef],
+fn hash_single_array(
+ _array: &dyn Array,
_random_state: &RandomState,
- hashes_buffer: &'a mut Vec<u64>,
-) -> Result<&'a mut Vec<u64>> {
+ hashes_buffer: &mut [u64],
+ _rehash: bool,
+) -> Result<()> {
for hash in hashes_buffer.iter_mut() {
*hash = 0
}
- Ok(hashes_buffer)
+ Ok(())
+}
+
+/// Something that can be returned as a `&dyn Array`.
+///
+/// We want `create_hashes` to accept either `&dyn Array` or `ArrayRef`,
+/// and this seems the best way to do so.
+///
+/// We tried having it accept `AsRef<dyn Array>`
+/// but that is not implemented for and cannot be implemented for
+/// `&dyn Array` so callers that have the latter would not be able
+/// to call `create_hashes` directly. This shim trait makes it possible.
+pub trait AsDynArray {
+ fn as_dyn_array(&self) -> &dyn Array;
+}
+
+impl AsDynArray for dyn Array {
+ fn as_dyn_array(&self) -> &dyn Array {
+ self
+ }
+}
+
+impl AsDynArray for &dyn Array {
+ fn as_dyn_array(&self) -> &dyn Array {
+ *self
+ }
+}
+
+impl AsDynArray for ArrayRef {
+ fn as_dyn_array(&self) -> &dyn Array {
+ self.as_ref()
+ }
}
-/// Creates hash values for every row, based on the values in the
-/// columns.
+impl AsDynArray for &ArrayRef {
+ fn as_dyn_array(&self) -> &dyn Array {
+ self.as_ref()
+ }
+}
+
+/// Creates hash values for every row, based on the values in the columns.
///
/// The number of rows to hash is determined by `hashes_buffer.len()`.
-/// `hashes_buffer` should be pre-sized appropriately
-#[cfg(not(feature = "force_hash_collisions"))]
-pub fn create_hashes<'a>(
- arrays: &[ArrayRef],
+/// `hashes_buffer` should be pre-sized appropriately.
+pub fn create_hashes<'a, I, T>(
+ arrays: I,
random_state: &RandomState,
hashes_buffer: &'a mut Vec<u64>,
-) -> Result<&'a mut Vec<u64>> {
- for (i, col) in arrays.iter().enumerate() {
- let array = col.as_ref();
+) -> Result<&'a mut Vec<u64>>
+where
+ I: IntoIterator<Item = T>,
+ T: AsDynArray,
+{
+ for (i, array) in arrays.into_iter().enumerate() {
// combine hashes with `combine_hashes` for all columns besides the first
let rehash = i >= 1;
- downcast_primitive_array! {
- array => hash_array_primitive(array, random_state, hashes_buffer, rehash),
- DataType::Null => hash_null(random_state, hashes_buffer, rehash),
- DataType::Boolean => hash_array(&as_boolean_array(array)?, random_state, hashes_buffer, rehash),
- DataType::Utf8 => hash_array(&as_string_array(array)?, random_state, hashes_buffer, rehash),
- DataType::Utf8View => hash_array(&as_string_view_array(array)?, random_state, hashes_buffer, rehash),
- DataType::LargeUtf8 => hash_array(&as_largestring_array(array), random_state, hashes_buffer, rehash),
- DataType::Binary => hash_array(&as_generic_binary_array::<i32>(array)?, random_state, hashes_buffer, rehash),
- DataType::BinaryView => hash_array(&as_binary_view_array(array)?, random_state, hashes_buffer, rehash),
- DataType::LargeBinary => hash_array(&as_generic_binary_array::<i64>(array)?, random_state, hashes_buffer, rehash),
- DataType::FixedSizeBinary(_) => {
- let array: &FixedSizeBinaryArray = array.as_any().downcast_ref().unwrap();
- hash_array(&array, random_state, hashes_buffer, rehash)
- }
- DataType::Dictionary(_, _) => downcast_dictionary_array! {
- array => hash_dictionary(array, random_state, hashes_buffer, rehash)?,
- _ => unreachable!()
- }
- DataType::Struct(_) => {
- let array = as_struct_array(array)?;
- hash_struct_array(array, random_state, hashes_buffer)?;
- }
- DataType::List(_) => {
- let array = as_list_array(array)?;
- hash_list_array(array, random_state, hashes_buffer)?;
- }
- DataType::LargeList(_) => {
- let array = as_large_list_array(array)?;
- hash_list_array(array, random_state, hashes_buffer)?;
- }
- DataType::Map(_, _) => {
- let array = as_map_array(array)?;
- hash_map_array(array, random_state, hashes_buffer)?;
- }
- DataType::FixedSizeList(_,_) => {
- let array = as_fixed_size_list_array(array)?;
- hash_fixed_list_array(array, random_state, hashes_buffer)?;
- }
- _ => {
- // This is internal because we should have caught this before.
- return _internal_err!(
- "Unsupported data type in hasher: {}",
- col.data_type()
- );
- }
- }
+ hash_single_array(array.as_dyn_array(), random_state, hashes_buffer, rehash)?;
}
Ok(hashes_buffer)
}
@@ -465,7 +510,7 @@ mod tests {
.collect::<Decimal128Array>()
.with_precision_and_scale(20, 3)
.unwrap();
- let array_ref = Arc::new(array);
+ let array_ref: ArrayRef = Arc::new(array);
let random_state = RandomState::with_seeds(0, 0, 0, 0);
let hashes_buff = &mut vec![0; array_ref.len()];
let hashes = create_hashes(&[array_ref], &random_state, hashes_buff)?;
@@ -478,15 +523,21 @@ mod tests {
let empty_array = FixedSizeListBuilder::new(StringBuilder::new(), 1).finish();
let random_state = RandomState::with_seeds(0, 0, 0, 0);
let hashes_buff = &mut vec![0; 0];
- let hashes = create_hashes(&[Arc::new(empty_array)], &random_state, hashes_buff)?;
+ let hashes = create_hashes(
+ &[Arc::new(empty_array) as ArrayRef],
+ &random_state,
+ hashes_buff,
+ )?;
assert_eq!(hashes, &Vec::<u64>::new());
Ok(())
}
#[test]
fn create_hashes_for_float_arrays() -> Result<()> {
- let f32_arr = Arc::new(Float32Array::from(vec![0.12, 0.5, 1f32, 444.7]));
- let f64_arr = Arc::new(Float64Array::from(vec![0.12, 0.5, 1f64, 444.7]));
+ let f32_arr: ArrayRef =
+ Arc::new(Float32Array::from(vec![0.12, 0.5, 1f32, 444.7]));
+ let f64_arr: ArrayRef =
+ Arc::new(Float64Array::from(vec![0.12, 0.5, 1f64, 444.7]));
let random_state = RandomState::with_seeds(0, 0, 0, 0);
let hashes_buff = &mut vec![0; f32_arr.len()];
@@ -514,8 +565,10 @@ mod tests {
Some(b"Longer than 12 bytes string"),
];
- let binary_array = Arc::new(binary.iter().cloned().collect::<$ARRAY>());
- let ref_array = Arc::new(binary.iter().cloned().collect::<BinaryArray>());
+ let binary_array: ArrayRef =
+ Arc::new(binary.iter().cloned().collect::<$ARRAY>());
+ let ref_array: ArrayRef =
+ Arc::new(binary.iter().cloned().collect::<BinaryArray>());
let random_state = RandomState::with_seeds(0, 0, 0, 0);
@@ -553,7 +606,7 @@ mod tests {
#[test]
fn create_hashes_fixed_size_binary() -> Result<()> {
let input_arg = vec![vec![1, 2], vec![5, 6], vec![5, 6]];
- let fixed_size_binary_array =
+ let fixed_size_binary_array: ArrayRef =
Arc::new(FixedSizeBinaryArray::try_from_iter(input_arg.into_iter()).unwrap());
let random_state = RandomState::with_seeds(0, 0, 0, 0);
@@ -580,8 +633,9 @@ mod tests {
Some("Longer than 12 bytes string"),
];
- let string_array = Arc::new(strings.iter().cloned().collect::<$ARRAY>());
- let dict_array = Arc::new(
+ let string_array: ArrayRef =
+ Arc::new(strings.iter().cloned().collect::<$ARRAY>());
+ let dict_array: ArrayRef = Arc::new(
strings
.iter()
.cloned()
@@ -629,8 +683,9 @@ mod tests {
fn create_hashes_for_dict_arrays() {
let strings = [Some("foo"), None, Some("bar"), Some("foo"), None];
- let string_array = Arc::new(strings.iter().cloned().collect::<StringArray>());
- let dict_array = Arc::new(
+ let string_array: ArrayRef =
+ Arc::new(strings.iter().cloned().collect::<StringArray>());
+ let dict_array: ArrayRef = Arc::new(
strings
.iter()
.cloned()
@@ -865,8 +920,9 @@ mod tests {
let strings1 = [Some("foo"), None, Some("bar")];
let strings2 = [Some("blarg"), Some("blah"), None];
- let string_array = Arc::new(strings1.iter().cloned().collect::<StringArray>());
- let dict_array = Arc::new(
+ let string_array: ArrayRef =
+ Arc::new(strings1.iter().cloned().collect::<StringArray>());
+ let dict_array: ArrayRef = Arc::new(
strings2
.iter()
.cloned()
@@ -896,4 +952,52 @@ mod tests {
assert_ne!(one_col_hashes, two_col_hashes);
}
+
+ #[test]
+ fn test_create_hashes_from_arrays() {
+ let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+ let float_array: ArrayRef =
+ Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));
+
+ let random_state = RandomState::with_seeds(0, 0, 0, 0);
+ let hashes_buff = &mut vec![0; int_array.len()];
+ let hashes =
+ create_hashes(&[int_array, float_array], &random_state, hashes_buff).unwrap();
+ assert_eq!(hashes.len(), 4,);
+ }
+
+ #[test]
+ fn test_create_hashes_from_dyn_arrays() {
+ let int_array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+ let float_array: ArrayRef =
+ Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0, 4.0]));
+
+ // Verify that we can call create_hashes with only &dyn Array
+ fn test(arr1: &dyn Array, arr2: &dyn Array) {
+ let random_state = RandomState::with_seeds(0, 0, 0, 0);
+ let hashes_buff = &mut vec![0; arr1.len()];
+ let hashes = create_hashes([arr1, arr2], &random_state, hashes_buff).unwrap();
+ assert_eq!(hashes.len(), 4,);
+ }
+ test(&*int_array, &*float_array);
+ }
+
+ #[test]
+ fn test_create_hashes_equivalence() {
+ let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4]));
+ let random_state = RandomState::with_seeds(0, 0, 0, 0);
+
+ let mut hashes1 = vec![0; array.len()];
+ create_hashes(
+ &[Arc::clone(&array) as ArrayRef],
+ &random_state,
+ &mut hashes1,
+ )
+ .unwrap();
+
+ let mut hashes2 = vec![0; array.len()];
+ create_hashes([array], &random_state, &mut hashes2).unwrap();
+
+ assert_eq!(hashes1, hashes2);
+ }
}
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index f2e18f7de8..52e0159111 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -878,10 +878,10 @@ impl Hash for ScalarValue {
fn hash_nested_array<H: Hasher>(arr: ArrayRef, state: &mut H) {
let len = arr.len();
- let arrays = vec![arr];
let hashes_buffer = &mut vec![0; len];
let random_state = ahash::RandomState::with_seeds(0, 0, 0, 0);
- let hashes = create_hashes(&arrays, &random_state, hashes_buffer).unwrap();
+ let hashes = create_hashes(&[arr], &random_state, hashes_buffer)
+ .expect("hash_nested_array: failed to create row hashes");
// Hash back to std::hash::Hasher
hashes.hash(state);
}
diff --git a/datafusion/physical-expr-common/src/binary_map.rs b/datafusion/physical-expr-common/src/binary_map.rs
index b37d9a7773..24bc430630 100644
--- a/datafusion/physical-expr-common/src/binary_map.rs
+++ b/datafusion/physical-expr-common/src/binary_map.rs
@@ -349,7 +349,7 @@ where
let batch_hashes = &mut self.hashes_buffer;
batch_hashes.clear();
batch_hashes.resize(values.len(), 0);
- create_hashes(&[Arc::clone(values)], &self.random_state, batch_hashes)
+ create_hashes([values], &self.random_state, batch_hashes)
// hash is supported for all types and create_hashes only
// returns errors for unsupported types
.unwrap();
diff --git a/datafusion/physical-expr-common/src/binary_view_map.rs b/datafusion/physical-expr-common/src/binary_view_map.rs
index 7ce943030a..2de563472c 100644
--- a/datafusion/physical-expr-common/src/binary_view_map.rs
+++ b/datafusion/physical-expr-common/src/binary_view_map.rs
@@ -19,6 +19,7 @@
//! `StringViewArray`/`BinaryViewArray`.
//! Much of the code is from `binary_map.rs`, but with simpler implementation because we directly use the
//! [`GenericByteViewBuilder`].
+use crate::binary_map::OutputType;
use ahash::RandomState;
use arrow::array::cast::AsArray;
use arrow::array::{Array, ArrayBuilder, ArrayRef, GenericByteViewBuilder};
@@ -28,8 +29,6 @@ use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};
use std::fmt::Debug;
use std::sync::Arc;
-use crate::binary_map::OutputType;
-
/// HashSet optimized for storing string or binary values that can produce that
/// the final set as a `GenericBinaryViewArray` with minimal copies.
#[derive(Debug)]
@@ -243,7 +242,7 @@ where
let batch_hashes = &mut self.hashes_buffer;
batch_hashes.clear();
batch_hashes.resize(values.len(), 0);
- create_hashes(&[Arc::clone(values)], &self.random_state, batch_hashes)
+ create_hashes([values], &self.random_state, batch_hashes)
// hash is supported for all types and create_hashes only
// returns errors for unsupported types
.unwrap();
diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs
index 0a582bd911..c552e6954c 100644
--- a/datafusion/physical-plan/src/joins/hash_join/exec.rs
+++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs
@@ -3452,11 +3452,7 @@ mod tests {
let random_state = RandomState::with_seeds(0, 0, 0, 0);
let hashes_buff = &mut vec![0; left.num_rows()];
- let hashes = create_hashes(
- &[Arc::clone(&left.columns()[0])],
- &random_state,
- hashes_buff,
- )?;
+ let hashes = create_hashes([&left.columns()[0]], &random_state, hashes_buff)?;
// Maps both values to both indices (1 and 2, representing input 0 and 1)
// 0 -> (0, 1)
@@ -3485,11 +3481,7 @@ mod tests {
let right_keys_values =
key_column.evaluate(&right)?.into_array(right.num_rows())?;
let mut hashes_buffer = vec![0; right.num_rows()];
- create_hashes(
- &[Arc::clone(&right_keys_values)],
- &random_state,
- &mut hashes_buffer,
- )?;
+ create_hashes([&right_keys_values], &random_state, &mut hashes_buffer)?;
let (l, r, _) = lookup_join_hashmap(
&join_hash_map,
@@ -3523,11 +3515,7 @@ mod tests {
let random_state = RandomState::with_seeds(0, 0, 0, 0);
let hashes_buff = &mut vec![0; left.num_rows()];
- let hashes = create_hashes(
- &[Arc::clone(&left.columns()[0])],
- &random_state,
- hashes_buff,
- )?;
+ let hashes = create_hashes([&left.columns()[0]], &random_state, hashes_buff)?;
hashmap_left.insert_unique(hashes[0], (hashes[0], 1u32), |(h, _)| *h);
hashmap_left.insert_unique(hashes[0], (hashes[0], 2u32), |(h, _)| *h);
@@ -3550,11 +3538,7 @@ mod tests {
let right_keys_values =
key_column.evaluate(&right)?.into_array(right.num_rows())?;
let mut hashes_buffer = vec![0; right.num_rows()];
- create_hashes(
- &[Arc::clone(&right_keys_values)],
- &random_state,
- &mut hashes_buffer,
- )?;
+ create_hashes([&right_keys_values], &random_state, &mut hashes_buffer)?;
let (l, r, _) = lookup_join_hashmap(
&join_hash_map,