Skip to content

Apache DataFusion SQL Query Engine

perf: speed up DictionaryArray hashing (#26159)

RIchard Baahb40ef06a7aaa
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs
index a51278b455..46b86d1a05 100644
--- a/datafusion/common/src/hash_utils.rs
+++ b/datafusion/common/src/hash_utils.rs
@@ -543,11 +543,13 @@ fn hash_dictionary_scatter<
if let Some(key) = key {
let idx = key.as_usize();
if !HAS_NULL_VALUES || dict_values.is_valid(idx) {
- if MULTI_COL {
- *hash = combine_hashes(dict_hashes[idx], *hash);
+ // SAFETY: `idx` is a valid Arrow dict key, in-bounds for `dict_hashes`.
+ let dict_hash = unsafe { *dict_hashes.get_unchecked(idx) };
+ *hash = if MULTI_COL {
+ combine_hashes(dict_hash, *hash)
} else {
- *hash = dict_hashes[idx];
- }
+ dict_hash
+ };
}
}
}
@@ -555,11 +557,13 @@ fn hash_dictionary_scatter<
for (hash, key) in hashes_buffer.iter_mut().zip(array.keys().values()) {
let idx = key.as_usize();
if !HAS_NULL_VALUES || dict_values.is_valid(idx) {
- if MULTI_COL {
- *hash = combine_hashes(dict_hashes[idx], *hash);
+ // SAFETY: `idx` is a valid Arrow dict key, in-bounds for `dict_hashes`.
+ let dict_hash = unsafe { *dict_hashes.get_unchecked(idx) };
+ *hash = if MULTI_COL {
+ combine_hashes(dict_hash, *hash)
} else {
- *hash = dict_hashes[idx];
- }
+ dict_hash
+ };
}
}
}
@@ -1304,6 +1308,9 @@ mod tests {
use std::hash::{BuildHasherDefault, Hasher};
use std::sync::Arc;
+ #[cfg(not(feature = "force_hash_collisions"))]
+ use arrow::buffer::{NullBuffer, ScalarBuffer};
+
use super::*;
#[cfg(not(feature = "force_hash_collisions"))]
@@ -1856,8 +1863,6 @@ mod tests {
// Tests actual values of hashes, which are different if forcing collisions
#[cfg(not(feature = "force_hash_collisions"))]
fn create_hashes_for_list_view_arrays() {
- use arrow::buffer::{NullBuffer, ScalarBuffer};
-
// Create values array: [0, 1, 2, 3, null, 5]
let values = Arc::new(Int32Array::from(vec![
Some(0),
@@ -1906,8 +1911,6 @@ mod tests {
// Tests actual values of hashes, which are different if forcing collisions
#[cfg(not(feature = "force_hash_collisions"))]
fn create_hashes_for_large_list_view_arrays() {
- use arrow::buffer::{NullBuffer, ScalarBuffer};
-
// Create values array: [0, 1, 2, 3, null, 5]
let values = Arc::new(Int32Array::from(vec![
Some(0),