Skip to content

Apache DataFusion SQL Query Engine

perf: unroll PrimitiveArray hashing for the no-null path (#26191)

RIchard Baahbcb33e27871c
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs
index f7a8b037fb..181cde28f9 100644
--- a/datafusion/common/src/hash_utils.rs
+++ b/datafusion/common/src/hash_utils.rs
@@ -321,15 +321,9 @@ fn hash_array_primitive<T>(
if array.null_count() == 0 {
if rehash {
- for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {
- let mut hasher = random_state.seeded_state(*hash).build_hasher();
- value.hash_write(&mut hasher);
- *hash = hasher.finish();
- }
+ hash_prim_rehash_dense(array.values(), hashes_buffer, random_state);
} else {
- for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {
- *hash = value.hash_one(random_state);
- }
+ hash_prim_fresh_dense(array.values(), hashes_buffer, random_state);
}
} else if rehash {
for i in array.nulls().unwrap().valid_indices() {
@@ -346,6 +340,53 @@ fn hash_array_primitive<T>(
}
}
+/// Overwrite each slot with `value.hash_one(state)`, 16 rows per iteration.
+/// Gathering + hashing a whole lane batch before storing keeps the independent
+/// hashes in flight so the pipeline isn't serialized on store-to-load deps.
+#[cfg(not(feature = "force_hash_collisions"))]
+#[inline(always)]
+fn hash_prim_fresh_dense<N, S>(values: &[N], hashes: &mut [u64], state: &S)
+where
+ N: HashValue + Copy,
+ S: HashState,
+{
+ const LANES: usize = 16;
+ let (value_chunks, value_tail) = values.as_chunks::<LANES>();
+ let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>();
+ for (h, v) in hash_chunks.iter_mut().zip(value_chunks) {
+ *h = v.map(|value| value.hash_one(state));
+ }
+ for (h, v) in hash_tail.iter_mut().zip(value_tail) {
+ *h = v.hash_one(state);
+ }
+}
+
+/// Fold `prev_hash` + `value` into a new hash, 8 rows per iteration. 8 is
+/// narrower than `hash_prim_fresh_dense` because each lane holds both `prev` and
+/// `value` live; widening past 8 spilled registers in the multi-column case.
+#[cfg(not(feature = "force_hash_collisions"))]
+#[inline(always)]
+fn hash_prim_rehash_dense<N, S>(values: &[N], hashes: &mut [u64], state: &S)
+where
+ N: HashValue + Copy,
+ S: HashState,
+{
+ const LANES: usize = 8;
+ let rehash_one = |prev_hash: u64, value: N| {
+ let mut hasher = state.seeded_state(prev_hash).build_hasher();
+ value.hash_write(&mut hasher);
+ hasher.finish()
+ };
+ let (value_chunks, value_tail) = values.as_chunks::<LANES>();
+ let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>();
+ for (h, v) in hash_chunks.iter_mut().zip(value_chunks) {
+ *h = std::array::from_fn(|lane| rehash_one(h[lane], v[lane]));
+ }
+ for (h, &v) in hash_tail.iter_mut().zip(value_tail) {
+ *h = rehash_one(*h, v);
+ }
+}
+
/// Hashes one array into the `hashes_buffer`
/// If `rehash==true` this combines the previous hash value in the buffer
/// with the new hash using `combine_hashes`