Apache DataFusion SQL Query Engine
perf: unroll PrimitiveArray hashing for the no-null path (#26191)
RIchard Baahbcb33e27871c
diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex f7a8b037fb..181cde28f9 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -321,15 +321,9 @@ fn hash_array_primitive<T>(if array.null_count() == 0 {if rehash {- for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {- let mut hasher = random_state.seeded_state(*hash).build_hasher();- value.hash_write(&mut hasher);- *hash = hasher.finish();- }+ hash_prim_rehash_dense(array.values(), hashes_buffer, random_state);} else {- for (hash, &value) in hashes_buffer.iter_mut().zip(array.values().iter()) {- *hash = value.hash_one(random_state);- }+ hash_prim_fresh_dense(array.values(), hashes_buffer, random_state);}} else if rehash {for i in array.nulls().unwrap().valid_indices() {@@ -346,6 +340,53 @@ fn hash_array_primitive<T>(}}+/// Overwrite each slot with `value.hash_one(state)`, 16 rows per iteration.+/// Gathering + hashing a whole lane batch before storing keeps the independent+/// hashes in flight so the pipeline isn't serialized on store-to-load deps.+#[cfg(not(feature = "force_hash_collisions"))]+#[inline(always)]+fn hash_prim_fresh_dense<N, S>(values: &[N], hashes: &mut [u64], state: &S)+where+ N: HashValue + Copy,+ S: HashState,+{+ const LANES: usize = 16;+ let (value_chunks, value_tail) = values.as_chunks::<LANES>();+ let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>();+ for (h, v) in hash_chunks.iter_mut().zip(value_chunks) {+ *h = v.map(|value| value.hash_one(state));+ }+ for (h, v) in hash_tail.iter_mut().zip(value_tail) {+ *h = v.hash_one(state);+ }+}++/// Fold `prev_hash` + `value` into a new hash, 8 rows per iteration. 8 is+/// narrower than `hash_prim_fresh_dense` because each lane holds both `prev` and+/// `value` live; widening past 8 spilled registers in the multi-column case.+#[cfg(not(feature = "force_hash_collisions"))]+#[inline(always)]+fn hash_prim_rehash_dense<N, S>(values: &[N], hashes: &mut [u64], state: &S)+where+ N: HashValue + Copy,+ S: HashState,+{+ const LANES: usize = 8;+ let rehash_one = |prev_hash: u64, value: N| {+ let mut hasher = state.seeded_state(prev_hash).build_hasher();+ value.hash_write(&mut hasher);+ hasher.finish()+ };+ let (value_chunks, value_tail) = values.as_chunks::<LANES>();+ let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<LANES>();+ for (h, v) in hash_chunks.iter_mut().zip(value_chunks) {+ *h = std::array::from_fn(|lane| rehash_one(h[lane], v[lane]));+ }+ for (h, &v) in hash_tail.iter_mut().zip(value_tail) {+ *h = rehash_one(*h, v);+ }+}+/// Hashes one array into the `hashes_buffer`/// If `rehash==true` this combines the previous hash value in the buffer/// with the new hash using `combine_hashes`