Apache DataFusion SQL Query Engine
Refactor ListArray hashing to consider only sliced values (#19500)
Jeffrey Vo7cf63f1b032f
diff --git a/Cargo.lock b/Cargo.lockindex b29a1f9903..63d32357b8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1898,6 +1898,7 @@ dependencies = ["hex","indexmap 2.13.0","insta",+ "itertools 0.14.0","libc","log","object_store",diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.tomlindex 710996707a..82e7aafcee 100644--- a/datafusion/common/Cargo.toml+++ b/datafusion/common/Cargo.toml@@ -72,6 +72,7 @@ half = { workspace = true }hashbrown = { workspace = true }hex = { workspace = true, optional = true }indexmap = { workspace = true }+itertools = { workspace = true }libc = "0.2.180"log = { workspace = true }object_store = { workspace = true, optional = true }diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rsindex 6631a90163..19c251c1a1 100644--- a/datafusion/common/src/hash_utils.rs+++ b/datafusion/common/src/hash_utils.rs@@ -23,6 +23,7 @@ use arrow::array::*;use arrow::datatypes::*;#[cfg(not(feature = "force_hash_collisions"))]use arrow::{downcast_dictionary_array, downcast_primitive_array};+use itertools::Itertools;#[cfg(not(feature = "force_hash_collisions"))]use crate::cast::{@@ -514,24 +515,41 @@ fn hash_list_array<OffsetSize>(whereOffsetSize: OffsetSizeTrait,{- let values = array.values();- let offsets = array.value_offsets();- let nulls = array.nulls();- let mut values_hashes = vec![0u64; values.len()];- create_hashes([values], random_state, &mut values_hashes)?;- if let Some(nulls) = nulls {- for (i, (start, stop)) in offsets.iter().zip(offsets.iter().skip(1)).enumerate() {- if nulls.is_valid(i) {+ // In case values is sliced, hash only the bytes used by the offsets of this ListArray+ let first_offset = array.value_offsets().first().cloned().unwrap_or_default();+ let last_offset = array.value_offsets().last().cloned().unwrap_or_default();+ let value_bytes_len = (last_offset - first_offset).as_usize();+ let mut values_hashes = vec![0u64; value_bytes_len];+ create_hashes(+ [array+ .values()+ .slice(first_offset.as_usize(), value_bytes_len)],+ random_state,+ &mut values_hashes,+ )?;++ if array.null_count() > 0 {+ for (i, (start, stop)) in array.value_offsets().iter().tuple_windows().enumerate()+ {+ if array.is_valid(i) {let hash = &mut hashes_buffer[i];- for values_hash in &values_hashes[start.as_usize()..stop.as_usize()] {+ for values_hash in &values_hashes[(*start - first_offset).as_usize()+ ..(*stop - first_offset).as_usize()]+ {*hash = combine_hashes(*hash, *values_hash);}}}} else {- for (i, (start, stop)) in offsets.iter().zip(offsets.iter().skip(1)).enumerate() {- let hash = &mut hashes_buffer[i];- for values_hash in &values_hashes[start.as_usize()..stop.as_usize()] {+ for ((start, stop), hash) in array+ .value_offsets()+ .iter()+ .tuple_windows()+ .zip(hashes_buffer.iter_mut())+ {+ for values_hash in &values_hashes+ [(*start - first_offset).as_usize()..(*stop - first_offset).as_usize()]+ {*hash = combine_hashes(*hash, *values_hash);}}@@ -1176,6 +1194,30 @@ mod tests {assert_eq!(hashes[1], hashes[6]); // null vs empty list}+ #[test]+ #[cfg(not(feature = "force_hash_collisions"))]+ fn create_hashes_for_sliced_list_arrays() {+ let data = vec![+ Some(vec![Some(0), Some(1), Some(2)]),+ None,+ // Slice from here+ Some(vec![Some(3), None, Some(5)]),+ Some(vec![Some(3), None, Some(5)]),+ None,+ // To here+ Some(vec![Some(0), Some(1), Some(2)]),+ Some(vec![]),+ ];+ let list_array =+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(data)) as ArrayRef;+ let list_array = list_array.slice(2, 3);+ let random_state = RandomState::with_seeds(0, 0, 0, 0);+ let mut hashes = vec![0; list_array.len()];+ create_hashes(&[list_array], &random_state, &mut hashes).unwrap();+ assert_eq!(hashes[0], hashes[1]);+ assert_ne!(hashes[1], hashes[2]);+ }+#[test]// Tests actual values of hashes, which are different if forcing collisions#[cfg(not(feature = "force_hash_collisions"))]